Skip to content

Commit

Permalink
add env var to disable duckdb compute
Browse files Browse the repository at this point in the history
- added SLING_DUCKDB_COMPUTE environment variable to allow disabling duckdb computation for tasks.  This allows for easier testing and debugging scenarios where duckdb may not be desired or available.
  • Loading branch information
flarco committed Jan 24, 2025
1 parent e414700 commit ccfc811
Showing 1 changed file with 4 additions and 0 deletions.
4 changes: 4 additions & 0 deletions core/sling/task.go
Original file line number Diff line number Diff line change
Expand Up @@ -421,6 +421,10 @@ func (t *TaskExecution) Cleanup() {
// at the moment, use duckdb only for parquet or partitioned
// target parquet or csv files
func (t *TaskExecution) shouldWriteViaDuckDB(uri string) bool {
if val := os.Getenv("SLING_DUCKDB_COMPUTE"); val != "" && !cast.ToBool(val) {
return false
}

if g.In(t.Config.Target.ObjectFileFormat(), dbio.FileTypeParquet, dbio.FileTypeCsv) && len(iop.ExtractPartitionFields(uri)) > 0 {
return true
}
Expand Down

0 comments on commit ccfc811

Please sign in to comment.