Skip to content

Commit

Permalink
chore: postable pipeline validation for time parser
Browse files Browse the repository at this point in the history
  • Loading branch information
raj-k-singh committed Nov 29, 2023
1 parent 8daeb6f commit 058b4c0
Show file tree
Hide file tree
Showing 3 changed files with 66 additions and 1 deletion.
3 changes: 2 additions & 1 deletion pkg/query-service/app/logparsingpipeline/pipelineBuilder.go
Original file line number Diff line number Diff line change
Expand Up @@ -133,11 +133,12 @@ func getOperators(ops []PipelineOperator) ([]PipelineOperator, error) {
}

operator.If = fmt.Sprintf(
`%s && %s matches "%s"`, operator.If, parseFromPath, valueRegex,
`%s && string(%s) matches "%s"`, operator.If, parseFromPath, valueRegex,
)

}
// TODO(Raj): Maybe add support for gotime too eventually
operator.OnError = "send"
}

filteredOp = append(filteredOp, operator)
Expand Down
22 changes: 22 additions & 0 deletions pkg/query-service/app/logparsingpipeline/postablePipeline.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (

v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
"go.signoz.io/signoz/pkg/query-service/queryBuilderToExpr"
"golang.org/x/exp/slices"
)

// PostablePipelines are a list of user defined pielines
Expand Down Expand Up @@ -164,6 +165,27 @@ func isValidOperator(op PipelineOperator) error {
if len(op.Fields) == 0 {
return fmt.Errorf(fmt.Sprintf("fields of %s retain operator cannot be empty", op.ID))
}

case "time_parser":
if op.ParseFrom == "" {
return fmt.Errorf(fmt.Sprintf("parse from of time parsing processor %s cannot be empty", op.ID))
}
if op.LayoutType != "epoch" && op.LayoutType != "strptime" {
return fmt.Errorf(fmt.Sprintf(
"invalid format type '%s' of time parsing processor %s", op.LayoutType, op.ID,
))
}
if op.Layout == "" {
return fmt.Errorf(fmt.Sprintf("format can not be empty for time parsing processor %s", op.ID))
}

validEpochLayouts := []string{"s", "ms", "us", "ns", "s.ms", "s.us", "s.ns"}
if op.LayoutType == "epoch" && !slices.Contains(validEpochLayouts, op.Layout) {
return fmt.Errorf(fmt.Sprintf(
"invalid epoch format '%s' of time parsing processor %s", op.LayoutType, op.ID,
))
}

default:
return fmt.Errorf(fmt.Sprintf("operator type %s not supported for %s, use one of (grok_parser, regex_parser, copy, move, add, remove, trace_parser, retain)", op.Type, op.ID))
}
Expand Down
42 changes: 42 additions & 0 deletions pkg/query-service/app/logparsingpipeline/postablePipeline_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,48 @@ var operatorTest = []struct {
},
},
IsValid: false,
}, {
Name: "Timestamp Parser - valid",
Operator: PipelineOperator{
ID: "time",
Type: "time_parser",
ParseFrom: "attributes.test_timestamp",
LayoutType: "epoch",
Layout: "s",
},
IsValid: true,
}, {
Name: "Timestamp Parser - invalid - bad parsefrom attribute",
Operator: PipelineOperator{
ID: "time",
Type: "time_parser",
ParseFrom: "timestamp",
LayoutType: "epoch",
Layout: "s",
},
IsValid: false,
}, {
Name: "Timestamp Parser - unsupported layout_type",
Operator: PipelineOperator{
ID: "time",
Type: "time_parser",
ParseFrom: "attributes.test_timestamp",
// TODO(Raj): Maybe add support for gotime format
LayoutType: "gotime",
Layout: "Mon Jan 2 15:04:05 -0700 MST 2006",
},
IsValid: false,
}, {
Name: "Timestamp Parser - invalid epoch layout",
Operator: PipelineOperator{
ID: "time",
Type: "time_parser",
ParseFrom: "attributes.test_timestamp",
// TODO(Raj): Maybe add support for gotime format
LayoutType: "epoch",
Layout: "%Y-%m-%d",
},
IsValid: false,
},
}

Expand Down

0 comments on commit 058b4c0

Please sign in to comment.