Fix Avro conversion for nullable avro arrays types (#167)

Co-authored-by: Jack Dingilian <[email protected]>
spotify · Aug 23, 2021 · a904733 · a904733
1 parent 558f295
commit a904733
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 2 deletions.
diff --git a/elitzur-avro/src/main/scala/com/spotify/elitzur/converters/avro/AvroConverter.scala b/elitzur-avro/src/main/scala/com/spotify/elitzur/converters/avro/AvroConverter.scala
@@ -297,6 +297,7 @@ final private[elitzur] case class DerivedConverter[T] private(caseClass: CaseCla
           val candidateSchema = innerSchema.getTypes.asScala.find(_.getType != Schema.Type.NULL)
           candidateSchema.map(_.getType) match {
             case Some(Schema.Type.RECORD) => candidateSchema.get
+            case Some(Schema.Type.ARRAY) => candidateSchema.get
             case _ => innerSchema
           }
         case _ => innerSchema

diff --git a/elitzur-scio/src/main/avro/RepeatedRecord.avsc b/elitzur-scio/src/main/avro/RepeatedRecord.avsc
@@ -35,6 +35,30 @@
           "doc": "repeated string field"
         }
       }
+    },
+    {
+      "name": "optional_repeated_record",
+      "type": ["null", {
+        "type": "array",
+        "items": {
+          "type": "record",
+          "name": "RepeatedInnerOptionalRecord",
+          "doc": "repeated record field",
+          "fields": [
+            {
+              "name": "string_field",
+              "type": "string",
+              "doc": "inner string field"
+            },
+            {
+              "name": "long_field",
+              "type": "long",
+              "doc": "inner long field"
+            }
+          ]
+        }
+      }],
+      "default": null
     }
   ]
 }
diff --git a/elitzur-scio/src/test/scala/com/spotify/elitzur/AvroTest.scala b/elitzur-scio/src/test/scala/com/spotify/elitzur/AvroTest.scala
@@ -79,7 +79,8 @@ object AvroTestCaseClasses {
   case class RepeatedInnerRecordCC(stringField: CountryCodeTesting,
                                    longField: NonNegativeLongTesting)
   case class RepeatedRecordCC(repeatedRecord: Seq[RepeatedInnerRecordCC],
-                              repeatedField: List[CountryCodeTesting])
+                              repeatedField: List[CountryCodeTesting],
+                              optionalRepeatedRecord: Option[List[RepeatedInnerRecordCC]])
   case class PartialRepeated(repeatedField: List[CountryCodeTesting])
   case class WithEnumField(enumTest: TestEnum)
 
@@ -152,7 +153,9 @@ class AvroTest extends PipelineSpec {
         nnl2 <- Gen.chooseNum(0, 150).map(NonNegativeLongTesting(_))
       } yield RepeatedRecordCC(
         Seq(RepeatedInnerRecordCC(cc, nnl), RepeatedInnerRecordCC(cc2, nnl2)),
-        List(cc, cc2))
+        List(cc, cc2),
+        Some(List(RepeatedInnerRecordCC(cc, nnl),  RepeatedInnerRecordCC(cc2, nnl2)))
+      )
     ).sample.get
 
     runWithContext { sc =>