{
  "type": "record",
  "name": "DataJobInputOutput",
  "namespace": "com.linkedin.pegasus2avro.datajob",
  "doc": "Information about the inputs and outputs of a Data processing job",
  "fields": [
    {
      "name": "inputDatasets",
      "type": {
        "type": "array",
        "items": "string"
      },
      "doc": "Input datasets consumed by the data job during processing",
      "Relationship": {
        "/*": {
          "entityTypes": [
            "dataset"
          ],
          "isLineage": true,
          "name": "Consumes"
        }
      },
      "Searchable": {
        "/*": {
          "fieldName": "inputs",
          "fieldType": "URN",
          "numValuesFieldName": "numInputDatasets",
          "queryByDefault": false
        }
      }
    },
    {
      "name": "outputDatasets",
      "type": {
        "type": "array",
        "items": "string"
      },
      "doc": "Output datasets produced by the data job during processing",
      "Relationship": {
        "/*": {
          "entityTypes": [
            "dataset"
          ],
          "isLineage": true,
          "isUpstream": false,
          "name": "Produces"
        }
      },
      "Searchable": {
        "/*": {
          "fieldName": "outputs",
          "fieldType": "URN",
          "numValuesFieldName": "numOutputDatasets",
          "queryByDefault": false
        }
      }
    },
    {
      "name": "inputDatajobs",
      "type": [
        "null",
        {
          "type": "array",
          "items": "string"
        }
      ],
      "doc": "Input datajobs that this data job depends on",
      "default": null,
      "Relationship": {
        "/*": {
          "entityTypes": [
            "dataJob"
          ],
          "isLineage": true,
          "name": "DownstreamOf"
        }
      }
    },
    {
      "name": "inputDatasetFields",
      "type": [
        "null",
        {
          "type": "array",
          "items": "string"
        }
      ],
      "doc": "Fields of the input datasets used by this job",
      "default": null,
      "Relationship": {
        "/*": {
          "entityTypes": [
            "schemaField"
          ],
          "name": "Consumes"
        }
      },
      "Searchable": {
        "/*": {
          "fieldName": "inputFields",
          "fieldType": "URN",
          "numValuesFieldName": "numInputFields",
          "queryByDefault": false
        }
      }
    },
    {
      "name": "outputDatasetFields",
      "type": [
        "null",
        {
          "type": "array",
          "items": "string"
        }
      ],
      "doc": "Fields of the output datasets this job writes to",
      "default": null,
      "Relationship": {
        "/*": {
          "entityTypes": [
            "schemaField"
          ],
          "name": "Produces"
        }
      },
      "Searchable": {
        "/*": {
          "fieldName": "outputFields",
          "fieldType": "URN",
          "numValuesFieldName": "numOutputFields",
          "queryByDefault": false
        }
      }
    },
    {
      "name": "fineGrainedLineages",
      "type": [
        "null",
        {
          "type": "array",
          "items": {
            "type": "record",
            "name": "FineGrainedLineage",
            "namespace": "com.linkedin.pegasus2avro.dataset",
            "doc": "A fine-grained lineage from upstream fields/datasets to downstream field(s)",
            "fields": [
              {
                "name": "upstreamType",
                "type": {
                  "type": "enum",
                  "name": "FineGrainedLineageUpstreamType",
                  "doc": "The type of upstream entity in a fine-grained lineage",
                  "symbols": [
                    "FIELD_SET",
                    "DATASET",
                    "NONE"
                  ],
                  "symbolDocs": {
                    "DATASET": " Indicates that this lineage is originating from upstream dataset(s)",
                    "FIELD_SET": " Indicates that this lineage is originating from upstream field(s)",
                    "NONE": " Indicates that there is no upstream lineage i.e. the downstream field is not a derived field"
                  }
                },
                "doc": "The type of upstream entity"
              },
              {
                "name": "upstreams",
                "type": [
                  "null",
                  {
                    "type": "array",
                    "items": "string"
                  }
                ],
                "doc": "Upstream entities in the lineage",
                "default": null
              },
              {
                "name": "downstreamType",
                "type": {
                  "type": "enum",
                  "name": "FineGrainedLineageDownstreamType",
                  "doc": "The type of downstream field(s) in a fine-grained lineage",
                  "symbols": [
                    "FIELD",
                    "FIELD_SET"
                  ],
                  "symbolDocs": {
                    "FIELD": " Indicates that the lineage is for a single, specific, downstream field",
                    "FIELD_SET": " Indicates that the lineage is for a set of downstream fields"
                  }
                },
                "doc": "The type of downstream field(s)"
              },
              {
                "name": "downstreams",
                "type": [
                  "null",
                  {
                    "type": "array",
                    "items": "string"
                  }
                ],
                "doc": "Downstream fields in the lineage",
                "default": null
              },
              {
                "name": "transformOperation",
                "type": [
                  "null",
                  "string"
                ],
                "doc": "The transform operation applied to the upstream entities to produce the downstream field(s)",
                "default": null
              },
              {
                "name": "confidenceScore",
                "type": "float",
                "doc": "The confidence in this lineage between 0 (low confidence) and 1 (high confidence)",
                "default": 1.0
              }
            ]
          }
        }
      ],
      "doc": "Fine-grained column-level lineages",
      "default": null
    }
  ],
  "Aspect": {
    "name": "dataJobInputOutput"
  }
}