Разверните схему avro, содержащую типы записей в полях - PullRequest
Новая
       4

Разверните схему avro, содержащую типы записей в полях

0 голосов
/ 29 января 2020

В Metri c .avs c, как развернуть все поля типа записи вместо объединения различных схем avro? Удалить все ссылки на "type": ["com.flex .avro. * "," null "] Я попытался сделать это, схема была проанализирована в библиотеке python spavro.

Я пишу с использованием оригинального Metri c .avs c в java и пытаюсь прочитать его в python с помощью spavro, но получаю ошибку " Failed to read from kafka topic Unable to process union schema ['string', 'null'], union index '4' doesn't exist. This is most likely because the read schema being used is not compatible with the write schema used to write the data. Traceback (most recent call last): "

Metri c .avs c

{
 "namespace": "com.flex.avro",
 "name": "Metric",
 "type": "record",
 "fields": [{
    "name": "groupId",
    "type": ["string", "null"]
    },{
    "name": "documentId",
    "type": ["string", "null"]
    },{
    "name": "MetricSource",
    "type": ["com.flex.avro.MetricSource", "null"]
    },{
    "name": "GreenHighway",
    "type": ["com.flex.avro.GreenHighway", "null"]
    },{
    "name": "IncidentEvent",
    "type": ["com.flex.avro.IncidentEvent", "null"]
    },{
    "name": "Notifier",
    "type": ["com.flex.avro.Notifier", "null"]
    },{
        "name" : "isPartOfCluster",
        "type" : "boolean",
        "doc" : "Flag to indicate that this is part of cluster or not",
        "default" : false
   },
   {
   "name": "metricPath",
   "type":{
       "type": "array",
       "items": ["string", "null"]
    }
   },{
   "name": "lastState",
   "type": ["string", "null"]
   },{
   "name": "logicalRoute",
   "type": ["null", "com.flex.avro.LogicalRoute" ]
   },{
    "name": "modelName",
    "type": ["null", "string"],
    "default": null
    },{
    "name": "scoringParamaters",
    "type": ["null", "string"],
    "default": null
    }
  ]
}

Metricsource.avs c: 

{
 "namespace": "com.flex.avro",
 "name": "MetricSource",
 "type": "record",
 "fields": [{
    "name": "policy",
    "type": ["string", "null"]
     },{
     "name": "encodedMetric",
     "type": ["string", "null"]
     },{
     "name": "entityMap",
        "type":{
            "type": "map",
            "values": "string"
         }
     },{
     "name": "algorithm",
     "type": ["string", "null"]
      },{
      "name": "metricClass",
      "order": "ignore",
      "type": ["null", "string"],
      "default": null
    }
    ]
 }

GreenHighway.avs c: 

{
 "namespace": "com.flex.avro",
 "name": "GreenHighway",
 "type": "record",
 "fields": [{
    "name": "percentile",
    "type": ["string", "null"]
    },{
    "name": "originalAlertCode",
    "type": ["null", "string"],
    "default": null
    }
 ]
}

Incidentevent.avs c

{
 "namespace": "com.flex.avro",
 "name": "IncidentEvent",
 "type": "record",
 "fields": [{
     "name": "previousSeverity",
     "type": "int"
     },{
      "name": "reasonForClosure",
      "type": "string",
      "default": "NA"
      },{
      "name": "newSeverity",
      "type": "int",
      "default": 0
      }

   ]
 }

Notifier.avs c: 

{
 "namespace": "com.flex.avro",
 "name": "Notifier",
 "type": "record",
 "fields": [{
    "name": "alertSuppressionCount",
    "type": "int"
    },
    {
    "name": "destination",
    "type": "string",
    "default": "opsmvs"
    }
  ]
}

LogicalRoute.avs c: 

{
  "namespace": "com.flex.avro",
  "name": "LogicalRoute",
  "type": "record",
  "fields": [
    {
      "name": "productName",
      "type": "string"
    },
    {
      "name": "routePlan",
      "type": "string"
    },
    {
      "name": "routePlanItem",
      "type": [
        "null",
        "string"
      ]
    },
    {
      "name": "actors",
      "type": {
        "type": "map",
        "values": "com.flex.avro.LogicalTopicGroup"
      }
    }
  ]
}

LogicalTopicGroup.avs c: 

{
  "namespace": "com.flex.avro",
  "name": "LogicalTopicGroup",
  "type": "record",
  "fields": [
    {
      "name": "logicalTopics",
      "type": {
        "type": "map",
        "values": "com.flex.avro.LogicalTopic"
        }
    }
  ]
}

LogicalTopi c .avs c: 

{
  "namespace": "com.flex.avro",
  "name": "LogicalTopic",
  "type": "record",
  "fields": [
    {
      "name": "topics",
      "type": {
        "type": "map",
        "values": "string"
      }
    }
  ]
}
...