Hi all, having some trouble using stream_maps to f...
# plugins-general
r
Hi all, having some trouble using stream_maps to filter schema messages. According to the docs, setting a field to null should remove it from both the RECORD and SCHEMA messages. I'm only seeing it remove the field from RECORD messages. Is this expected or a bug? Example config:
Copy code
"stream_maps": {
  "bounces": {
    "INSERTTS": null
  }
}
Resulting schema:
Copy code
{
  "type": "SCHEMA",
  "stream": "bounces",
  "schema": {
    "properties": {
      "ACCTCODE": {
        "type": [
          "number",
          "null"
        ]
      },
      "OYBACCTCODE": {
        "type": [
          "number",
          "null"
        ]
      },
      "JOBCODE": {
        "type": [
          "number",
          "null"
        ]
      },
      "BATCHCODE": {
        "type": [
          "number",
          "null"
        ]
      },
      "SUBSCRIBERCODE": {
        "type": [
          "string",
          "null"
        ]
      },
      "SUBSCRIBERKEY": {
        "type": [
          "string",
          "null"
        ]
      },
      "EVENTDATE": {
        "type": [
          "string",
          "null"
        ]
      },
      "BOUNCECATCODE": {
        "type": [
          "number",
          "null"
        ]
      },
      "BOUNCECAT": {
        "type": [
          "string",
          "null"
        ]
      },
      "BOUNCESUBCATCODE": {
        "type": [
          "number",
          "null"
        ]
      },
      "BOUNCESUBCAT": {
        "type": [
          "string",
          "null"
        ]
      },
      "BOUNCETYPECODE": {
        "type": [
          "number",
          "null"
        ]
      },
      "BOUNCETYPE": {
        "type": [
          "string",
          "null"
        ]
      },
      "SMTPMSG": {
        "type": [
          "string",
          "null"
        ]
      },
      "SMTPCODE": {
        "type": [
          "number",
          "null"
        ]
      },
      "SMTPBOUNCEREASON": {
        "type": [
          "string",
          "null"
        ]
      },
      "SENT_UUID": {
        "type": [
          "string",
          "null"
        ]
      },
      "APPLICATIONID": {
        "type": [
          "string",
          "null"
        ]
      },
      "ACCOUNTNUMBER": {
        "type": [
          "string",
          "null"
        ]
      },
      "LEADID": {
        "type": [
          "string",
          "null"
        ]
      },
      "INSERTTS": {
        "format": "date-time",
        "type": [
          "string",
          "null"
        ]
      }
    },
    "type": "object"
  },
  "key_properties": []
}
Resulting record:
Copy code
{
  "type": "RECORD",
  "stream": "bounces",
  "record": {
    "ACCTCODE": 7217054,
    "OYBACCTCODE": null,
    "JOBCODE": 17767768,
    "BATCHCODE": 442,
    "SUBSCRIBERCODE": "xxx",
    "SUBSCRIBERKEY": "<mailto:xxx@xxx.com|xxx@xxx.com>",
    "EVENTDATE": "2022-05-02 06:16:36",
    "BOUNCECATCODE": 1,
    "BOUNCECAT": "xxx",
    "BOUNCESUBCATCODE": 2001,
    "BOUNCESUBCAT": "xxx",
    "BOUNCETYPECODE": 0,
    "BOUNCETYPE": "xxx",
    "SMTPMSG": "xxx",
    "SMTPCODE": 552,
    "SMTPBOUNCEREASON": "xxx",
    "SENT_UUID": "",
    "APPLICATIONID": "",
    "ACCOUNTNUMBER": null,
    "LEADID": ""
  },
  "time_extracted": "2022-05-03T20:25:32.844146Z"
}