elasticsearch ingest pipeline to rename dynamic fields

Question

I have a fairly large document that i'm ingesting into elasticsearch (70-80 attributes). And there can be new fields that come in on a regular basis. I can use the rename operator to effectively rename each field but I was wondering if there was a more efficient way to dynamically rename every field using a script.

Current format for each field can be either all lower case separated by underscore or a combination of upper and lower case separated by underscore. I need to rename each field as follows -

(eg source field) TV_field_1 --> (needs to be renamed to) tvField1

(eg source field) feature_title_no --> (needs to be renamed to) featureTitleNo

Thanks

Answer 1

Since rename processor doesn't support replacing patterns , have written a custom script

PUT _ingest/pipeline/snakeCaseToCamelCase
{
  "description": "Convert snake case to camel case",
  "processors": [
    {
      "script": {
        "source": """
            // Iterate through all keys and create list of fields with snake case
            def loopAllFields(def x){
              def ret=[];
              if(x instanceof Map){
                for (entry in x.entrySet()) {
                  if (entry.getKey().indexOf("_")==0) { 
                    continue;
                  }
                  // Get value
                  def val=entry.getValue();
                  if(entry.getKey().indexOf("_")>-1)
                  {
                      ret.add(entry.getKey());
                  }
                  // If further object
                  if(val instanceof Map|| val instanceof HashMap)
                  {
                    def list =loopAllFields(val);
                    for(item in list)
                    {
                      ret.add(entry.getKey()+"."+ item);
                    }
                  }
                  // If array
                  if(val instanceof ArrayList)
                  {
                    for(v in val)
                    {
                       def list =loopAllFields(v);
                       for(item in list)
                       {
                          ret.add(entry.getKey()+"."+ item);
                       }
                    }
                  }
                }
              }
              return ret;
            }

            // Create a camel case field and delete snake case field
            def renameField(def ctx, def fieldName)
            {
               def str=splitText(fieldName,'.');
               if(str.length<=1)
               {
                  def newField=snakeToCamel(fieldName);
                  ctx[newField]=ctx[fieldName];
                  ctx.remove(fieldName);
               }
               else
               {
                   if(ctx[str[0]] instanceof ArrayList)
                   {
                       def fld=combineArray(str);
                       for(v in ctx[str[0]])
                       {
                          renameField(v,fld);
                       }
                   }
                   if(ctx[str[0]] instanceof Map)
                   {
                      def fld=combineArray(str);
                      renameField(ctx[str[0]],fld); 
                    }
               }
               return 1;
            }

            def combineArray(def str)
            {
              def fld="";
              for(int i=1;i<str.length;i++)
              {
                if(fld=="")
                {
                   fld+=str[i];
                }
                else
                {
                   fld+="."+str[i];
                }
              }
              return fld;
            }

            // Convert field name from snake case to camel case
            def snakeToCamel(def s){
              //def str=/_/.split(s);
              def str=splitText(s,'_');
              def fieldName="";
              for(int i=0;i<str.length;i++)
              {
                 if(i==0)
                 {
                   fieldName+=str[i].toLowerCase();
                 }
                 else{
                    if(str[i].length()==1)
                    {
                      fieldName+=str[i].substring(0,1).toUpperCase();
                    }else{
                      fieldName+=str[i].substring(0,1).toUpperCase()+str[i].substring(1,str[i].length());
                    }
                 }
              }
              return fieldName;
            }

            def splitText(def str, def seperator)
            {
              def pos= str.indexOf(seperator);
              def ret=[];
              while(pos>0)
              {
                 def split=str.substring(0,pos);
                 def rest= str.substring(pos+1,str.length());
                 ret.add(split);
                 pos=rest.indexOf(seperator);
                 if(pos==-1)
                 {
                   ret.add(rest);
                 }
                 str=rest;
              }
              return ret;
            }

            def fields=loopAllFields(ctx);
            fields.sort((s1, s2) -> s2.length() - s1.length());
            for(field in fields)
            {
              renameField(ctx,field);
            }

"""
      }
    }
  ]
}