1

I have an index created using the following pyelasticsearch code:

EDIT: UPDATED AGAIN 11/12/13 18:31 GMT

    entry_mapping = {
        'product': {
            'properties': {
                '_advertiser_id': {'type': 'integer'},
                'advertiser': {'type': 'string'},
                'category': {'type': 'string'},
                'created_at': {'type': 'date'},
                'description': {'type': 'string'},
                'fields': {
                    'type': 'nested',
                    'properties': {
                        'gender': {'type': 'string'},
                        'short_type': {'type': 'string'}
                    }
                },
                'id': {'type': 'string'},
                'name': {'type': 'string'},
                'price': {'type': 'float'},
                'product_type': {'type': 'string'},
                'updated_at': {'type': 'date'},
                'variations': {'type': 'nested'},
            }
        }
    }

    es.create_index('product', settings={'mappings': entry_mapping})

Query mapping returned using curl -XGET localhost:9200/products/_mapping after data has been imported:

{
  "product" : {
    "product" : {
      "properties" : {
        "_advertiser_id" : {
          "type" : "integer"
        },
        "advertiser" : {
          "type" : "string"
        },
        "category" : {
          "type" : "string"
        },
        "created_at" : {
          "type" : "date",
          "format" : "dateOptionalTime"
        },
        "description" : {
          "type" : "string"
        },
        "fields" : {
          "type" : "nested",
          "properties" : {
            "gender" : {
              "type" : "string"
            },
            "short_type" : {
              "type" : "string"
            }
          }
        },
        "id" : {
          "type" : "string"
        },
        "images" : {
          "properties" : {
            "id" : {
              "type" : "string"
            },
            "url" : {
              "type" : "string"
            }
          }
        },
        "name" : {
          "type" : "string"
        },
        "price" : {
          "type" : "float"
        },
        "product_type" : {
          "type" : "string"
        },
        "updated_at" : {
          "type" : "date",
          "format" : "dateOptionalTime"
        },
        "variations" : {
          "type" : "nested",
          "properties" : {
            "colour" : {
              "type" : "string"
            },
            "female_tops" : {
              "type" : "string"
            },
            "image" : {
              "type" : "string"
            },
            "length" : {
              "type" : "string"
            },
            "size" : {
              "type" : "string"
            },
            "sleeve_length" : {
              "type" : "string"
            },
            "type" : {
              "type" : "string"
            },
            "zip_type" : {
              "type" : "string"
            }
          }
        }
      }
    }
  }
}

I am successfully querying using the following query:

curl -XGET 'http://127.0.0.1:9200/products/_search?size=100' -d '{"query": {"filtered": {"query": {"query_string": {"query": "t-shirt"}}}}}'

The following is an example result:

{
   "_index":"product",
   "_type":"product",
   "_id":"525adf3fd1f4677e32d0f996",
   "_score":0.034907393,
   "_source":{
      "category":"T-Shirts",
      "advertiser":"string",
      "product_type":"Clothing",
      "description":"string",
      "fields":{
         "gender":"M"
      },
      "created_at":"2013-10-07T13:24:03.182000",
      "variations":[
         {
            "colour":"Grey",
            "sleeve_length":"Short sleeved",
            "size":"S"
         },
         {
            "colour":"Grey",
            "sleeve_length":"Short sleeved",
            "size":"M"
         },
         {
            "colour":"Grey",
            "sleeve_length":"Short sleeved",
            "size":"L"
         }
      ],
      "updated_at":"2013-10-19T13:54:29.796000",
      "price":12.0,
      "images":[
         {
            "url":"https://s3-eu-west-1.amazonaws.com/...",
            "id":"525adf23d1f4677e32d0f994",
            "resource_uri":""
         },
         {
            "url":"https://s3-eu-west-1.amazonaws.com/...",
            "id":"525adf30d1f4677e32d0f995",
            "resource_uri":""
         }
      ],
      "_advertiser_id":4,
      "id":"525adf3fd1f4677e32d0f996",
      "name":"Fresh Charcoal"
   }
}

I am trying to execute the following query using pyelsticsearch.

    self.query = {
        'query': {
            'filtered': {
                'query': {
                    'query_string': {'query': self.query_str}
                },
                'filter': {
                    'and': [
                        {
                            'range': {
                                'price': {
                                    'gte': self.min_price,
                                    'lte': self.max_price
                                }
                            },
                        },
                        {
                            'terms': {
                                '_advertiser_id': self.advertisers,
                            },
                        },
                        {
                            'term': {
                                'fields.gender': self.gender.lower(),
                            },
                        },
                        {
                            'nested': {
                                'path': 'variations',
                                'query': {'match_all': {}},
                                'filter': {
                                    'and': [
                                        {
                                            'terms': {
                                                'variations.size': [s.lower() for s in self.sizes]
                                            },
                                        },
                                        {
                                            'term': {
                                                'variations.colour': self.colour.lower(),
                                            }
                                        }
                                    ]
                                }
                            }
                        },
                    ]
                },
            }
        }
    }

Unfortunately it fails to return any results when there is data matching the query. Any help would be greatly appreciated.

UPDATE: 12/12/13 11:40 GMT

Below is an example of the JSON produced by the query code above.

curl -XGET 'http://127.0.0.1:9200/product/_search?size=100' -d '
{
   "query":{
      "filtered":{
         "filter":{
            "and":[
               {
                  "range":{}
               },
               {
                  "terms":{
                     "_advertiser_id":[
                        7,
                        4
                     ]
                  }
               },
               {
                  "term":{
                     "fields.gender":"m"
                  }
               },
               {
                  "nested":{
                     "filter":{
                        "and":[
                           {
                              "terms":{
                                 "variations.size":[
                                    "xs",
                                    "s",
                                    "m",
                                    "l",
                                    "xl",
                                    "xxl"
                                 ]
                              }
                           },
                           {
                              "term":{
                                 "variations.colour":"black"
                              }
                           }
                        ]
                     },
                     "path":"variations",
                     "query":{
                        "match_all":{

                        }
                     }
                  }
               }
            ]
         },
         "query":{
            "query_string":{
               "query":"t-shirt"
            }
         }
      }
   }
}'

UDPATED: 12/12/13 11:51 GMT

Things get stranger. Having stripped down the query the following gives results.

curl -XGET 'http://127.0.0.1:9200/product/_search?size=100' -d '{
   "query":{
      "filtered":{
         "filter":{
            "and":[
               {
                  "nested":{
                     "filter":{
                        "an":[
                           {
                              "terms":{
                                 "variations.size":[
                                    "xs",
                                    "s",
                                    "m",
                                    "l",
                                    "xl",
                                    "xxl"
                                 ]
                              }
                           },
                           {
                              "term":{
                                 "variations.colour":"black"
                              }
                           }
                        ]
                     },
                     "path":"variations",
                     "query":{
                        "match_all":{

                        }
                     }
                  }
               }
            ]
         },
         "query":{
            "query_string":{
               "query":"t-shirt"
            }
         }
      }
   }
}'

Example result data from above query: 

{
   "_index":"product",
   "_type":"product",
   "_id":"525ade5ad1f4677e32d0f993",
   "_score":0.10493462,
   "_source":{
      "category":"T-Shirts",
      "advertiser":"...",
      "product_type":"Clothing",
      "description":"...",
      "fields":{
         "gender":"M"
      },
      "created_at":"2013-10-07T13:24:03.182000",
      "variations":[
         {
            "colour":"Black",
            "sleeve_length":"Short sleeved",
            "size":"S"
         },
         {
            "colour":"Black",
            "sleeve_length":"Short sleeved",
            "size":"M"
         },
         {
            "colour":"Black",
            "sleeve_length":"Short sleeved",
            "size":"L"
         }
      ],
      "updated_at":"2013-10-19T14:05:34.299000",
      "price":0.0,
      "images":[
         {
            "url":"...",
            "id":"525ade50d1f4677e30a2cb3a",
            "resource_uri":""
         }
      ],
      "_advertiser_id":4,
      "id":"525ade5ad1f4677e32d0f993",
      "name":"..."
   }
}

*UPDATED: 21/12/2012 10:48 GMT *

I have isolated the part of the query that's being problematic—i.e. not returning any results—when combined with the entirety of the query.

{
    'term': {
        'fields.gender': self.gender.lower(),
    },
}

Exemplar working query:

curl -XGET 'http://127.0.0.1:9200/product/_search?size=100' -d '{
   "query":{
      "filtered":{
         "filter":{
            "and":[
               {
                  "range":{
                     "price":{
                        "gte":0.0,
                        "lte":200.0
                     }
                  }
               },
               {
                  "terms":{
                     "_advertiser_id":[
                        7,
                        4
                     ]
                  }
               },
               {
                  "nested":{
                     "filter":{
                        "and":[
                           {
                              "terms":{
                                 "variations.size":[
                                    "xs",
                                    "s",
                                    "m",
                                    "l",
                                    "xl",
                                    "xxl"
                                 ]
                              }
                           },
                           {
                              "term":{
                                 "variations.colour":"black"
                              }
                           }
                        ]
                     },
                     "path":"variations",
                     "query":{
                        "match_all":{

                        }
                     }
                  }
               }
            ]
         },
         "query":{
            "query_string":{
               "query":"t-shirt"
            }
         }
      }
   }
}'

Exemplar unworking query:

curl -XGET 'http://127.0.0.1:9200/product/_search?size=100' -d '{
   "query":{
      "filtered":{
         "filter":{
            "and":[
               {
                  "range":{
                     "price":{
                        "gte":0.0,
                        "lte":200.0
                     }
                  }
               },
               {
                  "terms":{
                     "_advertiser_id":[
                        7,
                        4
                     ]
                  }
               },
               {
                  "term":{
                     "fields.gender":"m"
                  }
               },
               {
                  "nested":{
                     "filter":{
                        "and":[
                           {
                              "terms":{
                                 "variations.size":[
                                    "xs",
                                    "s",
                                    "m",
                                    "l",
                                    "xl",
                                    "xxl"
                                 ]
                              }
                           },
                           {
                              "term":{
                                 "variations.colour":"black"
                              }
                           }
                        ]
                     },
                     "path":"variations",
                     "query":{
                        "match_all":{

                        }
                     }
                  }
               }
            ]
         },
         "query":{
            "query_string":{
               "query":"t-shirt"
            }
         }
      }
   }
}'
Prydie
  • 1,807
  • 1
  • 20
  • 30
  • Query looks correct to me. Have you installed the elasticsearch head plugin or any equivalent to check the index mapping? Alternatively curl the index to see if the mapping is correct: `curl -XGET localhost:9200/indexname/_mapping` – MeiSign Nov 26 '13 at 13:06
  • Thanks @MeiSign. The issue with price was an issue with the index. I have updated the post with the index information and a new exception as the full query is still not working. Thanks again, Andrew. – Prydie Nov 26 '13 at 13:25
  • My bad, I removed my wrong answer. Indeed it seems like you are indexing them as nested. I have no other idea and cant debug here at the moment. Sorry – MeiSign Nov 26 '13 at 13:44
  • 1
    i think you are mixing properties (object) and nested documents. – phoet Nov 26 '13 at 14:52
  • I'm not sure I understand what you mean @phoet. Can you expand? – Prydie Nov 26 '13 at 16:04
  • @phoet is correct here. Take a look here: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-nested-type.html. Your defining an "object" type. Not a "nested" type. You have to explicitly set the nested type in the mapping, see the example in the link – James R Nov 26 '13 at 16:06
  • I noticed you updated the mapping, you will have to either create a field with a new name (not the old name) or drop the index and recreate with the old name of the field. – James R Dec 10 '13 at 18:59
  • Not sure what you mean @JamesR. I've got more of the query working now. It's only the nested variations section that is broken. Any help would be amazing. – Prydie Dec 11 '13 at 14:25
  • the "fields" field above should have the "type":"nested", but I don't see that. Therefore, elasticsearch indexes objects there is the "object" type, which is the default for elasticsearch if you don't define a type when you index an object to the field. – James R Dec 11 '13 at 17:22
  • try something like this `mapping = {"settings":{},"mappings":{"mytest":{"properties":{"first_field":{"type":"string"},"fields":{"type":"nested","properties":{"first_name":{"type":"string"}}}}}}} es.create_index('new_test',settings=mapping) ` – James R Dec 11 '13 at 17:23
  • Changing the way the mapping has fixed the issue with the nesting, however, the query is now not returning any results. I've updated the post. If you could have a look it would be most appreciated. Thanks for your help :-) – Prydie Dec 11 '13 at 18:40
  • I've updated the post again with the final hurdle. If you wouldn't mind taking a look chaps it would be most appreciated. – Prydie Dec 21 '13 at 12:51
  • It looks like ES does not support queries in inner fields (object types), which sucks. At least I find trouble implementing that. I assume that `"fields.gender"` is the problem in your code. I opened a relevant [issue on github](https://github.com/elasticsearch/elasticsearch/issues/5488) and wait to see. – Diolor Mar 22 '14 at 02:27

2 Answers2

1

The mapping you get back from elasticsearch doesn't show your "fields" and "variations" nodes as nested. Make sure you put the mapping before indexing anything into this type. Delete the index, create the index with the correct mapping and only then index your objects.

Edit

After looking at your updated info, maybe the empty range filter you have on your original query is what is filtering out all results?
Also - your "fields" field is also supposed to be nested, is it really? You query doesn't treat it as nested.

Rotem Hermon
  • 2,127
  • 15
  • 19
  • I'm pretty sure this is what I'm doing. I'm deleting the index, creating it with the code above and then populating it using a custom Django management command. – Prydie Dec 11 '13 at 14:27
  • Again, in the mapping you get back from Elasticsearch that you posted in the question (the one you get from curl -XGET localhost:9200/products/_mapping), the variation node is *not* marked as nested. Make sure the mapping in Elasticsearch really has it as nested. – Rotem Hermon Dec 11 '13 at 14:38
  • The index creation code has it set as tested though doesn't it? i.e. `'variations': {'type': 'nested'}` – Prydie Dec 11 '13 at 15:13
  • Yes but the mapping you posted *doesn't* have it as nested. – Rotem Hermon Dec 11 '13 at 15:58
  • Ah sorry, I was reading that completely wrong. Can you think of any reason that it wouldn't be marked as nested despite me specifically marking it as such? – Prydie Dec 11 '13 at 16:59
  • It can happen if you index before you put the mapping, then Elasticsearch automatically creates the mapping and you can't alter it to nested after it's created. To make sure you need to delete the index or create a new one, put the mapping, get the mapping to verify it's really what you think it is, and only then index. – Rotem Hermon Dec 11 '13 at 19:37
  • I think I fixed that issue and have updated the post, however, I still don't get any results for my query. Any ideas? – Prydie Dec 11 '13 at 21:15
  • try to simplify your query. Do a search with only a simple nested query on variations and see that you're getting results back. Then try to expand your query with more fields and see when you're starting to get unexpected results. Also, post the actual JSON query that is issued against Elasticsearch. Your pyelsticsearch code doesn't help much. – Rotem Hermon Dec 11 '13 at 21:29
  • I've updated the post to add an example of the JSON output. Does this help at all? I'll try simplifying the query now. Thanks again. – Prydie Dec 12 '13 at 11:43
0

I think you need to set the nested fields as type nested, like this:

{
   "products":{
      "product":{
         "properties":{
            "_advertiser_id":{
               "type":"long"
            },
            "advertiser":{
               "type":"string"
            },
            "category":{
               "type":"string"
            },
            "created_at":{
               "type":"date",
               "format":"dateOptionalTime"
            },
            "description":{
               "type":"string"
            },
            "fields":{
               "type" : "nested",
               "properties":{
                  "gender":{
                     "type":"string"
                  },
                  "short_type":{
                     "type":"string"
                  }
               }
            },
            "id":{
               "type":"string"
            },
            "images":{
               "type" : "nested",
               "properties":{
                  "id":{
                     "type":"string"
                  },
                  "url":{
                     "type":"string"
                  }
               }
            },
            "name":{
               "type":"string"
            },
            "price":{
               "type":"double"
            },
            "product_type":{
               "type":"string"
            },
            "updated_at":{
               "type":"date",
               "format":"dateOptionalTime"
            },
            "variations":{
               "type" : "nested", 
               "properties":{
                  "colour":{
                     "type":"string"
                  },
                  "female_tops":{
                     "type":"string"
                  },
                  "image":{
                     "type":"string"
                  },
                  "length":{
                     "type":"string"
                  },
                  "size":{
                     "type":"string"
                  },
                  "sleeve_length":{
                     "type":"string"
                  },
                  "type":{
                     "type":"string"
                  },
                  "zip_type":{
                     "type":"string"
                  }
               }
            }
         }
      }
   }
}
aash
  • 1,323
  • 1
  • 14
  • 22
  • How does this work when the nested structure has different fields in it (i.e. some products have a short_type while others might have a top_type for example). – Prydie Dec 05 '13 at 15:48