I have the following config to get deleted entries from jdbc input and delete them from the ES index, I see that it runs and queries DB and gets entries but documents with those ids still exist in the index. What can be wrong here?
input {
jdbc {
statement => "SELECT id, ad FROM table WHERE last_updated > :sql_last_value AND date_deleted IS NOT NULL LIMIT :size OFFSET :offset"
jdbc_driver_class => "org.postgresql.Driver"
jdbc_connection_string => "${DB_URL}"
jdbc_user => "${DB_USER}"
jdbc_password => "${DB_PASSWORD}"
jdbc_driver_library => "/usr/share/logstash/vendor/jar/jdbc/postgresql-42.2.5.jar"
jdbc_page_size => 20000
jdbc_paging_enabled => true
jdbc_paging_mode => "explicit"
last_run_metadata_path => "/usr/share/logstash/opt/lastruns/deleted_last_run"
schedule => "*/15 * * * *"
# ... other configuration bits
}
}
output {
elasticsearch {
hosts => "${ES_HOST}"
index => "myindex"
document_id => "%{id}"
user => "elastic"
password => "${ELASTIC_PASSWORD}"
action => "delete"
}
}
Both elastic search and logstash are version 8.0.0.
