The logstash service runs on an AWS ec2 instance, and also I have a pipe.conf file that has this config:
input {
jdbc {
jdbc_connection_string => "jdbc:sqlserver://xxxxx:xxxx;databaseName=xxx;encrypt=true;trustServerCertificate=true"
jdbc_user => "xxxx"
jdbc_password => "xxxxx"
jdbc_driver_class => "com.microsoft.sqlserver.jdbc.SQLServerDriver"
statement => "select fecha_bus, id_ecom_envio, clave_pedido , CURRENT_TIMESTAMP as event_db_timestamp from xxxxx WITH(NOLOCK) WHERE id_ecom_envio is not null and fecha_bus >= '2022-09-13' and fecha_bus < '2022-09-14' and fecha_bus > :sql_last_value ORDER BY fecha_bus ASC"
tracking_column => "fecha_bus"
tracking_column_type => "timestamp"
use_column_value => true
last_run_metadata_path => "/tmp/logstash_extras/last_fecha_bus"
schedule => "* * * * * *"
}
}
output {
stdout {}
file {
path => "/tmp/logstash_extras/out.json"
}
}
and the directory where I try to write has these permissions:
drwxrwxr-x 2 ec2-user ec2-user 6 feb 28 05:46 logstash_extras
The main issue happens when I start the logstash service and therefore the pipeline starts to run and I always get this message from the logs:
[2023-02-28T05:51:06,343][ERROR][logstash.inputs.jdbc ][main][b1e5f988c36abb3b3305b180e61d334701780eddc70e4edfeaf536e1c3939b5f] Scheduler intercepted an error: {:exception=>Errno::EACCES, :message=>"Permission denied - /tmp/logstash_extras/last_fecha_bus", :backtrace=>["org/jruby/RubyIO.java:1227:in `sysopen'", "org/jruby/RubyIO.java:4012:in `write'", "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-integration-jdbc-5.4.1/lib/logstash/plugin_mixins/jdbc/value_tracking.rb:119:in `write'", "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-integration-jdbc-5.4.1/lib/logstash/plugin_mixins/jdbc/value_tracking.rb:43:in `write'", "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-integration-jdbc-5.4.1/lib/logstash/inputs/jdbc.rb:373:in `execute_query'", "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-integration-jdbc-5.4.1/lib/logstash/inputs/jdbc.rb:334:in `block in run'", "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/rufus-scheduler-3.8.2/lib/rufus/scheduler/jobs_core.rb:233:in `block in do_call'", "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/rufus-scheduler-3.8.2/lib/rufus/scheduler.rb:130:in `around_trigger'", "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/rufus-scheduler-3.8.2/lib/rufus/scheduler/jobs_core.rb:232:in `do_call'", "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/rufus-scheduler-3.8.2/lib/rufus/scheduler/jobs_core.rb:280:in `trigger_now'", "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/rufus-scheduler-3.8.2/lib/rufus/scheduler/jobs_core.rb:331:in `block in start_work_thread'", "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/rufus-scheduler-3.8.2/lib/rufus/scheduler/jobs_core.rb:334:in `block in start_work_thread'", "org/jruby/RubyKernel.java:1507:in `loop'", "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/rufus-scheduler-3.8.2/lib/rufus/scheduler/jobs_core.rb:317:in `block in start_work_thread'"], :now=>"ot 2023-02-28 05:51:06 +00:00 dst:false", :last_time=>"ot 2023-02-28 05:51:06 +00:00 dst:false", :next_time=>"ot 2023-02-28 05:51:07 +00:00 dst:false", :job=>#<Rufus::Scheduler::CronJob:0x25c9b816 @last_at=nil, @previous_time=#<EtOrbi::EoTime:0x2f38cc93 @time=nil, @zone=#<TZInfo::LinkedTimezone: UTC>, @seconds=1677563466.0>, @scheduled_at=#<EtOrbi::EoTime:0x7155158d @time=nil, @zone=#<TZInfo::LinkedTimezone: UTC>, @seconds=1677563465.071953>, @cron_line=#<Fugit::Cron:0x4f9f9803 @weekdays=nil, @timezone=nil, @day_and=nil, @zone=nil, @seconds=nil, @minutes=nil, @cron_s=nil, @hours=nil, @monthdays=nil, @months=nil, @original="* * * * * *">, @last_time=#<EtOrbi::EoTime:0x27fe90ee @time=2023-02-28 05:51:06 +0000, @zone=#<TZInfo::LinkedTimezone: UTC>, @seconds=1677563466.083214>, @times=nil, @locals={}, @unscheduled_at=nil, @local_mutex=#<Thread::Mutex:0x17eb11ff>, @mean_work_time=0.0, @last_work_time=0.0, @count=1, @opts={}, @id="cron_1677563465.071953_4104", @handler=#<Proc:0x2407dc81@/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-integration-jdbc-5.4.1/lib/logstash/inputs/jdbc.rb:334>, @resume_discard_past=nil, @original="* * * * * *", @tags=[], @callable=#<Proc:0x2407dc81@/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-integration-jdbc-5.4.1/lib/logstash/inputs/jdbc.rb:334>, @next_time=#<EtOrbi::EoTime:0x77ad410f @time=2023-02-28 05:51:07 +0000, @zone=#<TZInfo::LinkedTimezone: UTC>, @seconds=1677563467.0>, @name=nil, @scheduler=#<LogStash::PluginMixins::Scheduler::RufusImpl::SchedulerImpl:0x65925e00 @jobs=#<Rufus::Scheduler::JobArray:0x2a39cfa4 @mutex=#<Thread::Mutex:0x30c0fdb6>, @array=[#<Rufus::Scheduler::CronJob:0x25c9b816 ...>]>, @scheduler_lock=#<Rufus::Scheduler::NullLock:0x5aed4e79>, @started_at=#<EtOrbi::EoTime:0x3e4dfe6f @time=nil, @zone=#<TZInfo::LinkedTimezone: UTC>, @seconds=1677563465.065735>, @thread=#<Thread:0x1fe2b337@[main]|input|Jdbc|scheduler@/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/rufus-scheduler-3.8.2/lib/rufus/scheduler.rb:634 sleep>, @mutexes={}, @discard_past=true, @join_queue=#<Thread::Queue:0x957995d>, @work_queue=#<Thread::Queue:0x67fdfb48>, @frequency=1.0, @_work_threads=[#<Thread:0x63ccd5b@[main]|input|Jdbc|scheduler_worker-00@/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/rufus-scheduler-3.8.2/lib/rufus/scheduler/jobs_core.rb:307 run>], @trigger_lock=#<Rufus::Scheduler::NullLock:0x331431b0>, @paused_at=nil, @opts={:thread_name=>"[main]|input|Jdbc|scheduler", :max_work_threads=>1, :frequency=>1.0}, @thread_key="rufus_scheduler_4100", @_logger=#<LogStash::Logging::Logger:0x735abab1>, @max_work_threads=1, @stderr=#<IO:<STDERR>>>, @first_at=nil, @paused_at=nil>, :opts=>{:thread_name=>"[main]|input|Jdbc|scheduler", :max_work_threads=>1, :frequency=>1.0}, :started_at=>#<EtOrbi::EoTime:0x3e4dfe6f @time=nil, @zone=#<TZInfo::LinkedTimezone: UTC>, @seconds=1677563465.065735>, :thread=>"#<Thread:0x1fe2b337@[main]|input|Jdbc|scheduler@/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/rufus-scheduler-3.8.2/lib/rufus/scheduler.rb:634 sleep>", :jobs_size=>1, :work_threads_size=>1, :work_queue_size=>0}
[2023-02-28T05:51:06,498][INFO ][logstash.outputs.file ][main][becfe7d96044e231ddddc84d3c13fe5d22ec265ccaadf100ff252e8d1b0c8c74] Opening file {:path=>"/tmp/logstash_extras/out.json"}
[2023-02-28T05:51:06,504][ERROR][logstash.javapipeline ][main] Pipeline worker error, the pipeline will be stopped {:pipeline_id=>"main", :error=>"(EACCES) Permission denied - /tmp/logstash_extras/out.json", :exception=>Java::OrgJrubyExceptions::SystemCallError, :backtrace=>["org.jruby.RubyIO.sysopen(org/jruby/RubyIO.java:1227)", "org.jruby.RubyFile.initialize(org/jruby/RubyFile.java:362)", "org.jruby.RubyClass.new(org/jruby/RubyClass.java:911)", "org.jruby.RubyIO.new(org/jruby/RubyIO.java:868)", "RUBY.open(/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-output-file-4.3.0/lib/logstash/outputs/file.rb:276)", "RUBY.multi_receive_encoded(/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-output-file-4.3.0/lib/logstash/outputs/file.rb:119)", "org.jruby.RubyHash.each(org/jruby/RubyHash.java:1519)", "RUBY.multi_receive_encoded(/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-output-file-4.3.0/lib/logstash/outputs/file.rb:118)", "org.jruby.ext.thread.Mutex.synchronize(org/jruby/ext/thread/Mutex.java:171)", "RUBY.multi_receive_encoded(/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-output-file-4.3.0/lib/logstash/outputs/file.rb:117)", "RUBY.multi_receive(/usr/share/logstash/logstash-core/lib/logstash/outputs/base.rb:103)", "org.logstash.config.ir.compiler.AbstractOutputDelegatorExt.multi_receive(org/logstash/config/ir/compiler/AbstractOutputDelegatorExt.java:121)", "RUBY.start_workers(/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:301)"], :thread=>"#<Thread:0x63510022@/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:131 sleep>"}
[2023-02-28T05:51:07,086][WARN ][logstash.javapipeline ][main] Waiting for input plugin to close {:pipeline_id=>"main", :thread=>"#<Thread:0x63510022@/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:131 run>"}
[2023-02-28T05:51:08,508][INFO ][logstash.javapipeline ][main] Pipeline terminated {"pipeline.id"=>"main"}
[2023-02-28T05:51:08,578][INFO ][logstash.pipelinesregistry] Removed pipeline from registry successfully {:pipeline_id=>:main}
[2023-02-28T05:51:08,587][INFO ][logstash.runner ] Logstash shut down.
so I suppose that logstash crashes when it tries to write those values.
Logstash was installed as a daemon following these steps: https://www.elastic.co/guide/en/logstash/8.6/installing-logstash.html#_yum
How can I fix it?
