It's my first time to use monstache.
In fact I've migrated my infrastructure from on premise to the cloud and I'm now using mongoDB Atlas, and AWS opensearch.
I've installed monstache on an aws ec2 instance and well configured it. Everything seems working and monstache is connected to Elasticsearch and MongoDB, but it's indexing documents that have been migratred into mongoDB atlas in Elasticsearch. It keeps waiting for events on my collection/index like this
[ec2-user@ip-172-31-1-200 ~]$ journalctl -u monstache.service -f
-- Logs begin at Wed 2022-11-09 10:22:04 UTC. --
Jan 26 08:54:00 ip-172-31-1-200.eu-west-3.compute.internal systemd[1]: Starting monstache sync service...
Jan 26 08:54:00 ip-172-31-1-200.eu-west-3.compute.internal monstache[27813]: INFO 2023/01/26 08:54:00 Started monstache version 6.1.0
Jan 26 08:54:00 ip-172-31-1-200.eu-west-3.compute.internal monstache[27813]: INFO 2023/01/26 08:54:00 Successfully connected to MongoDB version 4.4.18
Jan 26 08:54:01 ip-172-31-1-200.eu-west-3.compute.internal monstache[27813]: INFO 2023/01/26 08:54:01 Successfully connected to Elasticsearch version 7.10.2
Jan 26 08:54:01 ip-172-31-1-200.eu-west-3.compute.internal systemd[1]: Started monstache sync service.
Jan 26 08:54:01 ip-172-31-1-200.eu-west-3.compute.internal monstache[27813]: INFO 2023/01/26 08:54:01 Joined cluster HA
Jan 26 08:54:01 ip-172-31-1-200.eu-west-3.compute.internal monstache[27813]: INFO 2023/01/26 08:54:01 Starting work for cluster HA
Jan 26 08:54:01 ip-172-31-1-200.eu-west-3.compute.internal monstache[27813]: INFO 2023/01/26 08:54:01 Listening for events
Jan 26 08:54:01 ip-172-31-1-200.eu-west-3.compute.internal monstache[27813]: INFO 2023/01/26 08:54:01 Watching changes on collection wtlive.myuser
Jan 26 08:54:01 ip-172-31-1-200.eu-west-3.compute.internal monstache[27813]: INFO 2023/01/26 08:54:01 Resuming from timestamp {T:1674723241 I:1}
Should I absolutely initiate a write on the mongoDB collection for monstache to start syncing? Why doesn't it start syncing current data from mongoDB?
My elasticsearch still shows 0 document count while the collection is full of document in mongoDB.
[ec2-user@ip-172-31-0-5 ~]$ curl --insecure -u es-appuser https://vpc-wtlive-domain-staging-om2cbdeex4qk6trkdrcb3dg4vm.eu-west-3.es.amazonaws.com/_cat/indices?v
Enter host password for user 'es-appuser':
health status index uuid pri rep docs.count docs.deleted store.size pri.store.size
green open wtlive.myuser YzqLx9_uTZ2qFVjFF2CMag 1 1 0 0 416b 208b
green open .opendistro_security Jb1fLqGjRd2vvluoX-ZgKw 1 1 9 4 129kb 64.4kb
green open .kibana_1 v6WdqQDvSN2L16EZTXxuHQ 1 1 30 2 70.4kb 33.4kb
green open .kibana_252235597_esappuser_1 OY1bbDGvTqK8oEgwopzbhQ 1 1 1 0 10.1kb 5kb
[ec2-user@ip-172-31-0-5 ~]$
Here is my monstache configuration :
[ec2-user@ip-172-31-1-200 ~]$ cat monstache/wtlive_pipeline.toml
enable-http-server = true
http-server-addr = ":8888"
#direct-read-namespaces = ["wtlive.myuser"]
change-stream-namespaces = ["wtlive.myuser"]
namespace-regex = '^wtlive.myuser$'
cluster-name="HA"
resume = true
replay = false
resume-write-unsafe = false
exit-after-direct-reads = false
elasticsearch-user = "es-appuser"
elasticsearch-password = "9V#xxxxxx"
elasticsearch-urls = ["https://vpc-wtlive-domain-staging-om2cbdeek6trkdrcb3dg4vm.eu-west-3.es.amazonaws.com"]
mongo-url = "mongodb://admin:VYn7ZD4CHDh8@wtlive-dedicated-shard-00-00.ynxpn.mongodb.net:27017,wtlive-dedicated-shard-00-01.ynxpn.mongodb.net:27017,wtlive-dedicated-shard-00-02.ynxpn.mongodb.net:27017/?tls=true&replicaSet=atlas-lmkye1-shard-0&authSource=admin&retryWrites=true&w=majority&tlsCAFile=/home/ec2-user/mongodb-ca.pem"
#[logs]
#info = "/home/ec2-user/logs/monstache/info.log"
#error = "/home/ec2-user/logs/monstache/error.log"
#warn = "/home/ec2-user/logs/monstache/warn.log"
#[[mapping]]
#namespace = "wtlive.myuser"
#index = "wtlive.myuser"
[[pipeline]]
namespace = "wtlive.myuser"
script = """
module.exports = function(ns, changeStream) {
if (changeStream) {
return [
{
$project: {
_id: 1,
operationType : 1,
clusterTime : 1,
documentKey : 1,
to : 1,
updateDescription : 1,
txnNumber : 1,
lsid : 1,
"fullDocument._id": 1,
"fullDocument.created": 1,
"fullDocument.lastVisit": 1,
"fullDocument.verified": 1,
"fullDocument.device.locale": "$fullDocument.device.locale",
"fullDocument.device.country": "$fullDocument.device.country",
"fullDocument.device.tz": "$fullDocument.device.tz",
"fullDocument.device.latLonCountry": "$fullDocument.device.latLonCountry",
"fullDocument.details.firstname": "$fullDocument._details.firstname",
"fullDocument.details.gender": "$fullDocument._details.gender",
"fullDocument.details.category": "$fullDocument._details.category",
"fullDocument.details.dob": "$fullDocument._details.dob",
"fullDocument.details.lookingFor": "$fullDocument._details.lookingFor",
"fullDocument.details.height": "$fullDocument._details.height",
"fullDocument.details.weight": "$fullDocument._details.weight",
"fullDocument.details.cigarette": "$fullDocument._details.cigarette",
"fullDocument.details.categorizedBy": "$fullDocument._details.categorizedBy",
"fullDocument.details.origin": "$fullDocument._details.origin",
"fullDocument.details.city": "$fullDocument._details.city",
"fullDocument.details.country": "$fullDocument._details.country",
"fullDocument.lifeSkills.educationLevel": "$fullDocument._lifeSkills.educationLevel",
"fullDocument.lifeSkills.pets": "$fullDocument._lifeSkills.pets",
"fullDocument.lifeSkills.religion": "$fullDocument._lifeSkills.religion",
"fullDocument.loveLife.children": "$fullDocument._loveLife.children",
"fullDocument.loveLife.relationType": "$fullDocument._loveLife.relationType",
"fullDocument.searchCriteria": "$fullDocument._searchCriteria",
"fullDocument.blocked" : 1,
"fullDocument.capping" : 1,
"fullDocument.fillingScore" : 1,
"fullDocument.viewed" : 1,
"fullDocument.likes" : 1,
"fullDocument.matches" : 1,
"fullDocument.blacklisted" : 1,
"fullDocument.uploadsList._id" : 1,
"fullDocument.uploadsList.status" : 1,
"fullDocument.uploadsList.url" : 1,
"fullDocument.uploadsList.position" : 1,
"fullDocument.uploadsList.imageSet" : 1,
"fullDocument.location" : 1,
"fullDocument.searchZone" : 1,
"fullDocument.locationPoint" : "$fullDocument.location.coordinates",
"fullDocument.selfieDateUpload" : 1,
"ns": 1
}
}
]
} else {
return [
{
$project: {
_id: 1,
"_id": 1,
"created": 1,
"lastVisit": 1,
"verified": 1,
"device.locale": "$device.locale",
"device.country": "$device.country",
"device.tz": "$device.tz",
"device.latLonCountry": "$device.latLonCountry",
"details.firstname": "$_details.firstname",
"details.gender": "$_details.gender",
"details.category": "$_details.category",
"details.dob": "$_details.dob",
"details.lookingFor": "$_details.lookingFor",
"details.height": "$_details.height",
"details.weight": "$_details.weight",
"details.cigarette": "$_details.cigarette",
"details.categorizedBy": "$_details.categorizedBy",
"details.origin": "$_details.origin",
"details.city": "$_details.city",
"details.country": "$_details.country",
"lifeSkills.educationLevel": "$_lifeSkills.educationLevel",
"lifeSkills.pets": "$_lifeSkills.pets",
"lifeSkills.religion": "$_lifeSkills.religion",
"loveLife.children": "$_loveLife.children",
"loveLife.relationType": "$_loveLife.relationType",
"searchCriteria": "$_searchCriteria",
"blocked" : 1,
"capping" : 1,
"fillingScore" : 1,
"viewed" : 1,
"likes" : 1,
"matches" : 1,
"blacklisted" : 1,
"uploadsList._id" : 1,
"uploadsList.status" : 1,
"uploadsList.url" : 1,
"uploadsList.position" : 1,
"uploadsList.imageSet" : 1,
"location" : 1,
"searchZone" : 1,
"selfieDateUpload" : 1,
"locationPoint" : "$location.coordinates"
}
}
]
}
}
"""
What could be the issue? And what action should I take from here please?
