Description
Our chat server hosted locally has crashed. The mongo service is failing to start up.
root@studiochat:~# docker-compose up -d
Recreating root_mongo_1 … done
Recreating root_rocketchat_1 … done
Recreating root_mongo-init-replica_1 … done
Recreating root_hubot_1 … done
The root_mongo_1 container keeps restarting
Following is the error reported in log ( full log attached ) -
{"log":"2021-07-08T03:19:28.379+0000 I NETWORK [LogicalSessionCacheReap] Successfully connected to localhost:27017 (1 connections now open to localhost:27017 with a 5 second timeout)\n","stream":"stdout","time":"2021-07-08T03:19:28.38007768Z"}
{"log":"2021-07-08T03:19:28.380+0000 W NETWORK [LogicalSessionCacheReap] Unable to reach primary for set rs0\n","stream":"stdout","time":"2021-07-08T03:19:28.380517864Z"}
{"log":"2021-07-08T03:19:28.381+0000 I REPL [replexec-0] transition to RECOVERING from STARTUP2\n","stream":"stdout","time":"2021-07-08T03:19:28.381474128Z"}
{"log":"2021-07-08T03:19:28.381+0000 I REPL [replexec-0] Starting replication fetcher thread\n","stream":"stdout","time":"2021-07-08T03:19:28.381501349Z"}
{"log":"2021-07-08T03:19:28.381+0000 I REPL [replexec-0] Starting replication applier thread\n","stream":"stdout","time":"2021-07-08T03:19:28.38180951Z"}
{"log":"2021-07-08T03:19:28.381+0000 I REPL [replexec-0] Starting replication reporter thread\n","stream":"stdout","time":"2021-07-08T03:19:28.381843154Z"}
{"log":"2021-07-08T03:19:28.381+0000 I REPL [rsSync-0] Starting oplog application\n","stream":"stdout","time":"2021-07-08T03:19:28.381932863Z"}
{"log":"2021-07-08T03:19:28.382+0000 I REPL [rsSync-0] transition to SECONDARY from RECOVERING\n","stream":"stdout","time":"2021-07-08T03:19:28.382933469Z"}
{"log":"2021-07-08T03:19:28.383+0000 I REPL [rsSync-0] conducting a dry run election to see if we could be elected. current term: 782\n","stream":"stdout","time":"2021-07-08T03:19:28.383206051Z"}
{"log":"2021-07-08T03:19:28.383+0000 I REPL [replexec-0] dry election run succeeded, running for election in term 783\n","stream":"stdout","time":"2021-07-08T03:19:28.383382223Z"}
{"log":"2021-07-08T03:19:28.393+0000 I REPL [replexec-0] election succeeded, assuming primary role in term 783\n","stream":"stdout","time":"2021-07-08T03:19:28.393883858Z"}
{"log":"2021-07-08T03:19:28.394+0000 I REPL [replexec-0] transition to PRIMARY from SECONDARY\n","stream":"stdout","time":"2021-07-08T03:19:28.394129048Z"}
{"log":"2021-07-08T03:19:28.394+0000 I REPL [replexec-0] Resetting sync source to empty, which was :27017\n","stream":"stdout","time":"2021-07-08T03:19:28.394291254Z"}
{"log":"2021-07-08T03:19:28.394+0000 I REPL [replexec-0] Entering primary catch-up mode.\n","stream":"stdout","time":"2021-07-08T03:19:28.394592251Z"}
{"log":"2021-07-08T03:19:28.394+0000 I REPL [replexec-0] Exited primary catch-up mode.\n","stream":"stdout","time":"2021-07-08T03:19:28.394624864Z"}
{"log":"2021-07-08T03:19:28.394+0000 I REPL [replexec-0] Stopping replication producer\n","stream":"stdout","time":"2021-07-08T03:19:28.394703921Z"}
{"log":"2021-07-08T03:19:28.394+0000 I **REPL [ReplBatcher] Oplog buffer has been drained in term 783\n","stream":"stdout","time":"2021-07-08T03:19:28.395103991Z"}**
**{"log":"2021-07-08T03:19:28.395+0000 E - [rsSync-0] Assertion: Location17322: write to oplog failed: DocTooLargeForCapped: document doesn't fit in capped collection. size: 124 storageSize:134217728** src/mongo/db/repl/oplog.cpp 142\n","stream":"stdout","time":"2021-07-08T03:19:28.396029271Z"}
{"log":"2021-07-08T03:19:28.396+0000 F - [rsSync-0] terminate() called. An exception is active; attempting to gather more information\n","stream":"stdout","time":"2021-07-08T03:19:28.396346626Z"}
{"log":"2021-07-08T03:19:28.432+0000 F - [rsSync-0] **DBException::toString(): Location17322: write to oplog failed: DocTooLargeForCapped: document doesn't fit in capped collection. size: 124 storageSize:134217728** \n","stream":"stdout","time":"2021-07-08T03:19:28.432546701Z"}
{"log":"Actual exception type: mongo::error_details::throwExceptionForStatus(mongo::Status const\u0026)::NonspecificAssertionException\n","stream":"stdout","time":"2021-07-08T03:19:28.432628803Z"}
Changing the oplog size from the default 128 to 512 in the docker-compose has no effect. We are not able to get to the mongo shell through docker exec command so unable to change the oplog size at all.
Any advice on how to resolve this issue?
Server Setup Information
- Version of Rocket.Chat Server: 3.9.0
- Operating System: Linux
- Deployment Method: docker
- Number of Running Instances: 1
- DB Replicaset Oplog:
- NodeJS Version:
- MongoDB Version: 4.0.22
- Proxy: nginx
- Firewalls involved:
Additional Info -
The docker-config is as follows -
version: '2'
services:
rocketchat:
image: rocketchat/rocket.chat:latest
command: >
bash -c
"for i in `seq 1 30`; do
node main.js &&
s=$$? && break || s=$$?;
echo \"Tried $$i times. Waiting 5 secs...\";
sleep 5;
done; (exit $$s)"
restart: unless-stopped
volumes:
- /root/uploads:/app/uploads
environment:
- PORT=3000
- ROOT_URL=http://192.168.2.201:3000
- MONGO_URL=mongodb://mongo:27017/rocketchat
- MONGO_OPLOG_URL=mongodb://mongo:27017/local
- MAIL_URL=smtp://smtp.email
# - USE_NATIVE_OPLOG=true
# - HTTP_PROXY=http://proxy.domain.com
# - HTTPS_PROXY=http://proxy.domain.com
depends_on:
- mongo
ports:
- 3000:3000
labels:
- "traefik.backend=rocketchat"
- "traefik.frontend.rule=Host: your.domain.tld"
mongo:
image: mongo:4.0
restart: unless-stopped
volumes:
- ./data/db:/data/db
#- ./data/dump:/dump
command: mongod --smallfiles --oplogSize 128 --replSet rs0 --storageEngine=mmapv1
labels:
- "traefik.enable=false"
# shm_size: '2gb'
# this container's job is just run the command to initialize the replica set.
# it will run the command and remove himself (it will not stay running)
mongo-init-replica:
image: mongo:4.0
command: >
bash -c
"for i in `seq 1 30`; do
mongo mongo/rocketchat --eval \"
rs.initiate({
_id: 'rs0',
members: [ { _id: 0, host: 'localhost:27017' } ]})\" &&
s=$$? && break || s=$$?;
echo \"Tried $$i times. Waiting 5 secs...\";
sleep 5;
done; (exit $$s)"
depends_on:
- mongo
# hubot, the popular chatbot (add the bot user first and change the password before starting this image)
hubot:
image: rocketchat/hubot-rocketchat:latest
restart: unless-stopped
environment:
- ROCKETCHAT_URL=rocketchat:3000
- ROCKETCHAT_ROOM=GENERAL
- ROCKETCHAT_USER=bot
- ROCKETCHAT_PASSWORD=botpassword
- BOT_NAME=bot
# you can add more scripts as you'd like here, they need to be installable by npm
- EXTERNAL_SCRIPTS=hubot-help,hubot-seen,hubot-links,hubot-diagnostics
depends_on:
- rocketchat
labels:
- "traefik.enable=false"
volumes:
- ./scripts:/home/hubot/scripts
# this is used to expose the hubot port for notifications on the host on port 3001, e.g. for hubot-jenkins-notifier
ports:
- 3001:8080