Redash crash looping


#1

I checked that redis is running

# tail -f /var/log/redis_6379.log 
 |    `-._`-._        _.-'_.-'    |                                  
  `-._    `-._`-.__.-'_.-'    _.-'                                   
      `-._    `-.__.-'    _.-'                                       
          `-._        _.-'                                           
              `-.__.-'                                               

[6907] 16 Mar 20:27:37.571 # Server started, Redis version 2.8.17
[6907] 16 Mar 20:27:37.571 # WARNING overcommit_memory is set to 0! Background save may fail under low memory condition. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect.
[6907] 16 Mar 20:27:37.957 * DB loaded from disk: 0.386 seconds
[6907] 16 Mar 20:27:37.957 * The server is now ready to accept connections on port 6379

# ps  -Aef |grep redis
root      6583 28099  0 20:34 pts/1    00:00:00 grep --color=auto redis
root      6907     1  0 20:27 ?        00:00:00 /usr/local/bin/redis-server 127.0.0.1:6379   

[2017-03-16 20:30:21,857][PID:19167][INFO][root] Generating grammar tables from /usr/lib/python2.7/lib2to3/Grammar.txt
[2017-03-16 20:30:21,859][PID:19155][INFO][redash.query_runner.google_analytics] cannot import name SignedJwtAssertionCredentials
[2017-03-16 20:30:21,953][PID:19167][INFO][root] Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
[2017-03-16 20:30:21,964][PID:19162][INFO][redash.query_runner.google_analytics] cannot import name SignedJwtAssertionCredentials
[2017-03-16 20:30:22,001][PID:19170][INFO][redash.query_runner.google_analytics] cannot import name SignedJwtAssertionCredentials
[2017-03-16 20:30:22,224][PID:19167][INFO][redash.query_runner.google_analytics] cannot import name SignedJwtAssertionCredentials

==> ../logs/supervisord.log <==
2017-03-16 20:30:23,226 INFO success: redash_celery entered RUNNING state, process has stayed up for > than 1 seconds (startsecs)

==> ../logs/celery_error.log <==
[2017-03-16 20:30:23,303][PID:19156][INFO][root] Generating grammar tables from /usr/lib/python2.7/lib2to3/Grammar.txt
[2017-03-16 20:30:23,384][PID:19156][INFO][root] Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
[2017-03-16 20:30:23,839][PID:19156][INFO][redash.query_runner.google_analytics] cannot import name SignedJwtAssertionCredentials
Traceback (most recent call last):
  File "/usr/local/bin/celery", line 11, in <module>
    sys.exit(main())
  File "/usr/local/lib/python2.7/dist-packages/celery/__main__.py", line 14, in main
    _main()
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/celery.py", line 326, in main
    cmd.execute_from_commandline(argv)
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/celery.py", line 488, in execute_from_commandline
    super(CeleryCommand, self).execute_from_commandline(argv)))
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/base.py", line 279, in execute_from_commandline
    argv = self.setup_app_from_commandline(argv)
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/base.py", line 481, in setup_app_from_commandline
    self.app = self.find_app(app)
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/base.py", line 503, in find_app
    return find_app(app, symbol_by_name=self.symbol_by_name)
  File "/usr/local/lib/python2.7/dist-packages/celery/app/utils.py", line 358, in find_app
    sym = imp(app)
  File "/usr/local/lib/python2.7/dist-packages/celery/utils/imports.py", line 101, in import_from_cwd
    return imp(module, package=package)
  File "/usr/lib/python2.7/importlib/__init__.py", line 37, in import_module
    __import__(name)
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/worker.py", line 10, in <module>
    from redash.metrics import celery as celery_metrics
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/metrics/celery.py", line 13, in <module>
    @task_prerun.connect
  File "/usr/local/lib/python2.7/dist-packages/celery/utils/dispatch/signal.py", line 115, in connect
    return _handle_options(*args[1:], **kwargs)(args[0])
  File "/usr/local/lib/python2.7/dist-packages/celery/utils/dispatch/signal.py", line 110, in _connect_signal
    self._connect_signal(fun, sender, weak, dispatch_uid)
  File "/usr/local/lib/python2.7/dist-packages/celery/utils/dispatch/signal.py", line 122, in _connect_signal
    'Signal receiver must accept keyword arguments.')
ValueError: Signal receiver must accept keyword arguments.
[2017-03-16 20:30:24,366][PID:19183][INFO][root] Generating grammar tables from /usr/lib/python2.7/lib2to3/Grammar.txt
[2017-03-16 20:30:24,477][PID:19183][INFO][root] Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt

==> ../logs/supervisord.log <==
2017-03-16 20:30:24,551 INFO exited: redash_celery_scheduled (exit status 1; not expected)

==> ../logs/celery_error.log <==
[2017-03-16 20:30:24,796][PID:19183][INFO][redash.query_runner.google_analytics] cannot import name SignedJwtAssertionCredentials

==> ../logs/supervisord.log <==
2017-03-16 20:30:24,798 INFO spawned: 'redash_celery_scheduled' with pid 19631

==> ../logs/celery_error.log <==
Traceback (most recent call last):
  File "/usr/local/bin/celery", line 11, in <module>
    sys.exit(main())
  File "/usr/local/lib/python2.7/dist-packages/celery/__main__.py", line 14, in main
    _main()
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/celery.py", line 326, in main
    cmd.execute_from_commandline(argv)
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/celery.py", line 488, in execute_from_commandline
    super(CeleryCommand, self).execute_from_commandline(argv)))
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/base.py", line 279, in execute_from_commandline
    argv = self.setup_app_from_commandline(argv)
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/base.py", line 481, in setup_app_from_commandline
    self.app = self.find_app(app)
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/base.py", line 503, in find_app
    return find_app(app, symbol_by_name=self.symbol_by_name)
  File "/usr/local/lib/python2.7/dist-packages/celery/app/utils.py", line 358, in find_app
    sym = imp(app)
  File "/usr/local/lib/python2.7/dist-packages/celery/utils/imports.py", line 101, in import_from_cwd
    return imp(module, package=package)
  File "/usr/lib/python2.7/importlib/__init__.py", line 37, in import_module
    __import__(name)
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/worker.py", line 10, in <module>
    from redash.metrics import celery as celery_metrics
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/metrics/celery.py", line 13, in <module>
    @task_prerun.connect
  File "/usr/local/lib/python2.7/dist-packages/celery/utils/dispatch/signal.py", line 115, in connect
    return _handle_options(*args[1:], **kwargs)(args[0])
  File "/usr/local/lib/python2.7/dist-packages/celery/utils/dispatch/signal.py", line 110, in _connect_signal
    self._connect_signal(fun, sender, weak, dispatch_uid)
  File "/usr/local/lib/python2.7/dist-packages/celery/utils/dispatch/signal.py", line 122, in _connect_signal
    'Signal receiver must accept keyword arguments.')
ValueError: Signal receiver must accept keyword arguments.

==> ../logs/supervisord.log <==
2017-03-16 20:30:25,421 INFO exited: redash_celery (exit status 1; not expected)

==> ../logs/api_error.log <==
[2017-03-16 20:30:25 +0000] [19155] [ERROR] Exception in worker process
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/gunicorn/arbiter.py", line 578, in spawn_worker
    worker.init_process()
  File "/usr/local/lib/python2.7/dist-packages/gunicorn/workers/base.py", line 126, in init_process
    self.load_wsgi()
  File "/usr/local/lib/python2.7/dist-packages/gunicorn/workers/base.py", line 135, in load_wsgi
    self.wsgi = self.app.wsgi()
  File "/usr/local/lib/python2.7/dist-packages/gunicorn/app/base.py", line 67, in wsgi
    self.callable = self.load()
  File "/usr/local/lib/python2.7/dist-packages/gunicorn/app/wsgiapp.py", line 65, in load
    return self.load_wsgiapp()
  File "/usr/local/lib/python2.7/dist-packages/gunicorn/app/wsgiapp.py", line 52, in load_wsgiapp
    return util.import_app(self.app_uri)
  File "/usr/local/lib/python2.7/dist-packages/gunicorn/util.py", line 376, in import_app
    __import__(module)
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/wsgi.py", line 3, in <module>
    app = create_app()
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/__init__.py", line 86, in create_app
    from redash import handlers
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/handlers/__init__.py", line 4, in <module>
    from redash.handlers.api import api
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/handlers/api.py", line 6, in <module>
    from redash.handlers.base import org_scoped_rule
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/handlers/base.py", line 10, in <module>
    from redash.authentication import current_org
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/authentication/__init__.py", line 12, in <module>
    from redash.tasks import record_event
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/tasks/__init__.py", line 1, in <module>
    from .general import record_event, version_check, send_mail
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/tasks/general.py", line 4, in <module>
    from redash.worker import celery
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/worker.py", line 10, in <module>
    from redash.metrics import celery as celery_metrics
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/metrics/celery.py", line 13, in <module>
    @task_prerun.connect
  File "/usr/local/lib/python2.7/dist-packages/celery/utils/dispatch/signal.py", line 115, in connect
    return _handle_options(*args[1:], **kwargs)(args[0])
  File "/usr/local/lib/python2.7/dist-packages/celery/utils/dispatch/signal.py", line 110, in _connect_signal
    self._connect_signal(fun, sender, weak, dispatch_uid)
  File "/usr/local/lib/python2.7/dist-packages/celery/utils/dispatch/signal.py", line 122, in _connect_signal
    'Signal receiver must accept keyword arguments.')
ValueError: Signal receiver must accept keyword arguments.

==> ../logs/supervisord.log <==
2017-03-16 20:30:25,576 INFO spawned: 'redash_celery' with pid 19717

==> ../logs/api_error.log <==
[2017-03-16 20:30:25 +0000] [19155] [INFO] Worker exiting (pid: 19155)
[2017-03-16 20:30:25 +0000] [19162] [ERROR] Exception in worker process
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/gunicorn/arbiter.py", line 578, in spawn_worker
    worker.init_process()
  File "/usr/local/lib/python2.7/dist-packages/gunicorn/workers/base.py", line 126, in init_process
    self.load_wsgi()
  File "/usr/local/lib/python2.7/dist-packages/gunicorn/workers/base.py", line 135, in load_wsgi
    self.wsgi = self.app.wsgi()
  File "/usr/local/lib/python2.7/dist-packages/gunicorn/app/base.py", line 67, in wsgi
    self.callable = self.load()
  File "/usr/local/lib/python2.7/dist-packages/gunicorn/app/wsgiapp.py", line 65, in load
    return self.load_wsgiapp()
  File "/usr/local/lib/python2.7/dist-packages/gunicorn/app/wsgiapp.py", line 52, in load_wsgiapp
    return util.import_app(self.app_uri)
  File "/usr/local/lib/python2.7/dist-packages/gunicorn/util.py", line 376, in import_app
    __import__(module)
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/wsgi.py", line 3, in <module>
    app = create_app()
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/__init__.py", line 86, in create_app
    from redash import handlers
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/handlers/__init__.py", line 4, in <module>
    from redash.handlers.api import api
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/handlers/api.py", line 6, in <module>
    from redash.handlers.base import org_scoped_rule
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/handlers/base.py", line 10, in <module>
    from redash.authentication import current_org
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/authentication/__init__.py", line 12, in <module>
    from redash.tasks import record_event
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/tasks/__init__.py", line 1, in <module>
    from .general import record_event, version_check, send_mail
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/tasks/general.py", line 4, in <module>
    from redash.worker import celery
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/worker.py", line 10, in <module>
    from redash.metrics import celery as celery_metrics
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/metrics/celery.py", line 13, in <module>
    @task_prerun.connect
  File "/usr/local/lib/python2.7/dist-packages/celery/utils/dispatch/signal.py", line 115, in connect
    return _handle_options(*args[1:], **kwargs)(args[0])
  File "/usr/local/lib/python2.7/dist-packages/celery/utils/dispatch/signal.py", line 110, in _connect_signal
    self._connect_signal(fun, sender, weak, dispatch_uid)
  File "/usr/local/lib/python2.7/dist-packages/celery/utils/dispatch/signal.py", line 122, in _connect_signal
    'Signal receiver must accept keyword arguments.')
ValueError: Signal receiver must accept keyword arguments.

==> ../logs/supervisord.log <==
2017-03-16 20:30:25,917 INFO success: redash_celery_scheduled entered RUNNING state, process has stayed up for > than 1 seconds (startsecs)

==> ../logs/api_error.log <==
[2017-03-16 20:30:25 +0000] [19162] [INFO] Worker exiting (pid: 19162)
[2017-03-16 20:30:26 +0000] [19167] [ERROR] Exception in worker process
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/gunicorn/arbiter.py", line 578, in spawn_worker
    worker.init_process()
  File "/usr/local/lib/python2.7/dist-packages/gunicorn/workers/base.py", line 126, in init_process
    self.load_wsgi()
  File "/usr/local/lib/python2.7/dist-packages/gunicorn/workers/base.py", line 135, in load_wsgi
    self.wsgi = self.app.wsgi()
  File "/usr/local/lib/python2.7/dist-packages/gunicorn/app/base.py", line 67, in wsgi
    self.callable = self.load()
  File "/usr/local/lib/python2.7/dist-packages/gunicorn/app/wsgiapp.py", line 65, in load
    return self.load_wsgiapp()
  File "/usr/local/lib/python2.7/dist-packages/gunicorn/app/wsgiapp.py", line 52, in load_wsgiapp
    return util.import_app(self.app_uri)
  File "/usr/local/lib/python2.7/dist-packages/gunicorn/util.py", line 376, in import_app
    __import__(module)
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/wsgi.py", line 3, in <module>
    app = create_app()
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/__init__.py", line 86, in create_app
    from redash import handlers
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/handlers/__init__.py", line 4, in <module>
    from redash.handlers.api import api
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/handlers/api.py", line 6, in <module>
    from redash.handlers.base import org_scoped_rule
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/handlers/base.py", line 10, in <module>
    from redash.authentication import current_org
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/authentication/__init__.py", line 12, in <module>
    from redash.tasks import record_event
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/tasks/__init__.py", line 1, in <module>
    from .general import record_event, version_check, send_mail
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/tasks/general.py", line 4, in <module>
    from redash.worker import celery
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/worker.py", line 10, in <module>
    from redash.metrics import celery as celery_metrics
  File "/opt/redash/redash.1.0.0-rc.2.b2777/redash/metrics/celery.py", line 13, in <module>
    @task_prerun.connect
  File "/usr/local/lib/python2.7/dist-packages/celery/utils/dispatch/signal.py", line 115, in connect
    return _handle_options(*args[1:], **kwargs)(args[0])

#2

Looks like there was a pip upgrade to the system that upgraded celery. Reverting it back to 3.1.23 fixed everything