Redash 6.0.0 Internal Server Error

Issue Summary

Technical details:

  • Redash Version: 6.0.0
  • Browser/OS: Chrome/Mac
  • How did you install Redash: docker swarm stack
version: '2'
services:
  server:
    image: redash/redash:6.0.0.b8537
    command: server
    depends_on:
      - postgres
    networks:
      - web
      - database
    environment:
      PYTHONUNBUFFERED: 0
      REDASH_HOST: "http://redash.xxxxx.com"
      REDASH_LOG_LEVEL: "DEBUG"
      REDASH_REDIS_URL: "redis://redis:6379/0"
      REDASH_DATABASE_URL: "postgresql://postgres@postgres/postgres"
      REDASH_COOKIE_SECRET: "xxxxxx"
      REDASH_WEB_WORKERS: 4
    deploy:
      replicas: 1
      labels:
        traefik.docker.network: "web"
        traefik.port: "5000"
        traefik.frontend.rule: "Host:redash.xxxx.com"
      placement:
        constraints: [node.role==worker]
  worker:
    image: redash/redash:6.0.0.b8537
    command: scheduler
    networks:
      - web
      - database
    environment:
      PYTHONUNBUFFERED: 0
      REDASH_HOST: "http://redash.xxxx.com"
      REDASH_LOG_LEVEL: "INFO"
      REDASH_REDIS_URL: "redis://redis:6379/0"
      REDASH_DATABASE_URL: "postgresql://postgres@postgres/postgres"
      QUEUES: "queries,scheduled_queries,celery"
      WORKERS_COUNT: 2
    deploy:
      replicas: 1     
      placement:
        constraints: [node.role==worker]  
  postgres:
    image: postgres:9.5.6-alpine
    volumes:
      - postgresql_data:/var/lib/postgresql/data
    networks:
      - database
    deploy:
      replicas: 1     
      placement:
        constraints: [node.hostname==dev-swarm-xxxx.com]  

networks:
  web:
    external: true
  database:
    external: true

volumes:
  postgresql_data:

It had worked well for several monthes. But this morning, I found it doesn’t work.
PS. No one erase the Database, I’m sure 100%

Here is the DEBUG Log:

[SQL: 'SELECT organizations.updated_at AS organizations_updated_at, organizations.created_at AS organizations_created_at, organizations.id AS organizations_id, organizations.name AS organizations_name, organizations.slug AS organizations_slug, organizations.settings AS organizations_settings \nFROM organizations \nWHERE organizations.slug = %(slug_1)s \n LIMIT %(param_1)s'] [parameters: {'slug_1': 'default', 'param_1': 1}] (Background on this error at: http://sqlalche.me/e/f405)
[2019-08-20 05:54:28,444][PID:13][ERROR][redash] Exception on /favicon.ico [GET]
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1988, in wsgi_app
    response = self.full_dispatch_request()
  File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1641, in full_dispatch_request
    rv = self.handle_user_exception(e)
  File "/usr/local/lib/python2.7/dist-packages/flask_restful/__init__.py", line 271, in error_router
    return original_handler(e)
  File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1544, in handle_user_exception
    reraise(exc_type, exc_value, tb)
  File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1639, in full_dispatch_request
    rv = self.dispatch_request()
  File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1625, in dispatch_request
    return self.view_functions[rule.endpoint](**req.view_args)
  File "/usr/local/lib/python2.7/dist-packages/flask_login/utils.py", line 226, in decorated_view
    elif not current_user.is_authenticated:
  File "/usr/local/lib/python2.7/dist-packages/werkzeug/local.py", line 343, in __getattr__
    return getattr(self._get_current_object(), name)
  File "/usr/local/lib/python2.7/dist-packages/werkzeug/local.py", line 302, in _get_current_object
    return self.__local()
  File "/usr/local/lib/python2.7/dist-packages/flask_login/utils.py", line 26, in <lambda>
    current_user = LocalProxy(lambda: _get_user())
  File "/usr/local/lib/python2.7/dist-packages/flask_login/utils.py", line 302, in _get_user
    current_app.login_manager._load_user()
  File "/usr/local/lib/python2.7/dist-packages/flask_login/login_manager.py", line 311, in _load_user
    return self._load_from_cookie(request.cookies[cookie_name])
  File "/usr/local/lib/python2.7/dist-packages/flask_login/login_manager.py", line 350, in _load_from_cookie
    self.reload_user()
  File "/usr/local/lib/python2.7/dist-packages/flask_login/login_manager.py", line 279, in reload_user
    user = self.user_callback(user_id)
  File "/app/redash/authentication/__init__.py", line 44, in load_user
    org = current_org._get_current_object()
  File "/usr/local/lib/python2.7/dist-packages/werkzeug/local.py", line 302, in _get_current_object
    return self.__local()
  File "/app/redash/authentication/org_resolving.py", line 14, in _get_current_org
    g.org = Organization.get_by_slug(slug)
  File "/app/redash/models.py", line 325, in get_by_slug
    return cls.query.filter(cls.slug == slug).first()
  File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2825, in first
    ret = list(self[0:1])
  File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2617, in __getitem__
    return list(res)
  File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2925, in __iter__
    return self._execute_and_instances(context)
  File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2948, in _execute_and_instances
    result = conn.execute(querycontext.statement, self._params)
  File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 948, in execute
    return meth(self, multiparams, params)
  File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/sql/elements.py", line 269, in _execute_on_connection
    return connection._execute_clauseelement(self, multiparams, params)
  File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1060, in _execute_clauseelement
    compiled_sql, distilled_params
  File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1200, in _execute_context
    context)
  File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1413, in _handle_dbapi_exception
    exc_info
  File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/util/compat.py", line 203, in raise_from_cause
    reraise(type(exception), exception, tb=exc_tb, cause=cause)
  File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1193, in _execute_context
    context)
  File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 507, in do_execute
    cursor.execute(statement, parameters)
ProgrammingError: (psycopg2.ProgrammingError) relation "organizations" does not exist
LINE 2: FROM organizations 
             ^
 [SQL: 'SELECT organizations.updated_at AS organizations_updated_at, organizations.created_at AS organizations_created_at, organizations.id AS organizations_id, organizations.name AS organizations_name, organizations.slug AS organizations_slug, organizations.settings AS organizations_settings \nFROM organizations \nWHERE organizations.slug = %(slug_1)s \n LIMIT %(param_1)s'] [parameters: {'slug_1': 'default', 'param_1': 1}] (Background on this error at: http://sqlalche.me/e/f405)
[2019-08-20 05:54:28,445][PID:13][INFO][metrics] method=GET path=/favicon.ico endpoint=redash_index status=500 content_type=? content_length=-1 duration=3.81 query_count=0 query_duration=0.00

Does anyone have some ideas to solve this problem?

Hi @robinwangrubin and welcome!

Original error says that “relation “organizations” does not exist”. In this case, I’d first check if Redash is trying to connect the right Postgres instance (not by checking docker configs, but in fact - connect to the same network and check what’s available from there). Then, I’d connect to that Postgres instance and check if other tables are missing as well.

If only “organizations” table is missing - most likely somebody removed it accidentally (maybe even via executing a wrong query in Redash UI). In this case you should restore it manually.

If all tables are missing - first of all, check if Postgres instance uses right docker volumes (if no - point it to the right place). Then check if DB files are available there and if they’re not corrupted.

After all, it’s possible that you have some issues with your HDD/SSD (I personally faced an issue with Samsung SSD which after some time started reporting that all file operations are successful but in fact didn’t store any data).

At worst, I hope you have a DB backups which you can use to restore your data.

@levko Oh…It’s you! Thanks a lot for your advice and being patient. I’ve tried to follow your steps to check my Databases. And i found there is nothing in my databases.
Maybe it’s because i clean disk by using [docker system prune -af] this command.

Thank you again.

Now, I try to initialize Database

root@dev-swarm-worker03:~# docker run -it --rm     --network redash-temp     --env PYTHONUNBUFFERED=0     --env REDASH_LOG_LEVEL="INFO"     --env REDASH_REDIS_URL="redis://dev-xxxxxx.com:8379/0"     --env REDASH_DATABASE_URL="postgresql://postgres@postgres/postgres"     --env REDASH_COOKIE_SECRET="xxxxx"     --env REDASH_WEB_WORKERS=4     --log-driver json-file --log-opt max-size=10m     redash/redash:latest create_db
[2019-08-20 10:18:38,085][PID:1][INFO][root] Generating grammar tables from /usr/lib/python2.7/lib2to3/Grammar.txt
[2019-08-20 10:18:38,105][PID:1][INFO][root] Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
[2019-08-20 10:18:40,342][PID:1][INFO][alembic.runtime.migration] Context impl PostgresqlImpl.
[2019-08-20 10:18:40,343][PID:1][INFO][alembic.runtime.migration] Will assume transactional DDL.
[2019-08-20 10:18:40,356][PID:1][INFO][alembic.runtime.migration] Running stamp_revision  -> e5c7a4e2df4d

It looks like my initialization is successful

When i try to acess the web:

The log like this:

[2019-08-20 10:35:00 +0000] [1] [INFO] Starting gunicorn 19.7.1
[2019-08-20 10:35:00 +0000] [1] [INFO] Listening at: http://0.0.0.0:5000 (1)
[2019-08-20 10:35:00 +0000] [1] [INFO] Using worker: sync
[2019-08-20 10:35:00 +0000] [10] [INFO] Booting worker with pid: 10
[2019-08-20 10:35:00 +0000] [13] [INFO] Booting worker with pid: 13
[2019-08-20 10:35:00 +0000] [16] [INFO] Booting worker with pid: 16
[2019-08-20 10:35:00 +0000] [19] [INFO] Booting worker with pid: 19
[2019-08-20 10:35:00,856][PID:13][INFO][root] Generating grammar tables from /usr/lib/python2.7/lib2to3/Grammar.txt
[2019-08-20 10:35:00,902][PID:10][INFO][root] Generating grammar tables from /usr/lib/python2.7/lib2to3/Grammar.txt
[2019-08-20 10:35:00,984][PID:13][INFO][root] Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
[2019-08-20 10:35:01,043][PID:10][INFO][root] Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
[2019-08-20 10:35:01,113][PID:16][INFO][root] Generating grammar tables from /usr/lib/python2.7/lib2to3/Grammar.txt
[2019-08-20 10:35:01,142][PID:19][INFO][root] Generating grammar tables from /usr/lib/python2.7/lib2to3/Grammar.txt
[2019-08-20 10:35:01,238][PID:16][INFO][root] Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
[2019-08-20 10:35:01,255][PID:19][INFO][root] Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
[2019-08-20 10:35:09,536][PID:10][INFO][metrics] method=GET path=/ endpoint=redash_index status=304 content_type=text/html; charset=utf-8 content_length=926 duration=21.88 query_count=2 query_duration=3.62
[2019-08-20 10:35:09,900][PID:10][INFO][metrics] method=GET path=/static/images/favicon-32x32.png endpoint=static status=200 content_type=image/png content_length=2005 duration=0.59 query_count=0 query_duration=0.00
[2019-08-20 10:35:09,924][PID:16][INFO][metrics] method=GET path=/api/session endpoint=redash_session status=200 content_type=application/json content_length=1185 duration=23.86 query_count=3 query_duration=5.36
[2019-08-20 10:35:09,966] ERROR in app: Exception on /api/organization/status [GET]
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1988, in wsgi_app
    response = self.full_dispatch_request()
  File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1641, in full_dispatch_request
    rv = self.handle_user_exception(e)
  File "/usr/local/lib/python2.7/dist-packages/flask_restful/__init__.py", line 271, in error_router
    return original_handler(e)
  File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1544, in handle_user_exception
    reraise(exc_type, exc_value, tb)
  File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1639, in full_dispatch_request
    rv = self.dispatch_request()
  File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1625, in dispatch_request
    return self.view_functions[rule.endpoint](**req.view_args)
  File "/usr/local/lib/python2.7/dist-packages/flask_login/utils.py", line 228, in decorated_view
    return func(*args, **kwargs)
  File "/app/redash/handlers/organization.py", line 15, in organization_status
    'data_sources': models.DataSource.all(current_org, group_ids=current_user.group_ids).count(),
  File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 3160, in count
    return self.from_self(col).scalar()
  File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2913, in scalar
    ret = self.one()
  File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2884, in one
    ret = self.one_or_none()
  File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2854, in one_or_none
    ret = list(self)
  File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2925, in __iter__
    return self._execute_and_instances(context)
  File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2948, in _execute_and_instances
    result = conn.execute(querycontext.statement, self._params)
  File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 948, in execute
    return meth(self, multiparams, params)
  File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/sql/elements.py", line 269, in _execute_on_connection
    return connection._execute_clauseelement(self, multiparams, params)
  File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1060, in _execute_clauseelement
    compiled_sql, distilled_params
  File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1200, in _execute_context
    context)
  File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1413, in _handle_dbapi_exception
    exc_info
  File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/util/compat.py", line 203, in raise_from_cause
    reraise(type(exception), exception, tb=exc_tb, cause=cause)
  File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1193, in _execute_context
    context)
  File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 507, in do_execute
    cursor.execute(statement, parameters)
ProgrammingError: (psycopg2.ProgrammingError) column data_sources.options does not exist
LINE 2: ...ces_name, data_sources.type AS data_sources_type, data_sourc...
                                                             ^
 [SQL: 'SELECT count(*) AS count_1 \nFROM (SELECT DISTINCT data_sources.id AS data_sources_id, data_sources.org_id AS data_sources_org_id, data_sources.name AS data_sources_name, data_sources.type AS data_sources_type, data_sources.options AS data_sources_options, data_sources.queue_name AS data_sources_queue_name, data_sources.scheduled_queue_name AS data_sources_scheduled_queue_name, data_sources.created_at AS data_sources_created_at \nFROM data_sources JOIN data_source_groups ON data_sources.id = data_source_groups.data_source_id \nWHERE %(param_1)s = data_sources.org_id AND data_source_groups.group_id IN (%(group_id_1)s, %(group_id_2)s) ORDER BY data_sources.id ASC) AS anon_1'] [parameters: {'group_id_1': 1, 'group_id_2': 2, 'param_1': 1}] (Background on this error at: http://sqlalche.me/e/f405)

It’s a totally new database;

Try to run migrations as well: docker-compose run --rm server manage db upgrade (please change it accordingly to your setup)