Как заставить задачи clery ceerybeat cron запускаться в Docker-контейнере для приложения Django?

Question

Я пытаюсь запустить задачи сельдерея в моем приложении django / nuxt. У меня есть отдельные каталоги веб-интерфейса и внутреннего интерфейса, и я использую docker-compose для сборки и запуска моего приложения.(pycharm professional-mac-oS seria

Я могу отлично выполнять свои задачи, не используя докер-контейнеры локально, но когда я пытаюсь создать контейнеры для работы, и сельдерей, и сельдерея не будут работать. Я считаю, что документация оченьплохой онлайн и даже в официальной документации сельдерея или докеров нет упоминания о запуске celery & beat с докером Может кто-нибудь сказать мне, если моя конфигурация неверна или что мне нужно сделать, чтобы мое приложение приняло cronjobs в моем settings.py?

Я создаю свои контейнеры с помощью docker-compose up-d и запускаю свое приложение с docker-compose exec django bash

Может ли кто-нибудь указать мне правильное направление?

settings.py

    import os
    from configurations import Configuration, values
    from datetime import timedelta
    #print(os.environ)


    class Base(Configuration):

        DEBUG = values.BooleanValue(False)
        SECRET_KEY = '2pj=b#ywty7ojkv_gd#!$!vzywakop1azlxiqxrl^r50i(nf-^'
        BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
        DATABASES = values.DatabaseURLValue()
        ALLOWED_HOSTS = []

        INSTALLED_APPS = [
            "django.contrib.admin",
            "django.contrib.auth",
            "django.contrib.contenttypes",
            "django.contrib.sessions",
            "django.contrib.messages",
            "django.contrib.staticfiles",
            "rest_framework",
            "rest_framework.authtoken",
            "django_celery_results" ,
            "django_celery_beat" ,
            "corsheaders",
            "djoser",
            "accounts",
            "posts",
            "comments",
            "events",
        ]

        MIDDLEWARE = [
            "django.middleware.security.SecurityMiddleware",
            "django.contrib.sessions.middleware.SessionMiddleware",
            "corsheaders.middleware.CorsMiddleware",
            "django.middleware.common.CommonMiddleware",
            "django.middleware.csrf.CsrfViewMiddleware",
            "django.contrib.auth.middleware.AuthenticationMiddleware",
            "django.contrib.messages.middleware.MessageMiddleware",
            "django.middleware.clickjacking.XFrameOptionsMiddleware",
        ]

        ROOT_URLCONF = "spacenews.urls"
        TEMPLATES = [
            {
                "BACKEND": "django.template.backends.django.DjangoTemplates",
                "DIRS": [],
                "APP_DIRS": True,
                "OPTIONS": {
                    "context_processors": [
                        "django.template.context_processors.debug",
                        "django.template.context_processors.request",
                        "django.contrib.auth.context_processors.auth",
                        "django.contrib.messages.context_processors.messages",
                    ]
                },
            }
        ]

        # WSGI
        WSGI_APPLICATION = "spacenews.wsgi.application"
        # Password validators
        AUTH_PASSWORD_VALIDATORS = [
            {
                "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator"  # noqa
            },
            {
                "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"
            },
            {
                "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"
            },
            {
                "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"
            },
        ]
        AUTH_USER_MODEL = "accounts.User"

        # Internationalization
        LANGUAGE_CODE = "en-us"
        TIME_ZONE = "UTC"
        USE_I18N = True
        USE_L10N = True
        USE_TZ = True

        # Static files

        STATIC_URL = "/static/"
        # REST
        REST_FRAMEWORK = {
            "DEFAULT_AUTHENTICATION_CLASSES": (
                "rest_framework.authentication.BasicAuthentication",
                "rest_framework.authentication.SessionAuthentication",
                "rest_framework.authentication.TokenAuthentication",
            ),
            "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.PageNumberPagination",
            "PAGE_SIZE": 100,
        }


        CELERY_BROKER_URL = 'redis://localhost:6379/'
        CELERY_RESULT_BACKEND = 'redis://localhost:6379/'
        #CELERYD_HIJACK_ROOT_LOGGER = False
        # use json format for everything
        CELERY_ACCEPT_CONTENT = ['application/json']
        CELERY_TASK_SERIALIZER = 'json'
        CELERY_RESULT_SERIALIZER = 'json'
        CELERY_TIMEZONE = 'UTC'


        CELERY_BEAT_SCHEDULE = {


            'login_mb': {
                'task': 'events.tasks.login_mb',
                'schedule': timedelta(seconds=10),

            } ,
            'mb_get_events': {
                'task': 'events.tasks.mb_get_events' ,
                'schedule': timedelta(seconds=10) ,

            } ,
        }



    class Development(Base):
        DEBUG = values.BooleanValue(True)
        CORS_ORIGIN_ALLOW_ALL = True
        ALLOWED_HOSTS = ["localhost", "django","postgres","redis"]


    class Production(Base):
        pass


    class Testing(Base):
        pass

Dockerfile

    FROM python:3.6
    ENV PYTHONUNBUFFERED 1
    #ENV C_FORCE_ROOT true
    RUN apt-get update && apt-get install -y postgresql-client
    ADD . /app
    WORKDIR /app
    COPY requirements.txt /app/requirements.txt
    RUN pip install -r requirements.txt
    EXPOSE 8000

docker-compose.yaml

  version: '2'

  services:

  db:
    restart: always
    image: postgres

  redis:
    restart: always
    image: redis

  api:
    build:
      context: ./backend
    environment:
       - DATABASE_URL=postgres://postgres@db:5432/postgres
       - CELERY_BROKER_URL=redis://localhost:6379/
       - CELERY_RESULT_BACKEND =redis://localhost:6379/
       - DJANGO_SECRET_KEY=seekret
    volumes:
       - ./backend:/app

  celery:
    extends:
      service: api
    command:
      bash -c "cd spacenews && celery -A spacenews worker -B --loglevel=info"
    depends_on:
      - db
      - redis

  django:
    extends:
      service: api
    command: ./wait-for-it.sh db:5432 -- ./spacenews/manage.py runserver 0.0.0.0:8000
    ports:
      - "8000:8000"
    volumes:
      - ./backend:/app
    depends_on:
      - db
      - redis
      - celery
      - celery-beat

  celery-beat:
    extends:
      service: api

    command:
        bash -c "cd backend/spacenews && celery -A spacenews beat -B --loglevel=info"
    volumes:
      - ./backend:/app
    depends_on:
      - db
      - redis
      - celery
  nuxt:
    build:
      context: ./frontend
    environment:
      - API_URI=http://django:8000/api
    command: bash -c "npm install && npm run dev"
    volumes:
      - ./frontend:/app
    ports:
      - "3000:3000"
    depends_on:
      - django
      - redis

  volumes:
  pgdata:
  redisdata:

стековый контейнер сельдерея

    Traceback (most recent call last):
      File "/usr/local/lib/python3.6/site-packages/kombu/utils/objects.py", line 42, in __get__
        return obj.__dict__[self.__name__]
    KeyError: 'data'

    During handling of the above exception, another exception occurred:

    Traceback (most recent call last):
      File "/usr/local/bin/celery", line 11, in <module>
        sys.exit(main())
      File "/usr/local/lib/python3.6/site-packages/celery/__main__.py", line 16, in main
        _main()
      File "/usr/local/lib/python3.6/site-packages/celery/bin/celery.py", line 322, in main
        cmd.execute_from_commandline(argv)
      File "/usr/local/lib/python3.6/site-packages/celery/bin/celery.py", line 496, in execute_from_commandline
        super(CeleryCommand, self).execute_from_commandline(argv)))
      File "/usr/local/lib/python3.6/site-packages/celery/bin/base.py", line 275, in execute_from_commandline
        return self.handle_argv(self.prog_name, argv[1:])
      File "/usr/local/lib/python3.6/site-packages/celery/bin/celery.py", line 488, in handle_argv
        return self.execute(command, argv)
      File "/usr/local/lib/python3.6/site-packages/celery/bin/celery.py", line 420, in execute
        ).run_from_argv(self.prog_name, argv[1:], command=argv[0])
      File "/usr/local/lib/python3.6/site-packages/celery/bin/worker.py", line 221, in run_from_argv
        *self.parse_options(prog_name, argv, command))
      File "/usr/local/lib/python3.6/site-packages/celery/bin/base.py", line 398, in parse_options
        self.parser = self.create_parser(prog_name, command)
      File "/usr/local/lib/python3.6/site-packages/celery/bin/base.py", line 414, in create_parser
        self.add_arguments(parser)
      File "/usr/local/lib/python3.6/site-packages/celery/bin/worker.py", line 277, in add_arguments
        default=conf.worker_state_db,
      File "/usr/local/lib/python3.6/site-packages/celery/utils/collections.py", line 126, in __getattr__
        return self[k]
      File "/usr/local/lib/python3.6/site-packages/celery/utils/collections.py", line 429, in __getitem__
        return getitem(k)
      File "/usr/local/lib/python3.6/site-packages/celery/utils/collections.py", line 278, in __getitem__
        return mapping[_key]
      File "/usr/local/lib/python3.6/collections/__init__.py", line 987, in __getitem__
        if key in self.data:
      File "/usr/local/lib/python3.6/site-packages/kombu/utils/objects.py", line 44, in __get__
        value = obj.__dict__[self.__name__] = self.__get(obj)
      File "/usr/local/lib/python3.6/site-packages/celery/app/base.py", line 141, in data
        return self.callback()
      File "/usr/local/lib/python3.6/site-packages/celery/app/base.py", line 924, in _finalize_pending_conf
        conf = self._conf = self._load_config()
      File "/usr/local/lib/python3.6/site-packages/celery/app/base.py", line 934, in _load_config
        self.loader.config_from_object(self._config_source)
      File "/usr/local/lib/python3.6/site-packages/celery/loaders/base.py", line 131, in config_from_object
        self._conf = force_mapping(obj)
      File "/usr/local/lib/python3.6/site-packages/celery/utils/collections.py", line 46, in force_mapping
        if isinstance(m, (LazyObject, LazySettings)):
      File "/usr/local/lib/python3.6/site-packages/django/utils/functional.py", line 215, in inner
        self._setup()
      File "/usr/local/lib/python3.6/site-packages/django/conf/__init__.py", line 43, in _setup
        self._wrapped = Settings(settings_module)
      File "/usr/local/lib/python3.6/site-packages/django/conf/__init__.py", line 106, in __init__
        mod = importlib.import_module(self.SETTINGS_MODULE)
      File "/usr/local/lib/python3.6/importlib/__init__.py", line 126, in import_module
        return _bootstrap._gcd_import(name[level:], package, level)
      File "<frozen importlib._bootstrap>", line 994, in _gcd_import
      File "<frozen importlib._bootstrap>", line 971, in _find_and_load
      File "<frozen importlib._bootstrap>", line 955, in _find_and_load_unlocked
      File "<frozen importlib._bootstrap>", line 665, in _load_unlocked
      File "<frozen importlib._bootstrap_external>", line 678, in exec_module
      File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
      File "/app/spacenews/spacenews/settings.py", line 7, in <module>
        class Base(Configuration):

celery.py

from __future__ import absolute_import
import os
from celery import Celery

# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'spacenews.settings')



app = Celery('spacenews')

# Using a string here means the worker will not have to
# pickle the object when using Windows.
app.config_from_object('django.conf:settings', namespace='CELERY')
app.autodiscover_tasks()


@app.task(bind=True)
def debug_task(self):
    print('Request: {0!r}'.format(self.request))

Answer 1

Ваш файл settings.py не выглядит как действительный для сельдерея.Обычно файлы настроек выглядят следующим образом:

DEBUG = True

<b>BROKER_URL = 'redis://localhost:6379/'</b>
CELERY_RESULT_BACKEND = 'redis://localhost:6379/'
#CELERYD_HIJACK_ROOT_LOGGER = False
# use json format for everything
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TIMEZONE = 'UTC'


CELERY_BEAT_SCHEDULE = {


    'login_mb': {
        'task': 'events.tasks.login_mb',
        'schedule': timedelta(seconds=10),

    } ,
    'mb_get_events': {
        'task': 'events.tasks.mb_get_events' ,
        'schedule': timedelta(seconds=10) ,

    } ,
}

без использования таких объектов, как Base, Production или Dev.Если вы хотите использовать объекты, которые у вас есть, вам нужно проделать дополнительную работу, чтобы выставить их в качестве настроек объекта настроек django (который используется для настройки сельдерея).

В качестве альтернативы вы можете предоставить экземпляр класса, который содержит эту конфигурацию сельдерея, для config_from_object.

Наконец, обратите внимание, что значение параметра broker_url должно быть BROKER_URL, а не CELERY_BROKER_URL.