Celery 및 Gino를 사용하여 개체 저장

다음 파이프 라인이 있습니다.

이벤트 모델 (Gino 'db'개체 기반) :

class Event(db.Model):
    __tablename__ = "events"

    id = db.Column(db.BigInteger(), primary_key=True)
    request_timestamp = db.Column(db.DateTime(), nullable=False)
    service = db.Column(db.String(), nullable=False)
    url = db.Column(db.String(), nullable=False)
    status_code = db.Column(db.Integer(), nullable=False)
    response_time = db.Column(db.DateTime(), nullable=False)

FastApi 앱에는 Event 개체의 데이터를 전달하기 위해 셀러리 작업을 호출하는 POST보기가 있습니다.

@router.post("/events/add")
async def add_event(event: EventModel):
    event_data = {'request_timestamp': event.request_timestamp.replace(tzinfo=None),
                  'service': event.service,
                  'url': event.url,
                  'status_code': event.status_code,
                  'response_time': event.response_time.replace(tzinfo=None)
                  }

    task = celery_app.send_task("monitoring_service.src.monitoring_service.worker.celery_worker.add_to_db",
                                kwargs=event_data)

    return JSONResponse(content="Event recorded successfully", status_code=200)

그리고 데이터베이스에 Event 객체를 저장해야하는 셀러리 작업자 :

@celery_app.task(acks_late=True)
async def add_to_db(request_timestamp, service, url, status_code, response_time):
    event = await Event.create(
        request_timestamp=datetime.strptime(request_timestamp, '%Y-%m-%dT%H:%M:%S.%f'),
        service=service,
        url=url,
        status_code=status_code,
        response_time=datetime.strptime(response_time, '%Y-%m-%dT%H:%M:%S.%f'),
    )

    return {"status": True}

이제 개체 저장 프로세스 중에 오류가 발생합니다.

Traceback (most recent call last):

File "/usr/local/lib/python3.8/site-packages/celery/app/trace.py", line 479, in trace_task

mark_as_done(

File "/usr/local/lib/python3.8/site-packages/celery/backends/base.py", line 158, in mark_as_done

self.store_result(task_id, result, state, request=request)

File "/usr/local/lib/python3.8/site-packages/celery/backends/base.py", line 442, in store_result

self._store_result(task_id, result, state, traceback,

File "/usr/local/lib/python3.8/site-packages/celery/backends/database/__init__.py", line 51, in _inner

return fun(*args, **kwargs)

File "/usr/local/lib/python3.8/site-packages/celery/backends/database/__init__.py", line 130, in _store_result

session.commit()

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 1042, in commit

self.transaction.commit()

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 504, in commit

self._prepare_impl()

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 483, in _prepare_impl

self.session.flush()

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 2523, in flush

self._flush(objects)

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 2664, in _flush

transaction.rollback(_capture_exception=True)

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/util/langhelpers.py", line 68, in __exit__

compat.raise_(

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 178, in raise_

raise exception

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 2624, in _flush

flush_context.execute()

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/unitofwork.py", line 422, in execute

rec.execute(self)

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/unitofwork.py", line 586, in execute

persistence.save_obj(

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/persistence.py", line 230, in save_obj

_emit_update_statements(

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/persistence.py", line 994, in _emit_update_statements

c = cached_connections[connection].execute(

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1014, in execute

return meth(self, multiparams, params)

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection

return connection._execute_clauseelement(self, multiparams, params)

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1127, in _execute_clauseelement

ret = self._execute_context(

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1207, in _execute_context

self._handle_dbapi_exception(

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1511, in _handle_dbapi_exception

util.raise_(

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 178, in raise_

raise exception

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1205, in _execute_context

context = constructor(dialect, self, conn, *args)

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 858, in _init_compiled

param = dict(

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 861, in <genexpr>

processors[key](compiled_params[key])

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/sql/sqltypes.py", line 1689, in process

value = dumps(value, protocol)

sqlalchemy.exc.SQLAlchemyError: (builtins.TypeError) cannot pickle 'coroutine' object


warn(RuntimeWarning(

[2021-02-19 10:19:39,010: ERROR/ForkPoolWorker-1] Task monitoring_service.src.monitoring_service.worker.celery_worker.add_to_db[994ae754-f216-4fa9-b502-5fbb0221011c] raised unexpected: SQLAlchemyError("(builtins.TypeError) cannot pickle 'coroutine' object")

Traceback (most recent call last):

File "/usr/local/lib/python3.8/site-packages/celery/app/trace.py", line 479, in trace_task

mark_as_done(

File "/usr/local/lib/python3.8/site-packages/celery/backends/base.py", line 158, in mark_as_done

self.store_result(task_id, result, state, request=request)

File "/usr/local/lib/python3.8/site-packages/celery/backends/base.py", line 442, in store_result

self._store_result(task_id, result, state, traceback,

File "/usr/local/lib/python3.8/site-packages/celery/backends/database/__init__.py", line 51, in _inner

return fun(*args, **kwargs)

File "/usr/local/lib/python3.8/site-packages/celery/backends/database/__init__.py", line 130, in _store_result

session.commit()

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 1042, in commit

self.transaction.commit()

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 504, in commit

self._prepare_impl()

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 483, in _prepare_impl

self.session.flush()

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 2523, in flush

self._flush(objects)

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 2664, in _flush

transaction.rollback(_capture_exception=True)

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/util/langhelpers.py", line 68, in __exit__

compat.raise_(

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 178, in raise_

raise exception

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 2624, in _flush

flush_context.execute()

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/unitofwork.py", line 422, in execute

rec.execute(self)

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/unitofwork.py", line 586, in execute

persistence.save_obj(

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/persistence.py", line 230, in save_obj

_emit_update_statements(

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/persistence.py", line 994, in _emit_update_statements

c = cached_connections[connection].execute(

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1014, in execute

return meth(self, multiparams, params)

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection

return connection._execute_clauseelement(self, multiparams, params)

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1127, in _execute_clauseelement

ret = self._execute_context(

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1207, in _execute_context

self._handle_dbapi_exception(

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1511, in _handle_dbapi_exception

util.raise_(

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 178, in raise_

raise exception

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1205, in _execute_context

context = constructor(dialect, self, conn, *args)

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 858, in _init_compiled

param = dict(

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 861, in <genexpr>

processors[key](compiled_params[key])

File "/usr/local/lib/python3.8/site-packages/sqlalchemy/sql/sqltypes.py", line 1689, in process

value = dumps(value, protocol)

sqlalchemy.exc.SQLAlchemyError: (builtins.TypeError) cannot pickle 'coroutine' object

event = await Event.create(...)뷰 내에서 호출하면 성공하고 데이터를 db에 저장합니다. Celery 작업자에서 'async / await'를 삭제하면이 오류가 표시되지 않고 Celery 작업은 'SUCCESS'상태이지만 데이터베이스가 비어 있습니다. 옳지 않은 것이 무엇인지 정말로 이해할 수 없습니다.