Модель очень медленной загрузки в Django Admin и DRF API - PullRequest
Новая
       5

Модель очень медленной загрузки в Django Admin и DRF API

0 голосов
/ 05 февраля 2020

У меня есть модель с 11 миллионами экземпляров (при тестировании модели на небольшом количестве данных проблем не возникает). Модель НЕ имеет полей ForeignKey или ManyToMany . Когда я пытаюсь загрузить страницу с моей моделью (DRF API или Django Admin), страница загружается в течение 5-7 минут, также начинает замедляться вся система. Когда я работаю с другими моделями, такой проблемы нет.

Модель: 

class MyModel(models.Model):
    process_number = models.CharField(max_length=255, blank=True, null=True)
    order_number = models.CharField(max_length=64, null=True, blank=True)
    dep_name_l1 = models.CharField(max_length=128, null=True, blank=True)
    worksite = models.CharField(max_length=64, null=True, blank=True)
    date_create = models.DateTimeField(null=True, blank=True)
    date_end = models.DateTimeField(null=True, blank=True)
    date_close = models.DateTimeField(null=True, blank=True)
    client_type = models.CharField(max_length=254, null=True, blank=True)
    account = models.CharField(max_length=32, null=True, blank=True)
    address_full = models.CharField(max_length=4000, null=True, blank=True)
    work = models.CharField(max_length=4000, null=True, blank=True)
    mark_value = models.IntegerField(null=True, blank=True)
    mark_by_serv = models.IntegerField(null=True, blank=True)
    mark_by_add = models.IntegerField(null=True, blank=True)
    area = models.CharField(max_length=4000, null=True, blank=True)
    building_type = models.CharField(max_length=2, null=True, blank=True)
    mrf_name = models.CharField(max_length=16, null=True, blank=True)
    filial = models.CharField(max_length=128, null=True, blank=True)
    order_status = models.CharField(max_length=128, null=True, blank=True)

    given_main_works_amount = models.FloatField(blank=True, null=True)
    checked_main_works_amount = models.FloatField(blank=True, null=True)
    is_difference_of_main_works = models.BooleanField(default=False)

    given_additional_works_amount = models.FloatField(blank=True, null=True)
    checked_additional_works_amount = models.FloatField(blank=True, null=True)
    is_difference_of_additional_works = models.BooleanField(default=False)

    calculated_all_works_amount = models.FloatField(blank=True, null=True)
    checked_all_works_amount = models.FloatField(blank=True, null=True)
    is_difference_of_all_works = models.BooleanField(default=False)

    is_matching_the_manual = models.BooleanField(default=False)
    comment = models.TextField(blank=True, null=True)
    logs = models.TextField(blank=True, null=True)

Я заполнил эту модель скриптом который взял данные из другой БД, поэтому нет таких полей, как unique=True или db_index=True.

Просмотр: 

class MyModelView(
    mixins.RetrieveModelMixin, 
    mixins.ListModelMixin, 
    mixins.CreateModelMixin, 
    GenericViewSet
):
queryset = MyModel.objects.all().order_by('pk')
    serializer_class = MyModelSerializer
    filter_class = MyModelFilter
    filter_backends = (OrderingFilter, django_filters.rest_framework.DjangoFilterBackend, )
    pagination_class = ResultsSetPagination

Сериализатор: 

class MyModelSerializer(serializers.ModelSerializer):
    class Meta:
        model = MyModel
        fields = '__all__'  # I know this is not the best practice

Фильтр: 

class MyModelFilter(django_filters.FilterSet):
    process_number = django_filters.Filter(name='process_number', lookup_expr='icontains')
    process_number_exact = django_filters.Filter(name='process_number', lookup_expr='exact')

    order_number = django_filters.Filter(name='order_number', lookup_expr='icontains')
    order_number_exact = django_filters.Filter(name='order_number', lookup_expr='exact')

    dep_name_l1 = django_filters.Filter(name='dep_name_l1', lookup_expr='icontains')
    dep_name_l1_exact = django_filters.Filter(name='dep_name_l1', lookup_expr='exact')

    worksite = django_filters.Filter(name='worksite', lookup_expr='icontains')
    worksite_exact = django_filters.Filter(name='worksite', lookup_expr='exact')

    date_create_gte = django_filters.DateFilter(name='date_create', input_formats=['%Y-%m-%d'], lookup_expr='gte')
    date_create_lte = django_filters.DateFilter(name='date_create', input_formats=['%Y-%m-%d'], lookup_expr='lte')

    date_end_gte = django_filters.DateFilter(name='date_end', input_formats=['%Y-%m-%d'], lookup_expr='gte')
    date_end_lte = django_filters.DateFilter(name='date_end', input_formats=['%Y-%m-%d'], lookup_expr='lte')

    date_close_gte = django_filters.DateFilter(name='date_close', input_formats=['%Y-%m-%d'], lookup_expr='gte')
    date_close_lte = django_filters.DateFilter(name='date_close', input_formats=['%Y-%m-%d'], lookup_expr='lte')

    client_type = django_filters.Filter(name='client_type', lookup_expr='icontains')
    client_type_exact = django_filters.Filter(name='client_type', lookup_expr='exact')

    account = django_filters.Filter(name='account', lookup_expr='icontains')
    account_exact = django_filters.Filter(name='account', lookup_expr='exact')

    order_status = django_filters.Filter(name='order_status', lookup_expr='icontains')
    order_status_exact = django_filters.Filter(name='order_status', lookup_expr='exact')

    filial = django_filters.Filter(name='filial', lookup_expr='icontains')
    filial_number_exact = django_filters.Filter(name='filial', lookup_expr='exact')

    mrf_name = django_filters.Filter(name='mrf_name', lookup_expr='icontains')
    mrf_name_exact = django_filters.Filter(name='mrf_name', lookup_expr='exact')

    is_difference_of_main_works = django_filters.Filter(name='is_difference_of_main_works', lookup_expr='icontains')
    is_difference_of_main_works_exact = django_filters.Filter(name='is_difference_of_main_works', lookup_expr='exact')

    is_difference_of_additional_works = django_filters.Filter(
        name='is_difference_of_additional_works',
        lookup_expr='icontains'
    )
    is_difference_of_additional_works_exact = django_filters.Filter(
        name='is_difference_of_main_works',
        lookup_expr='exact'
    )

    is_difference_of_all_works = django_filters.Filter(
        name='is_difference_of_all_works',
        lookup_expr='icontains'
    )
    is_difference_of_all_works_exact = django_filters.Filter(
        name='is_difference_of_all_works',
        lookup_expr='exact'
    )

    status = django_filters.Filter(name='status', lookup_expr='icontains')
    status_exact = django_filters.Filter(name='status', lookup_expr='exact')

    class Meta:
        model = MyModel
        fields = [
            'process_number',
            'order_number',
            'dep_name_l1',
            'worksite',
            'date_create',
            'date_end',
            'date_close',
            'client_type',
            'account',
            'order_status',
            'filial',
            'mrf_name',
            'is_difference_of_main_works',
            'is_difference_of_additional_works',
            'is_difference_of_all_works',
        ]

Paginator: 

class ResultsSetPagination(PageNumberPagination):
    page_size = 20
    page_size_query_param = 'page_size'
    max_page_size = 100

    def get_paginated_response(self, data):
        next_page_number, previous_page_number = (None, None)
        if self.page.has_next():
            next_page_number = self.page.next_page_number()
        if self.page.has_previous():
            previous_page_number = self.page.previous_page_number()
        return Response(OrderedDict([
            ('count', self.page.paginator.count),
            ('current', self.page.number),
            ('next', next_page_number),
            ('previous', previous_page_number),
            ('page_size', self.page.paginator.per_page),
            ('results', data)
        ]))

Admin: 

admin.site.register(MyModel)

Я пытался удалить фильтры, изменить нумерацию страниц, отображать только идентификаторы, но ничего не помогло. Я предполагаю, что дело в большом количестве данных, хотя нумерация страниц должна помочь в таких случаях. Google нашел только темы об оптимизации моделей с отношениями. Буду рад любой помощи! Спасибо!

  • Платформа: Ubuntu 18.04 LTS
  • Python версия: 3.5.2
  • Django версия: 1.11
  • Версия DRF: 3.6.2
  • DB: PostgreSQL
  • Postgres версия: psql основная версия 9.5, основная версия сервера 9.6.
...