From 75d1498ae8e7cd71c3a156e1be3b9fd23acb864b Mon Sep 17 00:00:00 2001 From: Alexandr Karpov Date: Sat, 22 Oct 2022 03:09:39 +0300 Subject: [PATCH] fixed excel parser, processed to db --- .gitignore | 2 + app/search/models.py | 26 +++++------ app/search/services/load_products.py | 64 ++++++++++++++++------------ 3 files changed, 51 insertions(+), 41 deletions(-) diff --git a/.gitignore b/.gitignore index 4de1b5a..51c51f6 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ app/media/ app/static/ .idea/ +product_100_000.json + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/app/search/models.py b/app/search/models.py index 166d04c..6dc7e69 100644 --- a/app/search/models.py +++ b/app/search/models.py @@ -2,8 +2,8 @@ from django.db import models class Characteristic(models.Model): - name = models.CharField("Имя", blank=False, max_length=1000) - value = models.CharField("Значение", blank=False, max_length=1000) + name = models.TextField("Имя", blank=False) + value = models.TextField("Значение", blank=False) def __str__(self): return str(self.name) @@ -12,13 +12,13 @@ class Characteristic(models.Model): return {"name": self.name, "value": self.value} class Meta: - db_table = "Characteristic" + db_table = "characteristic" class UnitCharacteristic(models.Model): - name = models.CharField("Имя", blank=False, max_length=1000) - value = models.CharField("Значение", blank=False, max_length=1000) - unit = models.CharField("Размерность", blank=False, max_length=1000) + name = models.TextField("Имя", blank=False) + value = models.TextField("Значение", blank=False) + unit = models.TextField("Размерность", blank=False) def __str__(self): return str(self.name) @@ -27,24 +27,24 @@ class UnitCharacteristic(models.Model): return {"name": self.name, "value": self.value, "unit": self.unit} class Meta: - db_table = "UnitCharacteristic" + db_table = "unit_characteristic" class Category(models.Model): - name = models.CharField("Имя", unique=True, blank=False, max_length=1000) + name = models.TextField("Имя", unique=True, blank=False) def __str__(self): return str(self.name) class Meta: - db_table = "Category" + db_table = "category" class Product(models.Model): id = models.IntegerField( "ID CTE", primary_key=True, unique=True, blank=False, null=False, db_index=True ) - name = models.CharField("Название CTE", unique=True, blank=False, max_length=1000) + name = models.TextField("Название CTE", unique=True, blank=False) category = models.ForeignKey( Category, related_name="products", on_delete=models.CASCADE ) @@ -62,7 +62,7 @@ class Product(models.Model): } class Meta: - db_table = "Product" + db_table = "product" class ProductCharacteristic(models.Model): @@ -77,7 +77,7 @@ class ProductCharacteristic(models.Model): return f"{self.product} in {self.characteristic}" class Meta: - db_table = "ProductCharacteristic" + db_table = "product_characteristic" class ProductUnitCharacteristic(models.Model): @@ -92,4 +92,4 @@ class ProductUnitCharacteristic(models.Model): return f"{self.product} in {self.characteristic}" class Meta: - db_table = "ProductUnitCharacteristic" + db_table = "product_unit_characteristic" diff --git a/app/search/services/load_products.py b/app/search/services/load_products.py index 7284ef1..c28f54c 100644 --- a/app/search/services/load_products.py +++ b/app/search/services/load_products.py @@ -22,32 +22,40 @@ def load(): def load_excel(): - df = pd.read_excel("data1.xlsx", sheet_name="Sheet1") + df = pd.read_excel("data.xlsx", sheet_name="Запрос1") for row in range(df.shape[0]): - print(df.iat[row, 0], df.iat[row, 1], df.iat[row, 2]) - if Product.objects.filter(id=df.iat[row, 0]).exists(): - Product.objects.filter(id=df.iat[row, 0]).delete() - product = Product(id=df.iat[row, 0]) - product.name = df.iat[row, 1] - category = Category.objects.get_or_create(name=df.iat[row, 2])[0] - product.category = category - product.save() - for cat in literal_eval(df.iat[row, 4]): - try: - if "Unit" in cat: - ProductUnitCharacteristic.objects.get_or_create( - characteristic=UnitCharacteristic.objects.get_or_create( - name=cat["Name"], value=cat["Value"], unit=cat["Unit"] - )[0], - product=product, - ) - else: - ProductCharacteristic.objects.get_or_create( - characteristic=Characteristic.objects.get_or_create( - name=cat["Name"], value=cat["Value"] - )[0], - product=product, - ) - except KeyError: - # Empty Value - continue + try: + print(df.iat[row, 0], df.iat[row, 1], df.iat[row, 2]) + if Product.objects.filter(id=df.iat[row, 0]).exists(): + Product.objects.filter(id=df.iat[row, 0]).delete() + product = Product(id=df.iat[row, 0]) + product.name = df.iat[row, 1] + category = Category.objects.get_or_create(name=df.iat[row, 2])[0] + product.category = category + product.save() + if df.iat[row, 4]: + for cat in literal_eval(df.iat[row, 4]): + try: + if "Unit" in cat: + ProductUnitCharacteristic.objects.get_or_create( + characteristic=UnitCharacteristic.objects.get_or_create( + name=cat["Name"], + value=cat["Value"], + unit=cat["Unit"], + )[0], + product=product, + ) + else: + ProductCharacteristic.objects.get_or_create( + characteristic=Characteristic.objects.get_or_create( + name=cat["Name"], value=cat["Value"] + )[0], + product=product, + ) + except KeyError: + # Empty Value + continue + except BaseException: + # malformed node or string: nan \ duplicate key + print("СКОРОСШИВАТЕЛЬ") + continue