fixed excel parser, processed to db

.gitignore

@@ -2,6 +2,8 @@ app/media/
 app/static/
 .idea/
 
+product_100_000.json
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

app/search/models.py

@@ -2,8 +2,8 @@ from django.db import models
 
 
 class Characteristic(models.Model):
-    name = models.CharField("Имя", blank=False, max_length=1000)
-    value = models.CharField("Значение", blank=False, max_length=1000)
+    name = models.TextField("Имя", blank=False)
+    value = models.TextField("Значение", blank=False)
 
     def __str__(self):
         return str(self.name)
@@ -12,13 +12,13 @@ class Characteristic(models.Model):
         return {"name": self.name, "value": self.value}
 
     class Meta:
-        db_table = "Characteristic"
+        db_table = "characteristic"
 
 
 class UnitCharacteristic(models.Model):
-    name = models.CharField("Имя", blank=False, max_length=1000)
-    value = models.CharField("Значение", blank=False, max_length=1000)
-    unit = models.CharField("Размерность", blank=False, max_length=1000)
+    name = models.TextField("Имя", blank=False)
+    value = models.TextField("Значение", blank=False)
+    unit = models.TextField("Размерность", blank=False)
 
     def __str__(self):
         return str(self.name)
@@ -27,24 +27,24 @@ class UnitCharacteristic(models.Model):
         return {"name": self.name, "value": self.value, "unit": self.unit}
 
     class Meta:
-        db_table = "UnitCharacteristic"
+        db_table = "unit_characteristic"
 
 
 class Category(models.Model):
-    name = models.CharField("Имя", unique=True, blank=False, max_length=1000)
+    name = models.TextField("Имя", unique=True, blank=False)
 
     def __str__(self):
         return str(self.name)
 
     class Meta:
-        db_table = "Category"
+        db_table = "category"
 
 
 class Product(models.Model):
     id = models.IntegerField(
         "ID CTE", primary_key=True, unique=True, blank=False, null=False, db_index=True
     )
-    name = models.CharField("Название CTE", unique=True, blank=False, max_length=1000)
+    name = models.TextField("Название CTE", unique=True, blank=False)
     category = models.ForeignKey(
         Category, related_name="products", on_delete=models.CASCADE
     )
@@ -62,7 +62,7 @@ class Product(models.Model):
         }
 
     class Meta:
-        db_table = "Product"
+        db_table = "product"
 
 
 class ProductCharacteristic(models.Model):
@@ -77,7 +77,7 @@ class ProductCharacteristic(models.Model):
         return f"{self.product} in {self.characteristic}"
 
     class Meta:
-        db_table = "ProductCharacteristic"
+        db_table = "product_characteristic"
 
 
 class ProductUnitCharacteristic(models.Model):
@@ -92,4 +92,4 @@ class ProductUnitCharacteristic(models.Model):
         return f"{self.product} in {self.characteristic}"
 
     class Meta:
-        db_table = "ProductUnitCharacteristic"
+        db_table = "product_unit_characteristic"

app/search/services/load_products.py

@@ -22,8 +22,9 @@ def load():
 
 
 def load_excel():
-    df = pd.read_excel("data1.xlsx", sheet_name="Sheet1")
+    df = pd.read_excel("data.xlsx", sheet_name="Запрос1")
     for row in range(df.shape[0]):
+        try:
             print(df.iat[row, 0], df.iat[row, 1], df.iat[row, 2])
             if Product.objects.filter(id=df.iat[row, 0]).exists():
                 Product.objects.filter(id=df.iat[row, 0]).delete()
@@ -32,12 +33,15 @@ def load_excel():
             category = Category.objects.get_or_create(name=df.iat[row, 2])[0]
             product.category = category
             product.save()
+            if df.iat[row, 4]:
                 for cat in literal_eval(df.iat[row, 4]):
                     try:
                         if "Unit" in cat:
                             ProductUnitCharacteristic.objects.get_or_create(
                                 characteristic=UnitCharacteristic.objects.get_or_create(
-                            name=cat["Name"], value=cat["Value"], unit=cat["Unit"]
+                                    name=cat["Name"],
+                                    value=cat["Value"],
+                                    unit=cat["Unit"],
                                 )[0],
                                 product=product,
                             )
@@ -51,3 +55,7 @@ def load_excel():
                     except KeyError:
                         # Empty Value
                         continue
+        except BaseException:
+            # malformed node or string: nan \ duplicate key
+            print("СКОРОСШИВАТЕЛЬ")
+            continue