Browse Source

fix:records.html

tanjingyu 1 week ago
parent
commit
da6667cd64
7 changed files with 492 additions and 430 deletions
  1. 2 0
      app/models.py
  2. 4 2
      app/schemas.py
  3. 3 1
      app/services/storage_service.py
  4. 7 1
      app/services/webhook_service.py
  5. 399 426
      app/static/records.html
  6. 44 0
      migrate_raw.py
  7. 33 0
      migrate_schema.py

+ 2 - 0
app/models.py

@@ -29,6 +29,7 @@ class DataVersion(Base):
     stage = Column(String(200), nullable=False)
     commit_id = Column(String(64), nullable=False)
     author = Column(String(50))
+    commit_message = Column(Text)
     manifest_snapshot = Column(Text)
     created_at = Column(DateTime(timezone=True), server_default=func.now())
 
@@ -66,6 +67,7 @@ class DataRecord(Base):
     version_id = Column(String(26), ForeignKey("data_versions.id"))
     stage = Column(String(200), index=True)
     commit_id = Column(String(64))
+    commit_message = Column(Text)
     group_key = Column(String(255))
     
     inputs = Column(JSON)

+ 4 - 2
app/schemas.py

@@ -52,6 +52,7 @@ class DataVersionBase(BaseModel):
     stage: str
     commit_id: str
     author: Optional[str] = None
+    commit_message: Optional[str] = None
     manifest_snapshot: Optional[str] = None
 
 
@@ -75,10 +76,11 @@ DataVersion = DataVersionOut
 class DataRecordBase(BaseModel):
     stage: str
     commit_id: str
+    commit_message: Optional[str] = None
     group_key: Optional[str] = None
     content_hash: Optional[str] = None
-    inputs: list = []
-    outputs: list = []
+    inputs: Optional[List] = []
+    outputs: Optional[List] = []
     author: Optional[str] = None
 
 

+ 3 - 1
app/services/storage_service.py

@@ -25,13 +25,14 @@ class StorageService:
             self.db.refresh(project)
         return project
 
-    def create_version(self, project_id: str, stage: str, commit_id: str, author: str, manifest: str) -> DataVersion | None:
+    def create_version(self, project_id: str, stage: str, commit_id: str, author: str, manifest: str, commit_message: str = None) -> DataVersion | None:
         """Create a new data version. Returns None if a duplicate exists (IntegrityError)."""
         version = DataVersion(
             project_id=project_id,
             stage=stage,
             commit_id=commit_id,
             author=author,
+            commit_message=commit_message,
             manifest_snapshot=manifest
         )
         try:
@@ -141,6 +142,7 @@ class StorageService:
                 version_id=version.id,
                 stage=version.stage,
                 commit_id=version.commit_id,
+                commit_message=version.commit_message,
                 group_key=group_key,
                 inputs=data["inputs"],
                 outputs=data["outputs"],

+ 7 - 1
app/services/webhook_service.py

@@ -114,9 +114,15 @@ class WebhookService:
                 logger.info(f"Stage '{stage_name}' already processed. Skipping.")
                 continue
 
+            # Get commit message from payload if available
+            commit_msg = None
+            commits = payload.get("commits", [])
+            if commits:
+                commit_msg = commits[0].get("message")
+
             # Create version for this stage
             version = self.storage.create_version(
-                project.id, stage_name, after_sha, author_name, manifest=manifest_content
+                project.id, stage_name, after_sha, author_name, manifest=manifest_content, commit_message=commit_msg
             )
 
             if not version:

File diff suppressed because it is too large
+ 399 - 426
app/static/records.html


+ 44 - 0
migrate_raw.py

@@ -0,0 +1,44 @@
+import pymysql
+import os
+
+def migrate():
+    try:
+        conn = pymysql.connect(
+            host='rm-t4n8oyqunr5b4461s6o.mysql.singapore.rds.aliyuncs.com',
+            port=3306,
+            user='developer_saas',
+            password='developer_saas#Aiddit',
+            db='data_nexus'
+        )
+        with conn.cursor() as cursor:
+            print("Connected to DB")
+            
+            # 1. Add commit_message to data_records
+            try:
+                cursor.execute("ALTER TABLE data_records ADD COLUMN commit_message TEXT DEFAULT NULL;")
+                print("Added commit_message to data_records")
+            except Exception as e:
+                print(f"Skipping data_records.commit_message: {e}")
+
+            # 2. Add commit_message to data_versions
+            try:
+                cursor.execute("ALTER TABLE data_versions ADD COLUMN commit_message TEXT DEFAULT NULL;")
+                print("Added commit_message to data_versions")
+            except Exception as e:
+                print(f"Skipping data_versions.commit_message: {e}")
+
+            # 3. Add content_hash to data_records
+            try:
+                cursor.execute("ALTER TABLE data_records ADD COLUMN content_hash VARCHAR(64) DEFAULT NULL;")
+                print("Added content_hash to data_records")
+            except Exception as e:
+                print(f"Skipping data_records.content_hash: {e}")
+
+        conn.commit()
+        conn.close()
+        print("Migration done")
+    except Exception as e:
+        print(f"Global error: {e}")
+
+if __name__ == "__main__":
+    migrate()

+ 33 - 0
migrate_schema.py

@@ -0,0 +1,33 @@
+from sqlalchemy import text
+from app.database import engine
+
+def run_migrations():
+    with engine.connect() as conn:
+        print("Starting migrations...")
+        
+        # 1. Add commit_message to data_records
+        try:
+            conn.execute(text("ALTER TABLE data_records ADD COLUMN commit_message TEXT DEFAULT NULL;"))
+            print("Added commit_message to data_records")
+        except Exception as e:
+            print(f"Skipping commit_message for data_records: {e}")
+
+        # 2. Add commit_message to data_versions
+        try:
+            conn.execute(text("ALTER TABLE data_versions ADD COLUMN commit_message TEXT DEFAULT NULL;"))
+            print("Added commit_message to data_versions")
+        except Exception as e:
+            print(f"Skipping commit_message for data_versions: {e}")
+
+        # 3. Add content_hash to data_records
+        try:
+            conn.execute(text("ALTER TABLE data_records ADD COLUMN content_hash VARCHAR(64) DEFAULT NULL;"))
+            print("Added content_hash to data_records")
+        except Exception as e:
+            print(f"Skipping content_hash for data_records: {e}")
+        
+        conn.commit()
+        print("Migrations complete.")
+
+if __name__ == "__main__":
+    run_migrations()

Some files were not shown because too many files changed in this diff