import json import re import sys import time import copy from notion_client import Client from specklepy.api.client import SpeckleClient from specklepy.api.credentials import get_default_account, get_local_accounts from specklepy.transports.server import ServerTransport from specklepy.api import operations from specklepy.objects.geometry import Polyline, Point from specklepy.objects import Base import os from functools import wraps import gradio as gr import requests from huggingface_hub import webhook_endpoint, WebhookPayload from fastapi import Request import datetime import gradio as gr from utils import * current_directory = os.path.dirname(os.path.abspath(__file__)) # Path to the config.json file config_file_path = os.path.join(current_directory, "config.json") with open(config_file_path, 'r') as f: config = json.load(f) # notion login speckle_token = os.environ.get("SPECKLE_TOKEN") notion_token = os.environ.get("NOTION_TOKEN") notion_token = notion_token notion = Client(auth=notion_token) # speckle CLIENT = SpeckleClient(host="https://speckle.xyz/") CLIENT.authenticate_with_token(token=speckle_token) # load config to variables def mainFunc(STREAM_ID, SOURCE_BRANCH, TARGET_BRANCH, UUID_COL, ATTR_METADATA, KPI_METADATA, DEFAULT_ATTRIBUTES, DATASET_NAME): if type(TARGET_BRANCH) == type([]): TARGET_BRANCH = SOURCE_BRANCH.replace(TARGET_BRANCH[0], TARGET_BRANCH[1]) # get data from notion databaseFUll_pages = fetch_all_database_pages(notion, ATTR_METADATA) time.sleep(1) kpi_database_pages = fetch_all_database_pages(notion, KPI_METADATA) time.sleep(1) # generate JSON files attributeMetaData, availableAttributes = notionTable2JSON(databaseFUll_pages, kpi_database_pages) # extract attribute/colum/feature names from notion table attributesOfInterest = DEFAULT_ATTRIBUTES for page in databaseFUll_pages: pv = get_property_value(page, "name") attributesOfInterest.append(pv) if UUID_COL not in attributesOfInterest: attributesOfInterest.append(UUID_COL) # get speckle data stream = getSpeckleStream(STREAM_ID, SOURCE_BRANCH, CLIENT) # navigate to list with speckle objects of interest try: stream_data = stream["@Data"]["@{0}"] except: print("something went wrong, try again with non-capital d") try: stream_data = stream["@data"]["@{0}"] except: print("check on speckle.com how to access the data") # ----- streamTrg = getSpeckleStream(STREAM_ID, TARGET_BRANCH, CLIENT) # navigate to list with speckle objects of interest try: stream_dataTrg = streamTrg["@Data"]["@{0}"] except: print("something went wrong, try again with non-capital d") try: stream_dataTrg = streamTrg["@analysis_geometry"]["@{0}"] except: print("check on speckle.com how to access the data") time.sleep(1) #----- idMapper = {} for i, obj in enumerate(stream_dataTrg): objDict = obj.__dict__ curUUid= objDict["uuid"] idMapper[curUUid] = i # ======== assamble new stream data ============ streamData_new = [] log = {"removedDatapoints":0,"removedID":[], "avg_attrRemoved":0, "removedAttr":[]} for i, obj in enumerate(stream_data): objDict = obj.__dict__ # REMOVE DATA POINTS ============================== # COND A: no landuses at all. pass_flag = True # REMOVE ATTRIBUTES =============================== datanew = Base() for k, v in objDict.items(): if k in attributesOfInterest: datanew[k] = v else: log["avg_attrRemoved"] +=1 log["removedAttr"].append(k) datanew["dataset"] = DATASET_NAME # keep geometry from target branch curUUid = objDict[UUID_COL] targetGeoRow = idMapper[curUUid] print(stream_dataTrg[targetGeoRow].__dict__.keys()) try: datanew["@geometry"] = stream_dataTrg[targetGeoRow].__dict__["@geometry"] except: datanew["@Geometry"] = stream_dataTrg[targetGeoRow].__dict__["@Geometry"] streamData_new.append(datanew) log["avg_attrRemoved"] = log["avg_attrRemoved"]/len(stream_data) log["removedAttr"] = list(set(log["removedAttr"])) stream_new = copy.deepcopy(stream) stream_new["@Data"]["@{0}"] = streamData_new #add additional data to stream stream_new["logs"] = json.dumps(log) stream_new["attributeMetaData"] = json.dumps(attributeMetaData) stream_new["availableAttributes"] = json.dumps(availableAttributes) stream_new["datasetMetadata"] = DATASET_NAME # set stream and branch # Get transport transport = ServerTransport(client=CLIENT, stream_id=STREAM_ID) # Send the data object to the speckle stream object_id = operations.send(stream_new, [transport]) time.sleep(1) # Create a new commit with the new object commit_id = CLIENT.commit.create( STREAM_ID, object_id= object_id, message="auto commit, removed datapoints: " + str( log["removedDatapoints"]) + "; avg. removed attributes: "+ str(int(log["avg_attrRemoved"])), branch_name=TARGET_BRANCH, ) print(commit_id) @webhook_endpoint async def update_streams(request: Request): # Initialize flag should_continue = False # Read the request body as JSON payload = await request.json() print("============= payload =============") print(payload) print("============= config ==============") print(config) payload = payload["payload"] # Check if the payload structure matches the expected format # Assuming payload["event"]["event_name"] gives you the event type event_name = payload["event"]["event_name"] streamName = payload.get("stream", {}).get("name") # Extract branchName for commit_update events from the "old" commit data if event_name == "commit_update": branchName = payload.get("event", {}).get("data", {}).get("old", {}).get("branchName") else: branchName = payload.get("event", {}).get("data", {}).get("commit", {}).get("branchName") # List of valid event types valid_event_types = ["commit_create", "commit_delete", "commit_update"] if event_name in valid_event_types: if streamName in config: if branchName in config[streamName]: stream_config = config[streamName][branchName] should_continue = True else: print(f"Branch name {branchName} not found in config for stream {streamName}.") else: print(f"Stream name {streamName} not found in config.") else: print(f"Event type {event_name} is not one of the specified types.") # If the flag is True, continue running the main part of the code if should_continue: config_entry = config[streamName][branchName] STREAM_NAME = config_entry["STREAM_NAME"] STREAM_ID = config_entry["STREAM_ID"] SOURCE_BRANCH = config_entry["SOURCE_BRANCH"] TARGET_BRANCH = config_entry["TARGET_BRANCH"] UUID_COL = config_entry["UUID_COL"] ATTR_METADATA = config_entry["ATTR_METADATA"] KPI_METADATA = config_entry["KPI_METADATA"] DEFAULT_ATTRIBUTES = config_entry["DEFAULT_ATTRIBUTES"] DATASET_NAME = config_entry.get("DATASET_NAME") # main mainFunc(STREAM_ID, SOURCE_BRANCH, TARGET_BRANCH, UUID_COL, ATTR_METADATA, KPI_METADATA, DEFAULT_ATTRIBUTES, DATASET_NAME)