Spaces:
Sleeping
Sleeping
| import specklepy | |
| from specklepy.api.client import SpeckleClient | |
| from specklepy.api.credentials import get_default_account, get_local_accounts | |
| from specklepy.transports.server import ServerTransport | |
| from specklepy.api import operations | |
| from specklepy.objects.geometry import Polyline, Point, Mesh | |
| import json | |
| import pandas as pd | |
| import numpy as n | |
| from specklepy.api.wrapper import StreamWrapper | |
| import requests | |
| from datetime import datetime | |
| import copy | |
| def get_dataframe(objects_raw, return_original_df=False): | |
| """ | |
| Creates a pandas DataFrame from a list of raw Speckle objects. | |
| Args: | |
| objects_raw (list): List of raw Speckle objects. | |
| return_original_df (bool, optional): If True, the function also returns the original DataFrame before any conversion to numeric. Defaults to False. | |
| Returns: | |
| pd.DataFrame or tuple: If return_original_df is False, returns a DataFrame where all numeric columns have been converted to their respective types, | |
| and non-numeric columns are left unchanged. | |
| If return_original_df is True, returns a tuple where the first item is the converted DataFrame, | |
| and the second item is the original DataFrame before conversion. | |
| This function iterates over the raw Speckle objects, creating a dictionary for each object that excludes the '@Geometry' attribute. | |
| These dictionaries are then used to create a pandas DataFrame. | |
| The function attempts to convert each column to a numeric type if possible, and leaves it unchanged if not. | |
| Non-convertible values in numeric columns are replaced with their original values. | |
| """ | |
| # dataFrame | |
| df_data = [] | |
| # Iterate over speckle objects | |
| for obj_raw in objects_raw: | |
| obj = obj_raw.__dict__ | |
| df_obj = {k: v for k, v in obj.items() if k != '@Geometry'} | |
| df_data.append(df_obj) | |
| # Create DataFrame and GeoDataFrame | |
| df = pd.DataFrame(df_data) | |
| # Convert columns to float or int if possible, preserving non-convertible values <- | |
| df_copy = df.copy() | |
| for col in df.columns: | |
| df[col] = pd.to_numeric(df[col], errors='coerce') | |
| df[col].fillna(df_copy[col], inplace=True) | |
| if return_original_df: | |
| return df, df_copy | |
| else: | |
| return df | |
| def aggregate_data_optimized(df_a, df_b, uuid_col_name, ref_col_name, exclude_columns): | |
| # Ensure the uuid_col_name is included for the merging process | |
| columns_to_use = [col for col in df_a.columns if col not in exclude_columns or col == uuid_col_name] | |
| df_a_filtered = df_a[columns_to_use] | |
| # Perform the merge without adding suffixes, as we intend to overwrite existing columns in df_b | |
| df_merged = pd.merge(df_b, df_a_filtered, how='left', left_on=ref_col_name, right_on=uuid_col_name, suffixes=(None, '_y')) | |
| # Initialize a dictionary for logging | |
| log_dict = { | |
| 'info': [], | |
| 'warning': [], | |
| 'summary': {} | |
| } | |
| # Logging matched and unmatched counts | |
| matched_count = df_merged[ref_col_name].notnull().sum() | |
| unmatched_count = df_b.shape[0] - matched_count | |
| log_dict['summary'] = { | |
| 'matched_count': matched_count, | |
| 'unmatched_count': unmatched_count, | |
| 'total_rows_processed': df_b.shape[0] | |
| } | |
| log_dict['info'].append("Data aggregation completed successfully.") | |
| # Explicitly overwrite columns in df_b with those from df_a, based on the merge | |
| for col in columns_to_use: | |
| if col not in exclude_columns and col != uuid_col_name and f'{col}_y' in df_merged: | |
| df_merged[col] = df_merged.pop(f'{col}_y') | |
| # Drop any remaining '_y' columns that were not explicitly handled | |
| df_merged = df_merged.loc[:, ~df_merged.columns.str.endswith('_y')] | |
| # Additionally, if the uuid_col_name is not part of the original df_b columns and is only used for matching, it should be removed | |
| if uuid_col_name not in df_b.columns: | |
| df_merged.drop(columns=[uuid_col_name], inplace=True, errors='ignore') | |
| return df_merged, log_dict | |
| def updateStreamAnalysisFast(client, new_data, stream_id, branch_name, geometryGroupPath=None, match_by_id="", return_original=False, comm_message=""): | |
| if geometryGroupPath is None: | |
| geometryGroupPath = ["@Speckle", "Geometry"] | |
| branch = client.branch.get(stream_id, branch_name, 2) | |
| latest_commit = branch.commits.items[0] | |
| commit = client.commit.get(stream_id, latest_commit.id) | |
| transport = ServerTransport(client=client, stream_id=stream_id) | |
| res = operations.receive(commit.referencedObject, transport) | |
| objects_raw = res[geometryGroupPath[0]][geometryGroupPath[1]] | |
| # Pre-create a mapping from IDs to objects for faster lookup | |
| id_to_object_map = {obj[match_by_id]: obj for obj in objects_raw} if match_by_id else {i: obj for i, obj in enumerate(objects_raw)} | |
| # Pre-process DataFrame if match_by_id is provided | |
| if match_by_id: | |
| new_data.set_index(match_by_id, inplace=True) | |
| # Update objects in a more efficient way using .items() | |
| for local_id, updates in new_data.iterrows(): | |
| target_object = id_to_object_map.get(str(local_id)) | |
| if target_object: | |
| for col_name, value in updates.items(): | |
| target_object[col_name] = value | |
| # Send updated objects back to Speckle | |
| new_objects_raw_speckle_id = operations.send(base=res, transports=[transport]) | |
| commit_id = client.commit.create(stream_id=stream_id, branch_name=branch_name, object_id=new_objects_raw_speckle_id, message=comm_message + "#+SourceCommit: "+latest_commit.id) | |
| print("commit created") | |
| if return_original: | |
| return objects_raw # as back-up | |
| return commit_id | |
| def getSpeckleStream(stream_id, | |
| branch_name, | |
| client, | |
| commit_id="" | |
| ): | |
| """ | |
| Retrieves data from a specific branch of a speckle stream. | |
| Args: | |
| stream_id (str): The ID of the speckle stream. | |
| branch_name (str): The name of the branch within the speckle stream. | |
| client (specklepy.api.client.Client, optional): A speckle client. Defaults to a global `client`. | |
| commit_id (str): id of a commit, if nothing is specified, the latest commit will be fetched | |
| Returns: | |
| dict: The speckle stream data received from the specified branch. | |
| This function retrieves the last commit from a specific branch of a speckle stream. | |
| It uses the provided speckle client to get the branch and commit information, and then | |
| retrieves the speckle stream data associated with the last commit. | |
| It prints out the branch details and the creation dates of the last three commits for debugging purposes. | |
| """ | |
| print("updated A") | |
| # set stream and branch | |
| try: | |
| branch = client.branch.get(stream_id, branch_name, 1) | |
| print(branch) | |
| except: | |
| branch = client.branch.get(stream_id, branch_name, 1) | |
| print(branch) | |
| print("branch info:", branch) | |
| #[print(ite.createdAt) for ite in branch.commits.items] | |
| if commit_id == "": | |
| latest_commit = branch.commits.items[0] | |
| choosen_commit_id = latest_commit.id | |
| commit = client.commit.get(stream_id, choosen_commit_id) | |
| print("latest commit ", branch.commits.items[0].createdAt, " was choosen") | |
| elif type(commit_id) == type("s"): # string, commit uuid | |
| choosen_commit_id = commit_id | |
| commit = client.commit.get(stream_id, choosen_commit_id) | |
| print("provided commit ", choosen_commit_id, " was choosen") | |
| elif type(commit_id) == type(1): #int | |
| latest_commit = branch.commits.items[commit_id] | |
| choosen_commit_id = latest_commit.id | |
| commit = client.commit.get(stream_id, choosen_commit_id) | |
| print(commit) | |
| print(commit.referencedObject) | |
| # get transport | |
| transport = ServerTransport(client=client, stream_id=stream_id) | |
| #speckle stream | |
| res = operations.receive(commit.referencedObject, transport) | |
| return res |