import streamlit as st import time from selenium import webdriver from selenium.webdriver.firefox.options import Options from selenium.webdriver.firefox.service import Service from webdriver_manager.firefox import GeckoDriverManager from datetime import datetime from bs4 import BeautifulSoup import pandas as pd import sqlite3 import matplotlib.pyplot as plt import requests import networkx as nx from sklearn.linear_model import LinearRegression from sklearn.model_selection import train_test_split from sklearn.metrics import mean_squared_error def make_graph(): G = nx.DiGraph() G = nx.Graph() G.add_edges_from([('Nimfadora T','Tad T'), ('Andromeda B','Nimfadora T'), ('Andromeda B','Tad T'), ('Andromeda B','Kingus B'), ('Druela R','Kingus B'), ('Andromeda B','Druela R'), ('Narcisa B','Druela R'), ('Narcisa B', 'Kingus B'), ('Lucius M','Narcisa B'), ('Draco M', 'Lucius M'), ('Draco M', 'Narcisa B'), ('Draco M','Astoria G'), ('Scorpius M','Astoria G'), ('Scorpius M', 'Draco M'), ('Rimus L','Nimfadora T'), ('Ted L', 'Rimus L'), ('Ted L','Nimfadora T')]) # Отображение графа fig, ax = plt.subplots() pos = nx.spring_layout(G) nx.draw_networkx(G, pos, with_labels=True, node_color='lightblue', node_size=500, edge_color='gray', width=2, alpha=0.7, ax=ax) st.pyplot(fig) def linear_regression(): df = pd.read_csv('imdb_top_1000.csv') df['Runtime'] = df['Runtime'].astype(str) df['IMDB_Rating'] = df['IMDB_Rating'].astype(str) df['Runtime'] = df['Runtime'].str.replace(r'\D', '') df['IMDB_Rating'] = df['IMDB_Rating'].str.replace(r'\D', '').astype(float) X = df['Runtime'].values.reshape(-1, 1) y = df['IMDB_Rating'].values X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) regressor = LinearRegression() regressor.fit(X_train, y_train) y_pred = regressor.predict(X_test) mse = mean_squared_error(y_test, y_pred) return mse print('Mean Squared Error:', mse) def createDriver(url): firefoxOptions = Options() firefoxOptions.add_argument("--headless") service = Service(GeckoDriverManager().install()) driver = webdriver.Firefox( options=firefoxOptions, service=service, ) driver.get(url) time.sleep(2) main_page = driver.page_source soup = BeautifulSoup(main_page, 'html.parser') return soup def scrape_weather_data(soup): data = soup.find(class_="chronicle-table").find('tbody').find_all('tr') data_value = data[244].find_all('nobr') data_month = data[0].find_all('td') temp = [] temp_month = [] for i in data_value: temp.append(float(i.text)) for j in range(0, len(data_month)): temp_month.append(data_month[j].text) temp_month.pop() temp.pop() return temp_month, temp def get_weather_data(station, start_date, end_date): if station == "Barnaul": station = "Asia/Barnaul" elif station == "Moscow": station = "Europe/Moscow" else: station = "Europe/Berlin" url = "https://meteostat.p.rapidapi.com/stations/hourly" querystring = {"station":"10637","start":start_date,"end":end_date,"tz":station} headers = { "X-RapidAPI-Key": "9c8eb62f1fmsh82eba345d265b05p1541b2jsna3309cd23406", "X-RapidAPI-Host": "meteostat.p.rapidapi.com" } response = requests.get(url, headers=headers, params=querystring) data = response.json() data_mas = [] for j in data['data']: data_date = j['time'] windy_date = str(j['dwpt'])+" km/h" osadki = str(j['prcp']) + " mm" temperature =str(j['temp']) + " °C" data_mas.append([data_date, temperature, windy_date, osadki]) return data_mas def process_data(data): df = pd.DataFrame(data) df = df.rename(columns={ 0 : 'date_time', 1: 'temperature', 2: 'wind_speed', 3: 'humidity'}) # Преобразование типов данных и очистка данных df["temperature"] = df["temperature"].str.extract(r"(\d+)").astype(float) df["humidity"] = df["humidity"].str.extract(r"(\d+)").astype(float) df["wind_speed"] = df["wind_speed"].str.extract(r"(\d+)").astype(float) df = df.drop_duplicates() df = df.fillna(0) return df def analyze_data(df): # Вычисление статистических метрик mean_temperature = round(df["temperature"].mean(), 2) median_temperature =round(df["temperature"].median(),2) std_temperature = round(df["temperature"].std(),2) results = { "mean_temperature": mean_temperature, "median_temperature": median_temperature, "std_temperature": std_temperature } return results def visualize_data_api(df): fig, ax = plt.subplots() ax.plot(df['date_time'], df['temperature']) plt.xticks(rotation=90) ax.set_xlabel('Date') ax.set_ylabel('Temperature') ax.set_title('Temperature Over Time') fig.set_size_inches(20, 15) st.pyplot(fig) def visualize_data_parsing(mas_month, math_temp): fig, ax = plt.subplots() ax.plot(mas_month, math_temp) plt.xticks(rotation=90) ax.set_xlabel('Month') ax.set_ylabel('Temperature') ax.set_title('Temperature per year 2022 in Moscow') fig.set_size_inches(10, 6) st.pyplot(fig) def save_to_database(dateNow,timeNow, station, start_date, end_date): conn = sqlite3.connect('statistic.db') sql = conn.cursor() sql.execute("""INSERT INTO statistic VALUES (?, ?, ?, ?,?)""", (dateNow,timeNow, station, start_date, end_date)) conn.commit() conn.close() def view_dataBase(): conn = sqlite3.connect('statistic.db') df = pd.read_sql_query("SELECT * from statistic", conn) return df # Демонстрация проекта с помощью Streamlit def streamlit_demo(): st.title("A few useful things!") st.title("Black family tree graph from harry potter:") make_graph() st.title('Rating depends on the length of the film:') mse_error = linear_regression() st.write(f'Mean Squared Error: {mse_error}') st.title("Weather Analysis") temperature_moscow2022_button = st.button("Show temperature in Moscow for 2022") #кнопка для парсинга температуры в москве за 2022 год if temperature_moscow2022_button: url = "http://www.pogodaiklimat.ru/history/27612.htm" soup = createDriver(url) scraped_month, scraped_temp = scrape_weather_data(soup) visualize_data_parsing(scraped_month, scraped_temp) # Добавить элементы управления для выбора города, временного диапазона и отображения результатов city = st.selectbox("Select City", ["Moscow", "Berlin", "Barnaul"]) start_date = st.date_input("Select Start Date") end_date = st.date_input("Select End Date") temperature_period_button = st.button("Submit") #кнопка для получения данных о погоде через api if temperature_period_button: now = datetime.now() timeNow = now.strftime("%H:%M:%S") dateNow = now.date() save_to_database(dateNow,timeNow, city, start_date, end_date) # Получение данных о погоде для выбранного города и временного диапазона weather_data = get_weather_data(city, start_date, end_date) processed_data = process_data(weather_data) # Анализ данных analyzed_data = analyze_data(processed_data) # Визуализация данных visualize_data_api(processed_data) st.title("Data analysis") for key, value in analyzed_data.items(): st.write(key, value) statistic_button = st.button("View visit statistics") #кнопка для просмотра статистика нажатий кнопки "Submit" if statistic_button: df = view_dataBase() st.write(df) def main(): streamlit_demo() if __name__ == '__main__': main()