updates

DataExploration.ipynb

@@ -599,19 +599,72 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
     "import polars as pl\n",
     "import pathlib\n",
-    "ROOT    = pathlib.Path(r\"C:\\Users\\Romain\\OneDrive - KU Leuven\\Masters\\MBIS\\Year 2\\Semester 2\\Modern Data Analytics\\CORDIS\")\n",
-    "OUTDIR  = ROOT / \"combined\"\n",
+    "ROOT    = pathlib.Path(r\"C:\\Users\\Romain\\OneDrive - KU Leuven\\MDA\\backend\\data\")\n",
+    "OUTDIR  = ROOT #/ \"combined\"\n",
     "\n",
-    "consolidated = pl.read_parquet(OUTDIR / \"consolidated.parquet\")\n",
+    "#consolidated = pl.read_parquet(OUTDIR / \"consolidated.parquet\")\n",
     "consolidated_clean = pl.read_parquet(OUTDIR / \"consolidated_clean.parquet\")\n"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div><style>\n",
+       ".dataframe > thead > tr,\n",
+       ".dataframe > tbody > tr {\n",
+       "  text-align: right;\n",
+       "  white-space: pre-wrap;\n",
+       "}\n",
+       "</style>\n",
+       "<small>shape: (5, 68)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>id</th><th>acronym</th><th>status</th><th>title</th><th>startDate</th><th>endDate</th><th>totalCost</th><th>ecMaxContribution</th><th>legalBasis</th><th>topics</th><th>ecSignatureDate</th><th>frameworkProgramme</th><th>masterCall</th><th>subCall</th><th>fundingScheme</th><th>nature</th><th>objective</th><th>contentUpdateDate</th><th>rcn</th><th>grantDoi</th><th>programmeFolder</th><th>list_deliverableType</th><th>list_url</th><th>list_contentUpdateDate</th><th>list_authors</th><th>list_title</th><th>list_doi</th><th>list_journalTitle</th><th>list_isPublishedAs</th><th>list_publishedYear</th><th>list_contentUpdateDate_publi</th><th>list_title_report</th><th>list_attachment</th><th>list_contentUpdateDate_report</th><th>list_organisationID</th><th>list_country</th><th>list_name</th><th>list_SME</th><th>list_city</th><th>list_geolocation</th><th>list_organizationURL</th><th>list_role</th><th>list_ecContribution</th><th>list_netEcContribution</th><th>list_totalCost</th><th>list_endOfParticipation</th><th>list_activityType</th><th>list_contentUpdateDate_org</th><th>list_physUrl</th><th>list_availableLanguages</th><th>list_status</th><th>list_archivedDate</th><th>list_type</th><th>list_source</th><th>list_represents</th><th>list_legalBasis</th><th>list_title_legal</th><th>list_uniqueProgrammePart</th><th>list_topic</th><th>list_title_topic</th><th>list_euroSciVocTitle</th><th>list_euroSciVocPath</th><th>list_description</th><th>netEcContribution</th><th>startYear</th><th>endYear</th><th>durationDays</th><th>ecRatio</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td><td>date</td><td>date</td><td>f64</td><td>f64</td><td>str</td><td>str</td><td>date</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>datetime[μs]</td><td>i64</td><td>str</td><td>str</td><td>list[str]</td><td>list[str]</td><td>list[datetime[μs]]</td><td>list[str]</td><td>list[str]</td><td>list[str]</td><td>list[str]</td><td>list[str]</td><td>list[str]</td><td>list[datetime[μs]]</td><td>list[str]</td><td>list[str]</td><td>list[datetime[μs]]</td><td>list[str]</td><td>list[str]</td><td>list[str]</td><td>list[str]</td><td>list[str]</td><td>list[str]</td><td>list[str]</td><td>list[str]</td><td>list[str]</td><td>list[str]</td><td>list[f64]</td><td>list[str]</td><td>list[str]</td><td>list[datetime[μs]]</td><td>list[str]</td><td>list[str]</td><td>list[str]</td><td>list[str]</td><td>list[str]</td><td>list[str]</td><td>list[str]</td><td>list[str]</td><td>list[str]</td><td>list[str]</td><td>list[str]</td><td>list[str]</td><td>list[str]</td><td>list[str]</td><td>list[str]</td><td>f64</td><td>i32</td><td>i32</td><td>i64</td><td>f64</td></tr></thead><tbody><tr><td>&quot;624794&quot;</td><td>&quot;COMPACTABILITY&quot;</td><td>&quot;CLOSED&quot;</td><td>&quot;Contribution of Compact Neighb…</td><td>2014-12-01</td><td>2016-11-30</td><td>309235.2</td><td>309235.2</td><td>&quot;FP7-PEOPLE&quot;</td><td>&quot;FP7-PEOPLE-2013-IEF&quot;</td><td>null</td><td>&quot;FP7&quot;</td><td>null</td><td>&quot;FP7-PEOPLE-2013-IEF&quot;</td><td>&quot;MC-IEF&quot;</td><td>null</td><td>&quot;This research investigates how…</td><td>2017-04-10 11:25:29</td><td>187874</td><td>null</td><td>&quot;H2013&quot;</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>[&quot;Final Report Summary - COMPACTABILITY (Contribution of Compact Neighbourhoods to Social Sustainability)&quot;]</td><td>[&quot;/docs/results/624/624794/final1-table-1.jpg&quot;]</td><td>[2017-03-07 17:25:15]</td><td>[&quot;999446873&quot;]</td><td>[&quot;UK&quot;]</td><td>[&quot;OXFORD BROOKES UNIVERSITY&quot;]</td><td>[null]</td><td>[&quot;Oxford&quot;]</td><td>[&quot;51.7520131,-1.2578498&quot;]</td><td>[&quot;http://www.brookes.ac.uk&quot;]</td><td>[&quot;coordinator&quot;]</td><td>[&quot;309235.2&quot;]</td><td>[null]</td><td>[null]</td><td>[&quot;false&quot;]</td><td>[&quot;HES&quot;]</td><td>[2017-04-10 11:25:29]</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>[&quot;FP7-PEOPLE&quot;]</td><td>[&quot;Specific programme &quot;People&quot; implementing the Seventh Framework Programme of the European Community for research, technological development and demonstration activities (2007 to 2013)&quot;]</td><td>[null]</td><td>[&quot;FP7-PEOPLE-2013-IEF&quot;]</td><td>[&quot;Marie-Curie Action: Intra-European fellowships for career development&quot;]</td><td>null</td><td>null</td><td>null</td><td>0.0</td><td>2014</td><td>2016</td><td>730</td><td>0.0</td></tr><tr><td>&quot;276810&quot;</td><td>&quot;ARCHOSL&quot;</td><td>&quot;CLOSED&quot;</td><td>&quot;Archives of Early Human Occupa…</td><td>2011-03-01</td><td>2014-02-28</td><td>75000.0</td><td>75000.0</td><td>&quot;FP7-PEOPLE&quot;</td><td>&quot;FP7-PEOPLE-2009-RG&quot;</td><td>null</td><td>&quot;FP7&quot;</td><td>null</td><td>&quot;FP7-PEOPLE-2010-RG&quot;</td><td>&quot;MC-IRG&quot;</td><td>null</td><td>&quot;A number of important archaeol…</td><td>2019-08-02 13:24:51</td><td>98178</td><td>null</td><td>&quot;H2013&quot;</td><td>null</td><td>null</td><td>null</td><td>[&quot;Arnold, L.J., Demuro, M., Parés, J.M., Arsuaga, J.L., Aranburu, A.,&quot;, &quot;Lee J. Arnold , Martina Demuro , Marta Navazo , Alfonso Benito-Calvo , Alfredo Pérez-González&quot;, … &quot;F. Gutiérrez , B. Valero-Garcés , G. Desir , P. González-Sampériz , M. Gutiérrez , R. Linares , M. Zarroca , A. Moreno , J. Guerrero , C. Roqué&quot;]</td><td>[&quot;Luminescence dating and palaeomagnetic age constraint on hominins from Sima de los Huesos, Atapuerca, Spain&quot;, &quot;OSL dating of the Middle Palaeolithic Hotel California site, Sierra de Atapuerca, north-central Spain&quot;, … &quot;Late Holocene evolution of playa lakes in the central Ebro depression based on geophysical surveys and morpho-stratigraphic analysis of lacustrine terraces&quot;]</td><td>[&quot;http://dx.doi.org/10.1016/j.jhevol.2013.12.001&quot;, &quot;10.1111/j.1502-3885.2012.00262.x&quot;, … &quot;http://dx.doi.org/10.1016/j.geomorph.2012.02.013&quot;]</td><td>[&quot;Journal of Human Evolution&quot;, &quot;Boreas&quot;, … &quot;Geomorphology&quot;]</td><td>[&quot;PEER REVIEWED ARTICLE&quot;, &quot;PEER REVIEWED ARTICLE&quot;, … &quot;PEER REVIEWED ARTICLE&quot;]</td><td>[null, null, … null]</td><td>[null, null, … null]</td><td>[&quot;Final Report Summary - ARCHOSL (Archives of Early Human Occupation in Western Europe: OSL Chronologies beyond the Middle Pleistocene in the Iberian Peninsula)&quot;]</td><td>[null]</td><td>[2014-11-07 13:26:06]</td><td>[&quot;986579241&quot;]</td><td>[&quot;ES&quot;]</td><td>[&quot;CENTRO NACIONAL DE INVESTIGACION SOBRE LA EVOLUCION HUMANA&quot;]</td><td>[null]</td><td>[&quot;Burgos&quot;]</td><td>[&quot;42.3396185,-3.6967044&quot;]</td><td>[&quot;http://www.cenieh.es&quot;]</td><td>[&quot;coordinator&quot;]</td><td>[&quot;75000&quot;]</td><td>[null]</td><td>[null]</td><td>[&quot;false&quot;]</td><td>[&quot;REC&quot;]</td><td>[2019-08-02 13:24:51]</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>[&quot;FP7-PEOPLE&quot;]</td><td>[&quot;Specific programme &quot;People&quot; implementing the Seventh Framework Programme of the European Community for research, technological development and demonstration activities (2007 to 2013)&quot;]</td><td>[null]</td><td>[&quot;FP7-PEOPLE-2009-RG&quot;]</td><td>[&quot;Marie Curie Action: Reintegration Grants&quot;]</td><td>[&quot;ethnoarchaeology&quot;, &quot;physical anthropology&quot;]</td><td>[&quot;/humanities/history and archaeology/archaeology/ethnoarchaeology&quot;, &quot;/social sciences/sociology/anthropology/physical anthropology&quot;]</td><td>[null, null]</td><td>0.0</td><td>2011</td><td>2014</td><td>1095</td><td>0.0</td></tr><tr><td>&quot;622478&quot;</td><td>&quot;DETforDRF 2.0&quot;</td><td>&quot;CLOSED&quot;</td><td>&quot;Design and Expansion Turbine f…</td><td>null</td><td>null</td><td>161968.8</td><td>161968.8</td><td>&quot;FP7-PEOPLE&quot;</td><td>&quot;FP7-PEOPLE-2013-IEF&quot;</td><td>null</td><td>&quot;FP7&quot;</td><td>null</td><td>&quot;FP7-PEOPLE-2013-IEF&quot;</td><td>&quot;MC-IEF&quot;</td><td>null</td><td>&quot;This proposal for a Marie Curi…</td><td>2016-03-31 21:10:31</td><td>187686</td><td>null</td><td>&quot;H2013&quot;</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>[&quot;953573536&quot;]</td><td>[&quot;DE&quot;]</td><td>[&quot;BSH HAUSGERATE GMBH&quot;]</td><td>[null]</td><td>[&quot;Munchen&quot;]</td><td>[&quot;48.0887063,11.6433468&quot;]</td><td>[&quot;http://www.bsh-group.com&quot;]</td><td>[&quot;coordinator&quot;]</td><td>[&quot;161968.8&quot;]</td><td>[null]</td><td>[null]</td><td>[&quot;false&quot;]</td><td>[&quot;PRC&quot;]</td><td>[2016-03-31 21:10:31]</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>[&quot;FP7-PEOPLE&quot;]</td><td>[&quot;Specific programme &quot;People&quot; implementing the Seventh Framework Programme of the European Community for research, technological development and demonstration activities (2007 to 2013)&quot;]</td><td>[null]</td><td>[&quot;FP7-PEOPLE-2013-IEF&quot;]</td><td>[&quot;Marie-Curie Action: Intra-European fellowships for career development&quot;]</td><td>[&quot;fluid dynamics&quot;]</td><td>[&quot;/natural sciences/physical sciences/classical mechanics/fluid mechanics/fluid dynamics&quot;]</td><td>[null]</td><td>0.0</td><td>null</td><td>null</td><td>null</td><td>0.0</td></tr><tr><td>&quot;615785&quot;</td><td>&quot;EMERGING SUBJECTS&quot;</td><td>&quot;CLOSED&quot;</td><td>&quot;Emerging Subjects of the New E…</td><td>2014-09-01</td><td>2019-06-30</td><td>1.658373e6</td><td>1.658373e6</td><td>&quot;FP7-IDEAS-ERC&quot;</td><td>&quot;ERC-CG-2013-SH2&quot;</td><td>null</td><td>&quot;FP7&quot;</td><td>null</td><td>&quot;ERC-2013-CoG&quot;</td><td>&quot;ERC-CG&quot;</td><td>null</td><td>&quot;This project examines how pred…</td><td>2023-04-05 11:40:06</td><td>188675</td><td>null</td><td>&quot;H2013&quot;</td><td>null</td><td>null</td><td>null</td><td>[&quot;Rebekah Plueckhahn&quot;, &quot;Dulam, Bumochir&quot;, … &quot;•Empson, R. A.&quot;]</td><td>[&quot;Tragic Spirits: Shamanism, Memory, and Gender in Contemporary Mongolia by Manduhai Buyandelger.&quot;, &quot;The Afterlife of Nomadism: Pastoralism, environmentalism, civilization and identity in Mongolia and China&quot;, … &quot;A Space That Will Never Be Filled Sharp Communication and the Simultaneity of Opposites.&quot;]</td><td>[&quot;10.1111/aman.12304&quot;, null, … null]</td><td>[&quot;American Anthropologist&quot;, &quot;Pastoralist Livelihoods in Asian Drylands: Environment, Governance and Risk&quot;, … &quot;Current Anthropology&quot;]</td><td>[&quot;PEER_REVIEWED_ARTICLE&quot;, &quot;ARTICLE&quot;, … &quot;ARTICLE&quot;]</td><td>[null, null, … null]</td><td>[null, null, … null]</td><td>[&quot;Final Report Summary - EMERGING SUBJECTS (Emerging Subjects of the New Economy: Tracing Economic Growth in Mongolia)&quot;]</td><td>[null]</td><td>[2018-01-15 17:25:25]</td><td>[&quot;888898146&quot;]</td><td>[null]</td><td>[&quot;UNIVERSITY COLLEGE LONDON&quot;]</td><td>[null]</td><td>[&quot;LONDON&quot;]</td><td>[&quot;51.5236746,-0.1339608&quot;]</td><td>[&quot;http://www.ucl.ac.uk&quot;]</td><td>[&quot;coordinator&quot;]</td><td>[&quot;1658373&quot;]</td><td>[null]</td><td>[null]</td><td>[&quot;false&quot;]</td><td>[&quot;HES&quot;]</td><td>[2023-04-05 11:40:06]</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>[&quot;FP7-IDEAS-ERC&quot;]</td><td>[&quot;Specific programme: &quot;Ideas&quot; implementing the Seventh Framework Programme of the European Community for research, technological development and demonstration activities (2007 to 2013)&quot;]</td><td>[null]</td><td>[&quot;ERC-CG-2013-SH2&quot;]</td><td>[&quot;ERC Consolidator Grant - Institutions Values Beliefs and behaviour&quot;]</td><td>[&quot;anthropology&quot;]</td><td>[&quot;/social sciences/sociology/anthropology&quot;]</td><td>[null]</td><td>0.0</td><td>2014</td><td>2019</td><td>1763</td><td>0.0</td></tr><tr><td>&quot;237010&quot;</td><td>&quot;DEER PALAEOBIOLOGY&quot;</td><td>&quot;CLOSED&quot;</td><td>&quot;Palaeobiological inference thr…</td><td>2009-04-09</td><td>2011-01-08</td><td>173416.47</td><td>173416.47</td><td>&quot;FP7-PEOPLE&quot;</td><td>&quot;FP7-PEOPLE-IEF-2008&quot;</td><td>null</td><td>&quot;FP7&quot;</td><td>null</td><td>&quot;FP7-PEOPLE-IEF-2008&quot;</td><td>&quot;MC-IEF&quot;</td><td>null</td><td>&quot;The present research aims to r…</td><td>2019-07-16 19:18:25</td><td>90424</td><td>null</td><td>&quot;H2013&quot;</td><td>null</td><td>null</td><td>null</td><td>[&quot;Lister, A.M., Breda, M. and others&quot;, &quot;Breda, M., Lister, A.M. &amp; others&quot;]</td><td>[&quot;Metric analysis of ungulate mammals in the early Middle Pleistocene of Britain, in relation to taxonomy and biostratigraphy. II. Cervidae, Equidae and Suidae.&quot;, &quot;Metric analysis of ungulate mammals in the early Middle Pleistocene of Britain, in relation to taxonomy and biostratigraphy. I: Rhinocerotidae and Bovidae.&quot;]</td><td>[null, null]</td><td>[&quot;Quaternary International&quot;, &quot;Quaternary International&quot;]</td><td>[&quot;PEER REVIEWED ARTICLE&quot;, &quot;PEER REVIEWED ARTICLE&quot;]</td><td>[null, null]</td><td>[null, null]</td><td>[&quot;Final Report Summary - DEER PALAEOBIOLOGY (Palaeobiological inference through phylogenetic analysis of Pleistocene deer)&quot;]</td><td>[null]</td><td>[2013-07-05 00:02:53]</td><td>[&quot;999642037&quot;]</td><td>[&quot;UK&quot;]</td><td>[&quot;NATURAL HISTORY MUSEUM&quot;]</td><td>[null]</td><td>[&quot;London&quot;]</td><td>[&quot;51.494882,-0.1847716&quot;]</td><td>[&quot;http://www.nhm.ac.uk/&quot;]</td><td>[&quot;coordinator&quot;]</td><td>[&quot;173416.47&quot;]</td><td>[null]</td><td>[null]</td><td>[&quot;false&quot;]</td><td>[&quot;PUB&quot;]</td><td>[2019-07-16 19:18:25]</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>[&quot;FP7-PEOPLE&quot;]</td><td>[&quot;Specific programme &quot;People&quot; implementing the Seventh Framework Programme of the European Community for research, technological development and demonstration activities (2007 to 2013)&quot;]</td><td>[null]</td><td>[&quot;FP7-PEOPLE-IEF-2008&quot;]</td><td>[&quot;Marie Curie Action: Intra-European Fellowships for Career Development&quot;]</td><td>[&quot;comparative morphology&quot;]</td><td>[&quot;/natural sciences/biological sciences/biological morphology/comparative morphology&quot;]</td><td>[null]</td><td>0.0</td><td>2009</td><td>2011</td><td>639</td><td>0.0</td></tr></tbody></table></div>"
+      ],
+      "text/plain": [
+       "shape: (5, 68)\n",
+       "┌────────┬──────────────┬────────┬──────────────┬───┬───────────┬─────────┬──────────────┬─────────┐\n",
+       "│ id     ┆ acronym      ┆ status ┆ title        ┆ … ┆ startYear ┆ endYear ┆ durationDays ┆ ecRatio │\n",
+       "│ ---    ┆ ---          ┆ ---    ┆ ---          ┆   ┆ ---       ┆ ---     ┆ ---          ┆ ---     │\n",
+       "│ str    ┆ str          ┆ str    ┆ str          ┆   ┆ i32       ┆ i32     ┆ i64          ┆ f64     │\n",
+       "╞════════╪══════════════╪════════╪══════════════╪═══╪═══════════╪═════════╪══════════════╪═════════╡\n",
+       "│ 624794 ┆ COMPACTABILI ┆ CLOSED ┆ Contribution ┆ … ┆ 2014      ┆ 2016    ┆ 730          ┆ 0.0     │\n",
+       "│        ┆ TY           ┆        ┆ of Compact   ┆   ┆           ┆         ┆              ┆         │\n",
+       "│        ┆              ┆        ┆ Neighb…      ┆   ┆           ┆         ┆              ┆         │\n",
+       "│ 276810 ┆ ARCHOSL      ┆ CLOSED ┆ Archives of  ┆ … ┆ 2011      ┆ 2014    ┆ 1095         ┆ 0.0     │\n",
+       "│        ┆              ┆        ┆ Early Human  ┆   ┆           ┆         ┆              ┆         │\n",
+       "│        ┆              ┆        ┆ Occupa…      ┆   ┆           ┆         ┆              ┆         │\n",
+       "│ 622478 ┆ DETforDRF    ┆ CLOSED ┆ Design and   ┆ … ┆ null      ┆ null    ┆ null         ┆ 0.0     │\n",
+       "│        ┆ 2.0          ┆        ┆ Expansion    ┆   ┆           ┆         ┆              ┆         │\n",
+       "│        ┆              ┆        ┆ Turbine f…   ┆   ┆           ┆         ┆              ┆         │\n",
+       "│ 615785 ┆ EMERGING     ┆ CLOSED ┆ Emerging     ┆ … ┆ 2014      ┆ 2019    ┆ 1763         ┆ 0.0     │\n",
+       "│        ┆ SUBJECTS     ┆        ┆ Subjects of  ┆   ┆           ┆         ┆              ┆         │\n",
+       "│        ┆              ┆        ┆ the New E…   ┆   ┆           ┆         ┆              ┆         │\n",
+       "│ 237010 ┆ DEER PALAEOB ┆ CLOSED ┆ Palaeobiolog ┆ … ┆ 2009      ┆ 2011    ┆ 639          ┆ 0.0     │\n",
+       "│        ┆ IOLOGY       ┆        ┆ ical         ┆   ┆           ┆         ┆              ┆         │\n",
+       "│        ┆              ┆        ┆ inference    ┆   ┆           ┆         ┆              ┆         │\n",
+       "│        ┆              ┆        ┆ thr…         ┆   ┆           ┆         ┆              ┆         │\n",
+       "└────────┴──────────────┴────────┴──────────────┴───┴───────────┴─────────┴──────────────┴─────────┘"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "consolidated_clean.head()\n",
+    "#ecMaxContribution, endDate, status, legalBasis, frameworkProgramme, fundingScheme, list_title_report, list_name, list_role, list_city, list_country, list_ecContribution, list_activityType, durationDays"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

backend/main.py

@@ -9,10 +9,12 @@ from pydantic import BaseModel
 #except:
 #    from .rag import get_rag_chain, RAGRequest, RAGResponse
 from contextlib import asynccontextmanager
-
+import os
 import polars as pl
 import gcsfs
 
+os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = r"C:\Users\Romain\OneDrive - KU Leuven\focal-pager-460414-e9-45369b738be0.json"
+
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     bucket = "mda_eu_project"
@@ -121,29 +123,30 @@ def get_stats(request: Request):
 @app.get("/api/project/{project_id}/organizations")
 def get_project_organizations(project_id: str):
     df = app.state.df
+
     sel = df.filter(pl.col("id") == project_id)
     if sel.is_empty():
         raise HTTPException(status_code=404, detail="Project not found")
 
     orgs_df = (
-        sel.select([
-            pl.explode("list_name").alias("name"),
-            pl.explode("list_country").alias("country"),
-            pl.explode("list_geolocation").alias("geoloc"),
+        sel
+        .select([
+            pl.col("list_name").explode().alias("name"),
+            pl.col("list_country").explode().alias("country"),
+            pl.col("list_geolocation").explode().alias("geoloc"),
         ])
         .with_columns([
-            pl.col("geoloc")
-              .str.split_exact(",", 1)
-              .alias("latlon")
+            # now this is a List(Utf8)
+            pl.col("geoloc").str.split(",").alias("latlon"),
         ])
         .with_columns([
-            pl.col("latlon").list.get(0).cast(float).alias("latitude"),
-            pl.col("latlon").list.get(1).cast(float).alias("longitude")
+            pl.col("latlon").list.get(0).cast(pl.Float64).alias("latitude"),
+            pl.col("latlon").list.get(1).cast(pl.Float64).alias("longitude"),
         ])
         .filter(pl.col("name").is_not_null())
-        .select(["name","country","latitude","longitude"])
+        .select(["name", "country", "latitude", "longitude"])
     )
-    print(orgs_df)
+
     return orgs_df.to_dicts()
 
 """def rag_chain_depender():

data_enhancement.py

@@ -0,0 +1,442 @@
+import csv
+import re
+import polars as pl
+from __future__ import annotations
+import re, csv, pathlib, polars as pl
+
+ROOT      = pathlib.Path(r"C:\Users\Romain\OneDrive - KU Leuven\Masters\MBIS\Year 2\Semester 2\Modern Data Analytics\CORDIS")
+DATASETS = [
+    "project",
+    "projectDeliverables",
+    "projectPublications",
+    "reportSummaries",
+    "organization",
+    "euroSciVoc",
+    "topics",
+    "webItem",
+    "webLink",
+    "legalBasis",
+]
+OUTDIR    = ROOT / "combined"
+OUTDIR.mkdir(exist_ok=True)
+
+###############################################################################
+# 2.  Generic cleaner –– parameterised version of the loop you wrote
+###############################################################################
+_PROJECT_ID_RE = re.compile(r"^(?:19|20)\d{2}")
+_GENERIC_NUM_RE  = re.compile(r"\d{4}")
+
+import csv, pathlib, polars as pl, re
+
+import csv, re, pathlib
+import polars as pl                       #  >=0.20
+
+import csv, pathlib, re
+import polars as pl                       # ≥ 0.20
+
+
+def _clean_one_file(csv_path: pathlib.Path,
+                    number_regex: re.Pattern[str], dataset: str) -> pl.DataFrame:
+    """
+    Clean a CORDIS CSV whose long *objective* field sometimes explodes into
+    extra columns because of stray quotes / semicolons.
+
+    Strategy
+    --------
+    * A well-formed row has 21 semicolon-separated columns.
+    * If we get more than 21 columns we treat columns 16 … -4 as belonging
+      to *objective* and stitch them back together with a semicolon.
+    * The last three columns are   contentUpdateDate | rcn | grantDoi.
+    """
+    # ---------- constants --------------------------------------------------
+    if dataset=="project":
+        EXPECTED_COLS   = 20          # final width
+        TITLE_COL       = 3           # 0-based index of *title*
+        DATE1_COL       = 4           # 0-based index of startDate
+        DATE2_COL       = 5           # 0-based index of endDate
+        OBJECTIVE_COL   = 16          # 0-based index of objective
+        TRAILING_KEEP   = 3           # last three fixed columns
+    elif dataset=="organization":
+        EXPECTED_COLS   = 25          # final width
+        TITLE_COL       = 3           # 0-based index of *title*
+        DATE1_COL       = 4           # 0-based index of startDate
+        DATE2_COL       = 5           # 0-based index of endDate
+        OBJECTIVE_COL   = 4           # 0-based index of objective
+        TRAILING_KEEP   = 20           # last three fixed columns
+    else:
+        EXPECTED_COLS   = 20          # final width
+        TITLE_COL       = 3           # 0-based index of *title*
+        DATE1_COL       = 4           # 0-based index of startDate
+        DATE2_COL       = 5           # 0-based index of endDate
+        OBJECTIVE_COL   = 16          # 0-based index of objective
+        TRAILING_KEEP   = 3           # last three fixed columns
+
+
+
+    date_rx   = re.compile(r"\d{4}-\d{2}-\d{2}$")
+    is_date   = lambda s: (s == "") or bool(date_rx.match(s))
+
+    tmp_clean = csv_path.with_suffix(".cleaned.csv")
+
+    with csv_path.open(encoding="utf-8", newline="") as fin, \
+         tmp_clean.open("w", encoding="utf-8", newline="") as fout:
+
+        writer = csv.writer(
+            fout,
+            delimiter="|",
+            quotechar='"',
+            quoting=csv.QUOTE_MINIMAL,
+            lineterminator="\n",
+        )
+
+        # ---------- iterate raw lines -------------------------------------
+        for raw in fin:
+            #print(raw)
+            raw = raw.rstrip("\n")
+            #print(raw)
+            cells = raw.split(";")                     # blind split
+
+            # ---- 1️⃣  repair *title* if dates are not where they belong --
+            if (len(cells) > EXPECTED_COLS) and  (not is_date(cells[DATE1_COL]) or not is_date(cells[DATE2_COL])) and dataset=="project":
+                # look for the first position where *two successive* cells
+                # are both valid dates / nulls
+                i = DATE1_COL
+                while i + 1 < len(cells):
+                    if is_date(cells[i]) and is_date(cells[i + 1]):
+                        break
+                    i += 1
+                else:
+                    # cannot find a valid date pair → give up on this line
+                    continue
+
+                head   = cells[:TITLE_COL]             # 0 … 2
+                title  = ";".join(cells[TITLE_COL:i])  # glue spill-over
+                cells  = head + [title] + cells[i:]    # rebuild the row
+            # ---- 2️⃣  repair *objective* overflow ------------------------
+            if len(cells) > EXPECTED_COLS and (dataset=="project" or  dataset=="organization"):
+                head = cells[:OBJECTIVE_COL]
+                tail = cells[-TRAILING_KEEP:]
+                obj  = ";".join(cells[OBJECTIVE_COL:-TRAILING_KEEP])
+                cells = head + [obj] + tail
+                #print("here 2")
+
+            # ---- 3️⃣  pad short rows, skip malformed ---------------------
+            if len(cells) < EXPECTED_COLS and (dataset=="project" or  dataset=="organization"):
+                cells.extend([""] * (EXPECTED_COLS - len(cells)))
+                #print("here again")
+
+            if len(cells) != EXPECTED_COLS and (dataset=="project" or  dataset=="organization"):            # still wrong → skip
+                #print(cells)
+                continue
+
+            # ---- 4️⃣  cell-level clean-ups -------------------------------
+            cleaned: list[str] = []
+            for cell in cells:
+
+                if cell in ('""', ""):
+                    cell = ""
+                else:
+                    cell = (cell.replace("\t", " ")
+                                 .replace('"""', '"')
+                                 .strip())
+                    if number_regex.fullmatch(cell):
+                        cell = cell.lstrip("0") or "0"
+                cleaned.append(cell.strip('"'))
+            cleaned[-1]=cleaned[-1].replace('"','').replace(',','')
+            cleaned[0]=cleaned[0].replace('"','')
+            writer.writerow(cleaned)
+
+    # ---------- read into Polars (all Utf8) -------------------------------
+    return pl.read_csv(
+        tmp_clean,
+        separator="|",
+        quote_char='"',
+        has_header=True,
+        infer_schema_length=0,
+        null_values=[""],
+        truncate_ragged_lines=True,
+    )
+
+
+def combine_all_programmes() -> None:
+    from pathlib import Path
+    for dataset in DATASETS:
+        combined: list[pl.DataFrame] = []
+
+        for i,programme_dir in enumerate(ROOT.iterdir()):
+            if not programme_dir.is_dir():
+                continue
+            csv_file = programme_dir / f"{dataset}.csv"
+            if not csv_file.exists():
+                continue
+
+            regex = _PROJECT_ID_RE if dataset == "project" else _GENERIC_NUM_RE
+            df    = _clean_one_file(csv_file, regex, dataset)
+            print(programme_dir)
+            # ---------- type coercions matching your original code ----------
+            if dataset == "project":
+                df = (
+                    df
+                    .with_columns([
+                        pl.col("id"),#.cast(pl.Int64),
+                        pl.col("acronym").cast(pl.Utf8, strict=False).str.strip_chars('"'),
+                        pl.col("status").cast(pl.Utf8, strict=False).str.strip_chars('"'),
+                        pl.col("title").cast(pl.Utf8, strict=False).str.strip_chars('"'),
+                        pl.col("legalBasis").cast(pl.Utf8, strict=False).str.strip_chars('"'),
+                        pl.col("topics").cast(pl.Utf8, strict=False).str.strip_chars('"'),
+                        pl.col("frameworkProgramme").cast(pl.Utf8, strict=False).str.strip_chars('"'),
+                        pl.col("masterCall").cast(pl.Utf8, strict=False).str.strip_chars('"'),
+                        pl.col("subCall").cast(pl.Utf8, strict=False).str.strip_chars('"'),
+                        pl.col("fundingScheme").cast(pl.Utf8, strict=False).str.strip_chars('"'),
+                        pl.col("nature").cast(pl.Utf8, strict=False).str.strip_chars('"'),
+                        pl.col("objective").cast(pl.Utf8, strict=False).str.strip_chars('"'),
+                        pl.col("grantDoi").cast(pl.Utf8, strict=False).str.strip_chars('"'),
+                        pl.col("totalCost").cast(pl.Utf8, strict=False).str.strip_chars('"').str.replace_all('"','').str.replace(",",".").cast(pl.Float64),
+                        pl.col("ecMaxContribution").cast(pl.Utf8, strict=False).str.strip_chars('"').str.replace_all('"','').str.replace(",",".").cast(pl.Float64),
+                        pl.col("startDate").cast(pl.Utf8, strict=False).str.strip_chars('"').str.strptime(pl.Date, "%Y-%m-%d", strict=False),
+                        pl.col("endDate").cast(pl.Utf8, strict=False).str.strip_chars('"').str.strptime(pl.Date, "%Y-%m-%d", strict=False),
+                        pl.col("ecSignatureDate").cast(pl.Utf8, strict=False).str.strip_chars('"').str.strptime(pl.Date, "%Y-%m-%d", strict=False),
+                        pl.col("contentUpdateDate").cast(pl.Utf8, strict=False).str.strip_chars('"').str.strptime(pl.Datetime, "%Y-%m-%d %H:%M:%S", strict=False),
+                        pl.col("rcn").cast(pl.Int64),
+                    ])
+                    .with_columns(
+                        pl.lit(programme_dir.name).alias("programmeFolder")   # <-- NEW COLUMN
+                    )
+                )
+            elif dataset == "organization":
+                df = df.with_columns([
+                    pl.col("contentUpdateDate").cast(pl.Utf8, strict=False).str.strptime(pl.Datetime, "%Y-%m-%d %H:%M:%S", strict=False),
+                    pl.col("totalCost").cast(pl.Utf8, strict=False).str.replace(",",".").cast(pl.Float64),
+                ])
+            elif dataset == "projectDeliverables":
+                df = df.with_columns([
+                    #pl.col("projectID").cast(pl.Int64),
+                    pl.col("contentUpdateDate").cast(pl.Utf8, strict=False)
+                    .str.strptime(pl.Datetime, "%Y-%m-%d %H:%M:%S", strict=False),
+                ])
+            elif dataset == "projectPublications":
+                if programme_dir==Path(r"C:\Users\Romain\OneDrive - KU Leuven\Masters\MBIS\Year 2\Semester 2\Modern Data Analytics\CORDIS\H2013"):
+                    rename_map = {
+                        "RECORD_ID":      "id",
+                        "TITLE":          "title",
+                        "AUTHOR":         "authors",
+                        "DOI":            "doi",
+                        "PROJECT_ID":     "projectID",
+                        "JOURNAL_TITLE":  "journalTitle",
+                        "PAGES":          "publishedPages",
+                        "PUBLICATION_TYPE": "isPublishedAs",
+                    }
+
+                    df = df.rename(rename_map)
+                else:
+                    df = df.with_columns([
+                        pl.col("contentUpdateDate").cast(pl.Utf8, strict=False)
+                        .str.strptime(pl.Datetime, "%Y-%m-%d %H:%M:%S", strict=False),
+                        pl.col("id").cast(pl.Utf8, strict=False)
+                        .str.extract(r"^(\d+)_", 1)
+                        #.cast(pl.Int64)
+                        .alias("projectID"),
+                    ])
+            elif dataset == "reportSummaries":
+                df = df.with_columns(
+                    pl.col("contentUpdateDate").cast(pl.Utf8, strict=False)
+                    .str.strptime(pl.Datetime, "%Y-%m-%d %H:%M:%S", strict=False),
+                )
+            elif dataset == "organization":
+                df = df.with_columns([
+                    pl.col("contentUpdateDate").cast(pl.Utf8, strict=False)
+                    .str.strptime(pl.Datetime, "%Y-%m-%d %H:%M:%S", strict=False),
+                    pl.col("totalCost").cast(pl.Utf8, strict=False)
+                    .str.replace(",", ".")
+                    .cast(pl.Float64),
+                ])
+            elif dataset == "webItem":
+                df = df.with_columns(
+                    pl.col("uri").cast(pl.Utf8, strict=False)
+                    .str.extract(r"/files/\d+/(\d+)/", 1)
+                    .cast(pl.Int64)
+                    .alias("projectID"),
+                )
+
+            # ---------------------------------------------------------------
+            combined.append(df)
+
+        # --------------------------------------------------------------------
+        # Write out per-dataset parquet
+        # --------------------------------------------------------------------
+        if combined:
+            how="vertical_relaxed"
+            if dataset=="projectPublications":
+                how="diagonal"
+            result = pl.concat(combined, how=how)
+            parquet_path = OUTDIR / f"{dataset}_all.parquet"
+            result.write_parquet(parquet_path)
+            print(f"✔  {dataset:15s} → {parquet_path}")
+
+import pathlib
+import polars as pl
+
+ROOT    = pathlib.Path(r"C:\Users\Romain\OneDrive - KU Leuven\Masters\MBIS\Year 2\Semester 2\Modern Data Analytics\CORDIS")
+OUTDIR  = ROOT / "combined"
+DATASETS = [
+    "project",
+    "projectDeliverables",
+    "projectPublications",
+    "reportSummaries",
+    "organization",
+    "euroSciVoc",
+    "topics",
+    "webItem",
+    "webLink",
+    "legalBasis",
+]
+
+dfs = {}
+for dataset in DATASETS:
+    path = OUTDIR / f"{dataset}_all.parquet"
+    dfs[dataset] = pl.read_parquet(path)
+
+projects         = dfs["project"]
+
+projects_deliv   = (
+    dfs["projectDeliverables"]
+    .group_by("projectID")
+    .agg([
+        pl.col("deliverableType").alias("list_deliverableType"),
+        pl.col("url")            .alias("list_url"),
+        pl.col("contentUpdateDate").alias("list_contentUpdateDate"),
+    ])
+)
+
+projects_publi   = (
+    dfs["projectPublications"]
+    .group_by("projectID")
+    .agg([
+        pl.col("authors")         .alias("list_authors"),
+        pl.col("title")           .alias("list_title"),
+        pl.col("doi")             .alias("list_doi"),
+        pl.col("journalTitle")    .alias("list_journalTitle"),
+        pl.col("isPublishedAs")   .alias("list_isPublishedAs"),
+        pl.col("publishedYear")   .alias("list_publishedYear"),
+        pl.col("contentUpdateDate").alias("list_contentUpdateDate"),
+    ])
+)
+
+report = (
+    dfs["reportSummaries"]
+    .group_by("projectID")
+    .agg([
+        pl.col("title")           .alias("list_title"),
+        pl.col("attachment")      .alias("list_attachment"),
+        pl.col("contentUpdateDate").alias("list_contentUpdateDate"),
+    ])
+)
+
+org = (
+    dfs["organization"]
+    .group_by("projectID")
+    .agg([
+        pl.col("organisationID")  .alias("list_organisationID"),
+        pl.col("country")         .alias("list_country"),
+        pl.col("name")            .alias("list_name"),
+        pl.col("SME")             .alias("list_SME"),
+        pl.col("city")            .alias("list_city"),
+        pl.col("geolocation")     .alias("list_geolocation"),
+        pl.col("organizationURL") .alias("list_organizationURL"),
+        pl.col("role")            .alias("list_role"),
+        pl.col("ecContribution")  .alias("list_ecContribution"),
+        pl.col("netEcContribution").alias("list_netEcContribution"),
+        pl.col("totalCost")       .alias("list_totalCost"),
+        pl.col("endOfParticipation").alias("list_endOfParticipation"),
+        pl.col("activityType")    .alias("list_activityType"),
+        pl.col("contentUpdateDate").alias("list_contentUpdateDate"),
+    ])
+)
+
+voc = (
+    dfs["euroSciVoc"]
+    .group_by("projectID")
+    .agg([
+        pl.col("euroSciVocTitle")      .alias("list_euroSciVocTitle"),
+        pl.col("euroSciVocPath")       .alias("list_euroSciVocPath"),
+        pl.col("euroSciVocDescription").alias("list_description"),
+    ])
+)
+
+topic = (
+    dfs["topics"]
+    .group_by("projectID")
+    .agg([
+        pl.col("topic")   .alias("list_topic"),
+        pl.col("title")   .alias("list_title"),
+    ])
+)
+
+web_item = dfs["webItem"]  # no aggregation
+
+web_link = (
+    dfs["webLink"]
+    .group_by("projectID")
+    .agg([
+        pl.col("physUrl")            .alias("list_physUrl"),
+        pl.col("availableLanguages") .alias("list_availableLanguages"),
+        pl.col("status")             .alias("list_status"),
+        pl.col("archivedDate")       .alias("list_archivedDate"),
+        pl.col("type")               .alias("list_type"),
+        pl.col("source")             .alias("list_source"),
+        pl.col("represents")         .alias("list_represents"),
+    ])
+)
+
+legal = (
+    dfs["legalBasis"]
+    .group_by("projectID")
+    .agg([
+        pl.col("legalBasis")         .alias("list_legalBasis"),
+        pl.col("title")              .alias("list_title"),
+        pl.col("uniqueProgrammePart").alias("list_uniqueProgrammePart"),
+    ])
+)
+
+consolidated = (
+    projects
+    .join(projects_deliv,   left_on="id", right_on="projectID", suffix="_deliv", how="left")
+    .join(projects_publi,   left_on="id", right_on="projectID", suffix="_publi", how="left")
+    .join(report,           left_on="id", right_on="projectID", suffix="_report", how="left")
+    .join(org,              left_on="id", right_on="projectID", suffix="_org", how="left")
+    .join(web_link,         left_on="id", right_on="projectID", suffix="_link", how="left")
+    .join(legal,            left_on="id", right_on="projectID", suffix="_legal", how="left")
+    .join(topic,            left_on="id", right_on="projectID", suffix="_topic", how="left")
+    .join(voc,              left_on="id", right_on="projectID", suffix="_voc", how="left")
+)
+
+for col in ["startDate", "endDate"]:
+    if consolidated[col].dtype == pl.Utf8:
+        consolidated = consolidated.with_column(
+            pl.col(col).str.strptime(pl.Date, "%Y-%m-%d").alias(col)
+        )
+
+consolidated = consolidated.with_columns(
+    pl.col("list_netEcContribution").list.eval(pl.element().cast(pl.Float64),parallel=True)
+    .list.sum().alias("netEcContribution")
+)
+
+consolidated = consolidated.with_columns(
+    pl.col("totalCost").cast(pl.Float64),
+    pl.col("netEcContribution").cast(pl.Float64)
+)
+
+consolidated = consolidated.with_columns([
+    pl.col("startDate").dt.year().alias("startYear"),
+    pl.col("endDate").  dt.year().alias("endYear"),
+    (pl.col("endDate") - pl.col("startDate")).dt.total_days().alias("durationDays"),
+    (pl.col("netEcContribution") / pl.col("totalCost")).alias("ecRatio"),
+])
+
+consolidated.write_parquet(OUTDIR / "consolidated.parquet")
+
+excluded_frameworks = ["FP1", "FP2", "FP3", "FP4", "FP5", "FP6"]
+
+consolidated_clean = (consolidated.filter(~pl.col("frameworkProgramme").is_in(excluded_frameworks)))
+
+consolidated_clean.write_parquet(OUTDIR / "consolidated_clean.parquet")

predictive_modelling.py

@@ -0,0 +1,360 @@
+import os
+import json
+import joblib
+import numpy as np
+import pandas as pd
+import shap
+import matplotlib.pyplot as plt
+import scipy.sparse
+
+from sklearn.base import BaseEstimator, TransformerMixin
+from sklearn.pipeline import Pipeline as SKPipeline
+from sklearn.compose import ColumnTransformer
+from sklearn.preprocessing import OneHotEncoder, StandardScaler, MultiLabelBinarizer
+from sklearn.impute import SimpleImputer
+from sklearn.model_selection import train_test_split, cross_val_score, StratifiedKFold
+from sklearn.feature_selection import SelectKBest, f_classif, VarianceThreshold
+from sklearn.metrics import classification_report, ConfusionMatrixDisplay, f1_score, make_scorer
+from sklearn.decomposition import TruncatedSVD
+from sklearn.calibration import CalibratedClassifierCV
+from sklearn.ensemble import IsolationForest
+
+from imblearn.pipeline import Pipeline as ImbPipeline
+from imblearn.over_sampling import ADASYN
+
+from sentence_transformers import SentenceTransformer
+from xgboost import XGBClassifier
+
+from evidently import Report
+from evidently.presets import DataDriftPreset
+
+import optuna
+
+# --- Custom Transformers ---
+class MultiLabelBinarizerTransformer(BaseEstimator, TransformerMixin):
+    def fit(self, X,y=None):
+        self.col = X.name
+        self.mlb = MultiLabelBinarizer()
+        self.mlb.fit(X)
+        return self
+    def transform(self, X):
+        return self.mlb.transform(X)
+    def get_feature_names_out(self, input_features=None):
+        return [f"{self.col}_{cls}" for cls in self.mlb.classes_]
+    def get_params(self, deep=True):
+        return {}
+    def set_params(self, **params):
+        return self
+
+class AnomalyScoreTransformer(BaseEstimator, TransformerMixin):
+    def __init__(self):
+        self.model = IsolationForest(n_estimators=200, contamination=0.1, random_state=42)
+
+    def fit(self, X, y=None):
+        self.model.fit(X)
+        return self
+
+    def transform(self, X):
+        scores = -self.model.decision_function(X)
+        return np.hstack([X, scores.reshape(-1, 1)])
+
+# --- Step 1: Data Preparation ---
+def prepare_data(df, is_train=True, model_dir="model_artifacts"):
+    df = df.copy()
+    
+    if is_train:
+        df['status'] = df['status'].astype(str).str.upper()
+        df = df[df['status'].isin(['CLOSED', 'TERMINATED'])]
+        df['label'] = df['status'].map({'CLOSED': 0, 'TERMINATED': 1})
+        assert df['label'].notna().all(), "Label column still has NaNs!"
+
+    multilabel_fields = [
+        'list_country', 'list_activityType', 'list_deliverableType',
+        'list_availableLanguages', 'list_euroSciVocTitle'
+    ]
+
+    def extract_intermediate_levels(paths):
+        tokens = []
+        if isinstance(paths, list):
+            for p in paths:
+                parts = p.strip('/').split('/')
+                tokens.extend(parts[:-1])
+        return list(set(tokens))
+    df['euroSciVoc_intermediate'] = df['list_euroSciVocPath'].apply(extract_intermediate_levels)
+    multilabel_fields.append('euroSciVoc_intermediate')
+    
+    for col in multilabel_fields:
+        df[col] = df[col].apply(lambda x: [] if x is None else (x.tolist() if hasattr(x, 'tolist') else x))
+        df[col] = df[col].apply(lambda x: list(x) if not isinstance(x, list) else x)
+        df[col] = df[col].apply(lambda x: [item for item in x if item is not None])
+        df[col] = df[col].apply(lambda x: [str(item).upper() for item in x])
+
+    
+    def split_languages(lang_list):
+        if not isinstance(lang_list, list):
+            return []
+        result = []
+        for entry in lang_list:
+            if isinstance(entry, str):
+                result.extend(entry.split(","))
+        return result
+
+    df["list_availableLanguages"] = df["list_availableLanguages"].apply(split_languages)
+        
+
+    for col in ['title', 'objective']:
+        df[col] = df[col].fillna("").astype(str)
+
+    df['n_partners'] = df['list_name'].apply(
+        lambda x: len(x.tolist()) if x is not None and hasattr(x, 'tolist') else (len(x) if isinstance(x, list) else 0)
+    )
+
+    df['n_country'] = df['list_country'].apply(
+        lambda x: len(x.tolist()) if x is not None and hasattr(x, 'tolist') else (len(x) if isinstance(x, list) else 0)
+    )
+
+    df['n_sme'] = df['list_SME'].apply(
+        lambda x: sum(1 for i in (x.tolist() if hasattr(x, 'tolist') else x) if i is True)
+        if x is not None and (hasattr(x, 'tolist') or isinstance(x, list)) else 0
+    )
+
+    return df
+
+# --- Step 2: Text Embedding ---
+def compute_embeddings(df, text_columns, model_name='sentence-transformers/LaBSE', svd_dim=50):
+    model = SentenceTransformer(model_name)
+    os.makedirs("/content/drive/MyDrive/model_artifacts", exist_ok=True)
+    os.makedirs("/content/drive/MyDrive/embeddings", exist_ok=True)
+    for col in text_columns:
+        embedding_file = f"/content/drive/MyDrive/embeddings/{col}_embeddings.npy"
+        svd_file = f"/content/drive/MyDrive/model_artifacts/{col}_svd.pkl"
+        if os.path.exists(embedding_file):
+            print(f"Loading saved embeddings for column '{col}'...")
+            embeddings = np.load(embedding_file)
+        else:
+            print(f"Computing embeddings for column '{col}'...")
+            embeddings = model.encode(df[col].tolist(), show_progress_bar=True)
+            np.save(embedding_file, embeddings)
+
+        print(f"Fitting SVD for column '{col}'...")
+        svd = TruncatedSVD(n_components=svd_dim, random_state=42)
+        svd.fit(embeddings)
+        joblib.dump(svd, svd_file)
+
+        reduced = svd.transform(embeddings)
+        embed_df = pd.DataFrame(reduced, columns=[f'{col}_embed_{i}' for i in range(reduced.shape[1])])
+        embed_df.index = df.index  # Force matching index
+        df = pd.concat([df, embed_df], axis=1)
+    return df
+
+
+# --- Step 3: Build Preprocessor ---
+def build_preprocessor(numeric_features, categorical_features, multilabel_fields):
+    numeric_pipeline = SKPipeline([
+        ('imputer', SimpleImputer(strategy='median')),
+        ('scaler', StandardScaler())], memory="cache_dir"
+    )
+
+    categorical_pipeline = SKPipeline([
+        ('imputer', SimpleImputer(strategy='constant', fill_value='missing')),
+        ('onehot', OneHotEncoder(handle_unknown='ignore', sparse_output=False))], memory="cache_dir"
+    )
+
+    transformers = [
+        ('num', numeric_pipeline, numeric_features),
+        ('cat', categorical_pipeline, categorical_features),
+      *[(f'mlb_{col}', MultiLabelBinarizerTransformer(), col) for col in multilabel_fields]]
+        
+
+    return ColumnTransformer(transformers, sparse_threshold=0.0)
+
+# --- Step 4: Build Pipeline ---
+def build_pipeline(preprocessor, base_model, k=250):
+    return ImbPipeline(steps=[
+        ('preprocessor', preprocessor),
+        ('anomaly', AnomalyScoreTransformer()),
+        ('resample', ADASYN()),
+        ("variance_filter", VarianceThreshold(threshold=0.0)),
+        ('feature_select', SelectKBest(score_func=f_classif, k=k)),
+        ('classifier', CalibratedClassifierCV(estimator=base_model, method='isotonic', cv=3))
+    ])
+
+# --- Step 5: Drift Monitoring ---
+def monitor_drift(reference, current, feature_names, output_html='drift_report.html'):
+    ref_df = pd.DataFrame(reference, columns=feature_names)
+    cur_df = pd.DataFrame(current, columns=feature_names)
+    
+    report = Report(metrics=[DataDriftPreset()])
+    report.run(reference_data=ref_df, current_data=cur_df)
+    report.save_html(output_html)
+    print(f"✅ Drift report saved to {output_html}")
+
+
+# --- Step 6: Evaluation + SHAP ---
+def evaluate_model(model, X_train, X_test, y_train, y_test, feature_names):
+    model.fit(X_train, y_train)
+    y_pred = model.predict(X_test)
+    print(classification_report(y_test, y_pred))
+    ConfusionMatrixDisplay.from_predictions(y_test, y_pred)
+    plt.title("Evaluation")
+    plt.tight_layout()
+    plt.show()
+
+    X_proc = model.named_steps['preprocessor'].transform(X_test)
+    if scipy.sparse.issparse(X_proc):
+        X_proc = X_proc.toarray()
+
+    selector = model.named_steps['feature_select']
+    X_selected = selector.transform(X_proc)
+
+    explainer = shap.Explainer(model.named_steps['classifier'].base_estimator, feature_names=feature_names)
+    shap_values = explainer(X_selected)
+    shap.summary_plot(shap_values, X_selected)
+
+# --- Final Orchestration ---
+def status_prediction_model(df):
+    os.makedirs("model_artifacts", exist_ok=True)
+    print("🧹 Preparing data...")
+    df = prepare_data(df, is_train=True)
+    print("💡 Embedding text...")
+    df = compute_embeddings(df, ['title', 'objective'])
+
+    text_embed_cols = [col for col in df.columns if '_embed_' in col]
+    numeric_features = ['durationDays', 'ecMaxContribution', 'totalCost',
+                        'n_partners', 'n_country', 'n_sme'] + text_embed_cols
+    categorical_features = ['fundingScheme', 'legalBasis', 'nature']
+    multilabel_fields =  ['list_country', 'list_activityType', 'list_deliverableType',
+        'list_availableLanguages', 'list_euroSciVocTitle','euroSciVoc_intermediate']
+    
+    
+    df = df[numeric_features + categorical_features + multilabel_fields + ['label']]
+    X = df.drop(columns='label')
+    y = df['label']
+
+
+    X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, test_size=0.2, random_state=42)
+
+    print("🧱 Building pipeline...")
+    preprocessor = build_preprocessor(numeric_features, categorical_features, multilabel_fields)
+    base_model = XGBClassifier(eval_metric='logloss', n_jobs=-1)
+
+    print("🎯 Training model with Optuna...")
+    def objective(trial):
+        params = {
+            'n_estimators': trial.suggest_int('n_estimators', 100, 300),
+            'max_depth': trial.suggest_int('max_depth', 3, 10),
+            'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3),
+            'scale_pos_weight': trial.suggest_float('scale_pos_weight', 2.0, 10.0)
+        }
+        base_model.set_params(**params)
+        pipeline = build_pipeline(preprocessor, base_model)
+        scores = cross_val_score(pipeline, X, y, cv=StratifiedKFold(3, shuffle=True, random_state=42),
+                                 scoring=make_scorer(f1_score, pos_label=1),n_jobs=-1)
+        return scores.mean()
+
+    study = optuna.create_study(direction='maximize')
+    study.optimize(objective, n_trials=24,n_jobs=6)
+    best_params = study.best_trial.params
+    base_model.set_params(**best_params)
+
+    print("✅ Training final model and evaluating...")
+    final_pipeline = build_pipeline(preprocessor, base_model)
+    selector = final_pipeline.named_steps['feature_select']
+    if hasattr(selector, 'get_support'):
+        feature_names = np.array(final_pipeline.named_steps['preprocessor'].get_feature_names_out())[selector.get_support()]
+    else:
+        feature_names = np.array(final_pipeline.named_steps['preprocessor'].get_feature_names_out())
+    evaluate_model(final_pipeline, X_train, X_test, y_train, y_test, feature_names)
+
+    print("📊 Monitoring drift...")
+    ref_data = preprocessor.transform(X_train)
+    cur_data = preprocessor.transform(X_test)
+    if scipy.sparse.issparse(ref_data): ref_data = ref_data.toarray()
+    if scipy.sparse.issparse(cur_data): cur_data = cur_data.toarray()
+    monitor_drift(pd.DataFrame(ref_data), pd.DataFrame(cur_data), feature_names)
+    print("💾 Saving model and artifacts...")
+    joblib.dump(final_pipeline, "model_artifacts/model.pkl")
+    joblib.dump(preprocessor, "model_artifacts/preprocessor.pkl")
+    X_train.to_csv("model_artifacts/X_train_processed.csv", index=False)
+    y_train.to_csv("model_artifacts/y_train.csv", index=False)
+    feature_config = {
+        "numeric_features": numeric_features,
+        "categorical_features": categorical_features,
+        "multilabel_fields": multilabel_fields
+    }
+    json.dump(feature_config, open("model_artifacts/feature_config.json", "w"))
+    print("✅ Training complete. Model artifacts saved.")
+
+def score(new_df, model_dir="model_artifacts"):
+    # 1) Load artifacts
+    pipe = joblib.load(os.path.join(model_dir, "model.pkl"))
+    config = json.load(open(os.path.join(model_dir, "feature_config.json")))
+    
+    # 2) Prepare & embed exactly as in training
+    df = prepare_data(new_df.copy(), is_train=False)
+    text_cols = ['title', 'objective']
+    sbert = SentenceTransformer('sentence-transformers/LaBSE')
+    for col in text_cols:
+        # load the SVD you trained
+        svd = joblib.load(os.path.join(model_dir, f"{col}_svd.pkl"))
+        emb = sbert.encode(df[col].tolist(), show_progress_bar=False)
+        reduced = svd.transform(emb)
+        emb_df = pd.DataFrame(reduced,
+                              columns=[f"{col}_embed_{i}" for i in range(reduced.shape[1])],
+                              index=df.index)
+        df = pd.concat([df, emb_df], axis=1)
+
+    # 3) Build the final feature set
+    X = df[ config["numeric_features"]
+          + config["categorical_features"]
+          + config["multilabel_fields"] ]
+
+    # 4) Predict & attach to DataFrame
+    preds = pipe.predict(X)
+    probs = pipe.predict_proba(X)[:, 1]   # assume binary and positive class = index 1
+    df["predicted_label"] = preds
+    df["predicted_prob"]  = probs
+
+    # 5) SHAP explanations on the *selected* features
+    #    (we need to re-run preprocessing + feature_selection)
+    preproc = pipe.named_steps["preprocessor"]
+    select  = pipe.named_steps["feature_select"]
+    clf     = pipe.named_steps["classifier"].base_estimator
+
+    X_proc = preproc.transform(X)
+    if scipy.sparse.issparse(X_proc):
+        X_proc = X_proc.toarray()
+    X_sel = select.transform(X_proc)
+
+    feature_names = select.get_feature_names_out(
+        preproc.get_feature_names_out()
+    )
+
+    # Use a TreeExplainer directly on the XGB base estimator
+    explainer = shap.Explainer(clf, X_sel, feature_names=feature_names)
+    shap_vals = explainer(X_sel)   # returns a ShapleyValues object
+
+    # 6) For each row, pick top-3 absolute contributors
+    shap_df = pd.DataFrame(shap_vals.values, columns=feature_names, index=df.index)
+    abs_shap = shap_df.abs()
+
+    top_feats = abs_shap.apply(lambda row: row.nlargest(4).index.tolist(), axis=1)
+    top_vals  = abs_shap.apply(lambda row: row.nlargest(4).values.tolist(), axis=1)
+
+    df[["top1_feature","top2_feature","top3_feature","top4_feature"]] = pd.DataFrame(
+        top_feats.tolist(), index=df.index
+    )
+    df[["top1_shap","top2_shap","top3_shap","top4_shap"]] = pd.DataFrame(
+        top_vals.tolist(),  index=df.index
+    )
+
+    return df
+
+if __name__ == "__main__":
+    df = pd.read_csv("your_data.csv")
+
+    status_prediction_model(df)
+
+    new_df = pd.read_csv("new_data.csv")
+    scored_df = score(new_df)
+    print(scored_df.head())

rag_test.py

@@ -1,223 +0,0 @@
-import streamlit as st
-import pandas as pd
-import numpy as np
-import faiss
-import pickle
-import spacy
-import re
-from sentence_transformers import SentenceTransformer
-from langchain.prompts import PromptTemplate
-from langchain.llms import OpenAI
-import matplotlib.pyplot as plt
-
-# --- Load Models and Data ---
-@st.cache_resource
-def load_models_and_data():
-    nlp = spacy.load("en_core_web_sm")
-    embedding_model = SentenceTransformer("all-MiniLM-L6-v2")
-
-    projects_df = pd.read_parquet("cordis_projects.parquet")
-    with open("topic_summaries.pkl", "rb") as f:
-        topic_summaries_df = pickle.load(f)
-
-    project_index = faiss.read_index("project_chunks.faiss")
-    topic_index = faiss.read_index("topic_summary_index.faiss")
-
-    return nlp, embedding_model, projects_df, topic_summaries_df, project_index, topic_index
-
-nlp, embedding_model, projects_df, topic_summaries_df, project_index, topic_index = load_models_and_data()
-
-# --- RAG Components ---
-
-class SessionContext:
-    def __init__(self):
-        self.last_query_type = None
-        self.last_entity = None
-        self.last_projects_df = None
-        self.last_project_id = None
-
-    def update(self, query_type, entity, df=None, project_id=None):
-        self.last_query_type = query_type
-        self.last_entity = entity
-        self.last_projects_df = df
-        self.last_project_id = project_id
-
-session_context = SessionContext()
-
-def classify_query_type(query: str) -> str:
-    query = query.lower()
-    if re.search(r'\b(project|grant agreement|projectid|rcn|ga)\b', query):
-        return "project"
-    if re.search(r'\b(topic|eurovoc|euro-scivoc)\b', query):
-        return "topic"
-    if re.search(r'\b(organization|institution|company|beneficiary)\b', query):
-        return "organization"
-    if re.search(r'\b(legalbasis|legislation|h2020|fp7)\b', query):
-        return "legalBasis"
-    return "general"
-
-def extract_entities_custom(query: str) -> dict:
-    entities = {"project_id": None, "organization": None, "topic": None}
-    doc = nlp(query)
-    match = re.search(r"\b\d{6,8}\b", query)
-    if match:
-        entities["project_id"] = match.group(0)
-    for ent in doc.ents:
-        if ent.label_ == "ORG":
-            entities["organization"] = ent.text
-        if ent.label_ == "MISC":
-            entities["topic"] = ent.text
-    return entities
-
-def retrieve_project_chunks_by_id(project_id: str) -> list:
-    subset = projects_df[projects_df['project_id'] == project_id]
-    return subset['chunk_text'].tolist()
-
-def retrieve_topic_summary_from_project(project_id: str) -> str:
-    row = projects_df[projects_df['project_id'] == project_id]
-    if row.empty:
-        return ""
-    topic = row['topic_path'].values[0]
-    match = topic_summaries_df[topic_summaries_df['topic'] == topic]
-    return match['summary'].values[0] if not match.empty else ""
-
-def get_kpi_context() -> str:
-    return (
-        "Average project duration: 780 days\n"
-        "Termination rate: 12.3%\n"
-        "Top countries by project count: Germany, France, Italy\n"
-        "Common termination reasons: coordination failure, underperformance"
-    )
-
-def run_reasoning(question: str, df: pd.DataFrame) -> str:
-    q = question.lower()
-    if "average" in q and "funding" in q:
-        avg = df['ecMaxContribution'].mean()
-        return f"The average funding is €{avg:,.2f}."
-    return ""
-
-def generate_follow_up_suggestions(query_type: str) -> str:
-    if query_type == "project":
-        return "Would you like more info on related topics or organizations involved?"
-    if query_type == "organization":
-        return "Want to explore more projects from this organization?"
-    if query_type == "topic":
-        return "Would you like to see top projects under this topic?"
-    return ""
-
-def is_follow_up_question(question: str) -> bool:
-    q = question.lower()
-    return any(phrase in q for phrase in [
-        "they", "those", "what do they", "what are they about", "explain them", "what's the topic"
-    ])
-
-def summarize_projects(df: pd.DataFrame, field="objective", top_n=5) -> str:
-    texts = df[field].dropna().tolist()[:top_n]
-    if not texts:
-        return "No objectives or summaries available for these projects."
-    combined_text = "\n\n".join(texts)
-    prompt = f"Summarize what these projects are generally about:\n\n{combined_text}"
-    return OpenAI(temperature=0)(prompt)
-
-rag_template = PromptTemplate(
-    input_variables=["project_info", "topic_summary", "kpi_context", "programmatic", "question", "followup"],
-    template="""
-You are a research assistant for a European funding agency. Use the provided context to answer the user query.
-
---- Project Info ---
-{project_info}
-
---- Topic Summary ---
-{topic_summary}
-
---- KPI Context ---
-{kpi_context}
-
---- Data Insights ---
-{programmatic}
-
---- Question ---
-{question}
-
---- Answer ---
-
-Also consider:
-{followup}
-"""
-)
-
-def chat_with_context(question: str, session: SessionContext) -> str:
-    if is_follow_up_question(question) and session.last_projects_df is not None:
-        return summarize_projects(session.last_projects_df)
-
-    query_type = classify_query_type(question)
-    entities = extract_entities_custom(question)
-    project_info = topic_summary = programmatic = ""
-    relevant_df = projects_df.copy()
-    project_id = None
-
-    if entities["project_id"]:
-        project_id = entities["project_id"]
-        project_info = "\n".join(retrieve_project_chunks_by_id(project_id))
-        topic_summary = retrieve_topic_summary_from_project(project_id)
-        relevant_df = projects_df[projects_df['project_id'] == project_id]
-
-    elif query_type == "organization":
-        org = entities.get("organization")
-        if org:
-            relevant_df = projects_df[projects_df['list_name'].str.contains(org, case=False, na=False)]
-            project_info = "\n".join(relevant_df['chunk_text'].head(3))
-
-    programmatic = run_reasoning(question, relevant_df)
-    kpi_context = get_kpi_context()
-    followup = generate_follow_up_suggestions(query_type)
-
-    prompt = rag_template.format(
-        project_info=project_info,
-        topic_summary=topic_summary,
-        kpi_context=kpi_context,
-        programmatic=programmatic,
-        question=question,
-        followup=followup
-    )
-
-    session.update(query_type, entities.get("organization") or entities.get("project_id"), relevant_df, project_id)
-
-    return OpenAI(temperature=0)(prompt)
-
-# --- Streamlit UI ---
-
-st.set_page_config(page_title="EU Funding Explorer", layout="wide")
-st.title("🇪🇺 EU Projects Dashboard")
-
-tabs = st.tabs(["📊 Dashboard", "📁 Projects + Chatbot"])
-
-# --- Tab 1: Dashboard ---
-with tabs[0]:
-    st.subheader("Funding Overview")
-    funding_by_year = projects_df.groupby("startYear")["ecMaxContribution"].sum().reset_index()
-    plt.figure(figsize=(10,4))
-    plt.bar(funding_by_year["startYear"], funding_by_year["ecMaxContribution"] / 1e6)
-    plt.ylabel("Total Funding (€M)")
-    st.pyplot(plt)
-
-    top_orgs = projects_df["list_name"].value_counts().head(10)
-    st.bar_chart(top_orgs)
-
-# --- Tab 2: Projects + Chatbot ---
-with tabs[1]:
-    left, right = st.columns([2, 1])
-
-    with left:
-        st.subheader("Browse Projects")
-        page_size = 10
-        page_num = st.number_input("Page", min_value=0, max_value=(len(projects_df) // page_size), step=1)
-        paginated = projects_df.iloc[page_num * page_size : (page_num + 1) * page_size]
-        st.dataframe(paginated[["title", "list_name", "ecMaxContribution", "startYear", "status"]], use_container_width=True)
-
-    with right:
-        st.subheader("Ask the Chatbot")
-        user_input = st.text_input("Ask a question...")
-        if st.button("Ask") and user_input:
-            answer = chat_with_context(user_input, session_context)
-            st.markdown(answer)