{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "edd8164e-9c1e-4919-98ae-d4e733f94951",
   "metadata": {},
   "outputs": [],
   "source": [
    "import plotly\n",
    "import pandas as pd\n",
    "import plotly.graph_objects as go\n",
    "import plotly.express as px\n",
    "import json\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import os\n",
    "import matplotlib.patches as mpatches\n",
    "import matplotlib.colors as mplc\n",
    "import re\n",
    "import plotly.colors\n",
    "from plotly.offline import plot\n",
    "from plotly.subplots import make_subplots\n",
    "import plotly.io as pio\n",
    "pio.renderers.default = \"plotly_mimetype\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4b460d1-ae0e-4498-9d36-171f95482952",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| label: fig1cell\n",
    "\n",
    "screening_info = ['Records obtained from Embase',\n",
    "                  'Records obtained from MEDLINE',\n",
    "                  'Records obtained from Google Scholar',\n",
    "                  'Records selected for screening',\n",
    "                  'Duplicate references removed',\n",
    "                  \"\"\"\n",
    "                    Title and abstract screened for relevance:<br>\n",
    "                    - Studies not concerning brain imaging<br>\n",
    "                    - Studies not concerning human populations<br>\n",
    "                    - Studies not concerning MRI or PET<br>\n",
    "                    - Studies not concerning Quebec population\n",
    "                  \"\"\",\n",
    "                  'Studies retrieved for evaluation',\n",
    "                  \"\"\"\n",
    "                    Exclusion criteria:<br>\n",
    "                    - Review articles (n=31);<br>\n",
    "                    - Ambiguous CT or MRI patient counts (n=14);<br>\n",
    "                    - Case Reports (n=99);<br>\n",
    "                    - Animal studies (n=14);<br>\n",
    "                    - Correspondence (n=1);<br>\n",
    "                    - Public dataset (n=84);<br>\n",
    "                    - Conference paper (n=16);<br>\n",
    "                    - Multisite dataset with unknown participant counts (n=776);<br>\n",
    "                    - Post-mortem brain (n=14);<br>\n",
    "                    - Registered report (n=2);<br>\n",
    "                    - Wrong study design (n=8);<br>\n",
    "                    - Unspecified location (n=37);<br>\n",
    "                    - Study population outside Quebec (n=2052);<br>\n",
    "                    - Duplicate dataset (n=104);\n",
    "                  \"\"\",\n",
    "                  'Studies selected for analysis']\n",
    "\n",
    "fig1 = go.Figure(data=[go.Sankey(\n",
    "    arrangement = \"freeform\",\n",
    "    node = dict(\n",
    "      pad = 80,\n",
    "      thickness = 10,\n",
    "      line = dict(color = \"black\", width = 0.5),\n",
    "      label = [\"References obtained from Embase\",#0\n",
    "               \"References obtained from MEDLINE\",#1\n",
    "               \"References obtained from Google Scholar\",#2\n",
    "               \"Main references identified\",#3\n",
    "               \"Studies screened\",#4\n",
    "               \"References removed\",#5\n",
    "               \"Studies excluded from retrieval\", #6\n",
    "               \"Studies assessed for eligibility\",#7\n",
    "               \"Studies exluded from review\",#8\n",
    "               \"Studies included in review\"],#9\n",
    "      x = [0, 0, 0, 0.3, 0.5, 0.5, 0.7, 0.7, 0.9, 0.9],\n",
    "      y = [2, 0, 0, 0.5, 0.3, 0.9, 0.8, 0.4, 0.6, 0.2],\n",
    "      hovertemplate = \"%{label}<extra>%{value}</extra>\",\n",
    "      color = [\"darkblue\",\"darkblue\",\"darkblue\",\"darkblue\",\"darkgreen\",\"darkred\",\"darkred\",\"darkgreen\",\"darkred\",\"darkgreen\"]\n",
    "    ),\n",
    "    link = dict(\n",
    "      source = [0, 1, 2, 3, 3, 4, 4, 7, 7],\n",
    "      target = [3, 3, 3, 4, 5, 6, 7, 8, 9],\n",
    "      value = [6429, 3816, 100, 7279, 3068, 2476, 4801, 3252, 1549],\n",
    "      customdata = screening_info,\n",
    "      hovertemplate = \"%{customdata}\",\n",
    "  ))])\n",
    "\n",
    "fig1.update_layout(width=750,\n",
    "                   height=500,\n",
    "                   font_size=11,\n",
    "                   margin=dict(l=0))\n",
    "    \n",
    "fig1.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1da70bac-0c16-42a5-ba33-f72aa9cbb34b",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = os.path.join('..', 'data','qc-imaging-demographics')\n",
    "\n",
    "fname_data = os.path.join(data_path, f\"fcdata.xlsx\")\n",
    "df = pd.read_excel(fname_data)\n",
    "fname2 = os.path.join(data_path, f\"Cleaned Full data.xlsx\")\n",
    "df_full = pd.read_excel(fname2, sheet_name='Full that matches the formatted')\n",
    "\n",
    "# merge superfluous ethnicity columns\n",
    "df['Black'] = df.loc[:,['Black','African-American']].sum(axis=1)\n",
    "df['White'] = df.loc[:,['Caucasian','Caucasian-Hispanic']].sum(axis=1)\n",
    "df['Asian'] = df.loc[:,['Asian','Asian American']].sum(axis=1)\n",
    "df = df.drop(columns=['African-American','Caucasian','Caucasian-Hispanic','Asian American'])\n",
    "df.fillna({'Other': 0, 'Not specified' : 0, 'Middle Eastern' : 0, 'Caribbean' : 0, 'Jewish' : 0}, inplace=True)\n",
    "\n",
    "###############\n",
    "# create total participant column on the rule that if PET=MRI then all participants were scanned in both, and if not then total = PET+MRI\n",
    "df['Total participants'] = df.loc[:,['PET participants','MRI participants']].sum(axis=1)\n",
    "df.loc[df['PET participants']==df['MRI participants'], 'Total participants'] = df['MRI participants']\n",
    "\n",
    "# hard coding execptions to the above rule\n",
    "df.loc[118, 'Total participants'] = 25\n",
    "df.loc[268, 'Total participants'] = 61\n",
    "df.loc[356, 'Total participants'] = 15\n",
    "df.loc[477, 'Total participants'] = 29\n",
    "df.loc[572, 'Total participants'] = 24\n",
    "df.loc[626, 'Total participants'] = 20\n",
    "df.loc[832, 'Total participants'] = 60\n",
    "df.loc[840, 'Total participants'] = 21\n",
    "df.loc[1113, 'Total participants'] = 34\n",
    "df.loc[1128, 'Total participants'] = 35\n",
    "df.loc[1143, 'Total participants'] = 14\n",
    "df.loc[1178, 'Total participants'] = 9\n",
    "df.loc[1215, 'Total participants'] = 40\n",
    "df.loc[1293, 'Total participants'] = 262\n",
    "df.loc[1325, 'Total participants'] = 31\n",
    "df.loc[1369, 'Total participants'] = 44\n",
    "df.loc[1446, 'Total participants'] = 54\n",
    "df.loc[1488, 'Total participants'] = 39\n",
    "###############\n",
    "\n",
    "df_full['MRI participants'] = pd.to_numeric(df_full['MRI participants'], errors='coerce').fillna(0)\n",
    "df_full['PET participants'] = pd.to_numeric(df_full['PET participants'], errors='coerce').fillna(0)\n",
    "df_full['Male'] = pd.to_numeric(df_full['Male'], errors='coerce').fillna(0)\n",
    "df_full['Female'] = pd.to_numeric(df_full['Female'], errors='coerce').fillna(0)\n",
    "df_full['Total participants'] = df_full.loc[:,['PET participants','MRI participants']].sum(axis=1)\n",
    "df_full.loc[df_full['PET participants']==df_full['MRI participants'], 'Total participants'] = df_full['MRI participants']\n",
    "df_full.loc[17, 'Total participants'] = 25\n",
    "df_full.loc[1077, 'Total participants'] = 61\n",
    "df_full.loc[833, 'Total participants'] = 15\n",
    "df_full.loc[870, 'Total participants'] = 29\n",
    "df_full.loc[667, 'Total participants'] = 24\n",
    "df_full.loc[498, 'Total participants'] = 20\n",
    "df_full.loc[757, 'Total participants'] = 60\n",
    "df_full.loc[342, 'Total participants'] = 21\n",
    "df_full.loc[1548, 'Total participants'] = 34\n",
    "df_full.loc[41, 'Total participants'] = 35\n",
    "df_full.loc[1128, 'Total participants'] = 14\n",
    "df_full.loc[517, 'Total participants'] = 9\n",
    "df_full.loc[987, 'Total participants'] = 40\n",
    "df_full.loc[1499, 'Total participants'] = 262\n",
    "df_full.loc[82, 'Total participants'] = 31\n",
    "df_full.loc[1061, 'Total participants'] = 44\n",
    "df_full.loc[591, 'Total participants'] = 54\n",
    "df_full.loc[42, 'Total participants'] = 39\n",
    "\n",
    "\n",
    "# Create unreported column based on total participants\n",
    "df['Unreported Ethnicity'] = df.loc[:,'Total participants'] - df.loc[:,['Black','Asian','Other','Not specified','Middle Eastern','Caribbean','Jewish','White']].sum(axis=1)\n",
    "i = list(df.columns)\n",
    "a, b = i.index('Unreported Ethnicity'), i.index('Total participants')\n",
    "i[b], i[a] = i[a], i[b]\n",
    "df = df[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "461f6b35-08dc-4fdc-9995-ccd0bd408969",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Categorize pathologies\n",
    "\n",
    "df_paths = df_full.copy()\n",
    "\n",
    "# categorizing patient populations\n",
    "pathology = {\"Epilepsy\" : \"epilepsy|epileptic\",\n",
    "             \"Parkinson's\" : \"Parkinson\",\n",
    "             \"Alzheimer's\" : \"Alzheimer\",\n",
    "             \"Mental and behavioral disorders\" : \"Schizophrenia|depression|depressive|anxiety|psychosis|addiction|autism|alcohol|amusia|serotonin|emotion|PTSD|suicide|suicidal|substance|Attention|bipolar|OCD|personality\",               \n",
    "             \"Multiple Sclerosis\" : \"Multiple Sclerosis\",                 \n",
    "             \"Cerebrovascular disease\" : \"Carotid|Stenosis|Stroke|vascualar\",                 \n",
    "             \"Cancer and tumors\" : \"tumor|cancer|adenoma|glioblastoma|angioma|neurofibromatosis|glioma|lymphoma|meningioma|schwannoma|hamartoma|sarcoma|leukemia\",                 \n",
    "             \"Newborns and infants\" : \"Neonatal|infant|newborn|child\",\n",
    "             \"Concussion and Injury\" : \"Injury|Concussion\",\n",
    "             \"Vision disorders\" : \"yopia|blind|hemianopia\",\n",
    "             \"Pain\" : \"pain\", \n",
    "             \"Healthy population\" : \"Not specified|Healthy|aging|volunteer\"}\n",
    "\n",
    "# make categorized pathology columns based on disease characteristics keywords\n",
    "for p, keywords in pathology.items():\n",
    "    df_paths[p] = df_paths['Disease Characteristics'].str.contains(keywords, flags=re.IGNORECASE, regex=True)\n",
    "\n",
    "df_paths['number of pathologies'] = df_paths[df_paths.columns[23:35]].sum(axis=1)\n",
    "df_paths['Pathology'] = \"\"\n",
    "\n",
    "# run through individual pathology columns and set the pathology for each true value\n",
    "for n in df_paths.columns[23:35]:\n",
    "    df_paths.loc[df_paths[n]==True,'Pathology'] = n\n",
    "\n",
    "# categorize anything with multiple pathologies or anything lying outside the scope of our keywords\n",
    "for i in df_paths.index:\n",
    "    if df_paths.loc[i,'number of pathologies'] == 0:\n",
    "        df_paths.loc[i,'Pathology'] = \"Other\"\n",
    "    elif df_paths.loc[i,'number of pathologies'] > 1:\n",
    "        df_paths.loc[i,'Pathology'] = \"Multiple categories\"\n",
    "\n",
    "# fix blank participant counts\n",
    "df_paths['MRI participants'] = pd.to_numeric(df_paths['MRI participants'], errors='coerce')\n",
    "df_paths['PET participants'] = pd.to_numeric(df_paths['PET participants'], errors='coerce')\n",
    "\n",
    "# drop studies with ambiguous mri/pet participant counts (no values listed) (2 such studies)\n",
    "i = df_paths.loc[df_paths['MRI participants'].isna() & df_paths['PET participants'].isna()].index\n",
    "df_paths = df_paths.drop(i)\n",
    "\n",
    "# catgeorize mri/pet/both\n",
    "df_paths['MRI or PET'] = \"\"\n",
    "for i in df_paths.index:\n",
    "    if (df_paths['MRI participants'][i]==0):\n",
    "        df_paths.loc[i,'MRI or PET'] = 'PET'\n",
    "    elif (df_paths['PET participants'][i]==0):\n",
    "        df_paths.loc[i,'MRI or PET'] = 'MRI'\n",
    "    else:\n",
    "        df_paths.loc[i,'MRI or PET'] = 'MRI and PET'\n",
    "\n",
    "# handle studies with multiple geographical locations\n",
    "idx = df_paths['Geographical location'].str.contains(',')\n",
    "df_paths.loc[idx,'Geographical location'] = 'Multiple regions'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c16085e-4fad-492f-bca5-afb962f94bbf",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| label: fig2cell\n",
    "\n",
    "year_str = df_paths['Year of Publication'].astype(str)\n",
    "df_paths['Geographical location'] = df_paths['Geographical location'].replace(\"Montreal\\n\",'Montreal')\n",
    "\n",
    "# list study info line as (first author et al., year)\n",
    "df_paths['Study info'] = df_paths['Authors'].str.split(',').str[0].str.split().str[-1] + ' et al., ' + year_str\n",
    "\n",
    "# construct hyperlink for each paper's unique DOI\n",
    "df_paths['Link'] = df_paths['DOI']\n",
    "df_paths['Link'] = df_paths['Link'].replace('http',\"\"\"<a style='color:white' href='http\"\"\", regex=True)\n",
    "df_paths['Link'] = df_paths['Link'] + \"\"\"'>->Go to the paper</a>\"\"\"\n",
    "\n",
    "# make text box for each entry with the link and title\n",
    "df_paths['Summary'] = df_paths['Link'] + '<br><br>Title: ' + df_paths['Title'].astype(str)\n",
    "df_paths['N'] = np.ones((len(df_paths),1))\n",
    "\n",
    "# handle duplicates of study info (multiple studies with same year and first author last name)\n",
    "for i in range(6):\n",
    "    dupes = df_paths['Study info'].duplicated()\n",
    "    df_paths.loc[dupes,'Study info'] = df_paths.loc[dupes,'Study info'] + ' '\n",
    "\n",
    "df_paths = df_paths.sort_values('Study info')\n",
    "\n",
    "# construct dataframe for the treemap from the list of studies\n",
    "args = dict(data_frame=df_paths, values='N',\n",
    "            color='N', hover_data='Study info',\n",
    "            path=['MRI or PET', 'Geographical location', 'Pathology', 'Study info'])\n",
    "args = px._core.build_dataframe(args, go.Treemap)\n",
    "treemap_df = px._core.process_dataframe_hierarchy(args)['data_frame']\n",
    "\n",
    "fig2 = go.Figure(go.Treemap(\n",
    "    ids=list(treemap_df['id']),\n",
    "    labels=list(treemap_df['labels']),\n",
    "    parents=list(treemap_df['parent']),\n",
    "    values=list(treemap_df['N']),\n",
    "    branchvalues='remainder',\n",
    "    text=df_paths['Summary'],\n",
    "    hoverinfo='label', # Set to None if sorting by DOI\n",
    "    # customdata = treemap_df['Study info'], # use if sorting by DOI\n",
    "    # hovertemplate = \"<b>%{customdata}</b><extra></extra>\", # to be used if sorting by DOI\n",
    "    textfont=dict(\n",
    "        size=15,\n",
    "    )\n",
    "))\n",
    "\n",
    "fig2 = fig2.update_layout(\n",
    "    autosize=False,\n",
    "    width=750,\n",
    "    height=550,\n",
    "    margin=dict(\n",
    "        l=0,\n",
    "        r=0,\n",
    "        t=0,\n",
    "        b=45,\n",
    "    )\n",
    ")\n",
    "\n",
    "fig2.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e5d7830-bd8e-42b7-acd0-eeae04e6ff17",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Categorize geographically by number of studies \n",
    "# Drop studies with multiple locations, plot number of studies by city\n",
    "filtered = df[~df['Geographical location'].str.contains(',')]\n",
    "# removed 10 studies with multiple locations listed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "933ee074-2e35-4042-bba6-b1340504520c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#counting studies by region\n",
    "# all\n",
    "s1 = filtered['Geographical location'].value_counts()\n",
    "# PET only\n",
    "s2 = filtered[filtered['PET participants'].notna() & filtered['MRI participants'].isna()]['Geographical location'].value_counts()\n",
    "# MRI only\n",
    "s3 = filtered[filtered['PET participants'].isna() & filtered['MRI participants'].notna()]['Geographical location'].value_counts()\n",
    "# PET+MRI studies\n",
    "s4 = filtered[filtered['PET participants'].notna() & filtered['MRI participants'].notna()]['Geographical location'].value_counts()\n",
    "\n",
    "# all\n",
    "regs_all = pd.DataFrame({'Region' : s1.index, 'count' : s1.values})\n",
    "regs_all['Region'] = ['Montreal', 'Estrie', 'Capitale-Nationale', 'Mauricie', 'Saguenay - Lac-Saint-Jean', 'Monteregie']\n",
    "regs_all.loc[len(regs_all.index)]=['Gaspesie - Iles-de-la-Madeleine', 0]\n",
    "regs_all.loc[len(regs_all.index)]=['Abitibi-Temiscamingue', 0]\n",
    "regs_all.loc[len(regs_all.index)]=['Outaouais', 0]\n",
    "regs_all.loc[len(regs_all.index)]=['Nord-du-Quebec', 0]\n",
    "regs_all.loc[len(regs_all.index)]=['Laurentides', 0]\n",
    "regs_all.loc[len(regs_all.index)]=['Lanaudiere', 0]\n",
    "regs_all.loc[len(regs_all.index)]=['Chaudiere-Appalaches', 0]\n",
    "regs_all.loc[len(regs_all.index)]=['Cote-Nord', 0]\n",
    "regs_all.loc[len(regs_all.index)]=['Bas-Saint-Laurent', 0]\n",
    "regs_all.loc[len(regs_all.index)]=['Centre-du-Quebec', 0]\n",
    "regs_all.loc[len(regs_all.index)]=['Laval', 0]\n",
    "\n",
    "# PET only\n",
    "regs_pet = pd.DataFrame({'Region' : s2.index, 'count' : s2.values})\n",
    "regs_pet['Region'] = ['Montreal', 'Capitale-Nationale', 'Estrie']\n",
    "regs_pet.loc[len(regs_pet.index)]=['Mauricie', 0]\n",
    "regs_pet.loc[len(regs_pet.index)]=['Saguenay - Lac-Saint-Jean', 0]\n",
    "regs_pet.loc[len(regs_pet.index)]=['Monteregie', 0]\n",
    "regs_pet.loc[len(regs_pet.index)]=['Gaspesie - Iles-de-la-Madeleine', 0]\n",
    "regs_pet.loc[len(regs_pet.index)]=['Abitibi-Temiscamingue', 0]\n",
    "regs_pet.loc[len(regs_pet.index)]=['Outaouais', 0]\n",
    "regs_pet.loc[len(regs_pet.index)]=['Nord-du-Quebec', 0]\n",
    "regs_pet.loc[len(regs_pet.index)]=['Laurentides', 0]\n",
    "regs_pet.loc[len(regs_pet.index)]=['Lanaudiere', 0]\n",
    "regs_pet.loc[len(regs_pet.index)]=['Chaudiere-Appalaches', 0]\n",
    "regs_pet.loc[len(regs_pet.index)]=['Cote-Nord', 0]\n",
    "regs_pet.loc[len(regs_pet.index)]=['Bas-Saint-Laurent', 0]\n",
    "regs_pet.loc[len(regs_pet.index)]=['Centre-du-Quebec', 0]\n",
    "regs_pet.loc[len(regs_pet.index)]=['Laval', 0]\n",
    "\n",
    "# MRI only\n",
    "regs_mri = pd.DataFrame({'Region' : s3.index, 'count' : s3.values})\n",
    "regs_mri['Region'] = ['Montreal', 'Estrie', 'Capitale-Nationale', 'Mauricie', 'Saguenay - Lac-Saint-Jean', 'Monteregie']\n",
    "regs_mri.loc[len(regs_mri.index)]=['Gaspesie - Iles-de-la-Madeleine', 0]\n",
    "regs_mri.loc[len(regs_mri.index)]=['Abitibi-Temiscamingue', 0]\n",
    "regs_mri.loc[len(regs_mri.index)]=['Outaouais', 0]\n",
    "regs_mri.loc[len(regs_mri.index)]=['Nord-du-Quebec', 0]\n",
    "regs_mri.loc[len(regs_mri.index)]=['Laurentides', 0]\n",
    "regs_mri.loc[len(regs_mri.index)]=['Lanaudiere', 0]\n",
    "regs_mri.loc[len(regs_mri.index)]=['Chaudiere-Appalaches', 0]\n",
    "regs_mri.loc[len(regs_mri.index)]=['Cote-Nord', 0]\n",
    "regs_mri.loc[len(regs_mri.index)]=['Bas-Saint-Laurent', 0]\n",
    "regs_mri.loc[len(regs_mri.index)]=['Centre-du-Quebec', 0]\n",
    "regs_mri.loc[len(regs_mri.index)]=['Laval', 0]\n",
    "\n",
    "# Studies with MRI and PET imaging\n",
    "regs_both = pd.DataFrame({'Region' : s4.index, 'count' : s4.values})\n",
    "regs_both['Region'] = ['Montreal', 'Estrie', 'Capitale-Nationale']\n",
    "regs_both.loc[len(regs_both.index)]=['Mauricie', 0]\n",
    "regs_both.loc[len(regs_both.index)]=['Saguenay - Lac-Saint-Jean', 0]\n",
    "regs_both.loc[len(regs_both.index)]=['Monteregie', 0]\n",
    "regs_both.loc[len(regs_both.index)]=['Gaspesie - Iles-de-la-Madeleine', 0]\n",
    "regs_both.loc[len(regs_both.index)]=['Abitibi-Temiscamingue', 0]\n",
    "regs_both.loc[len(regs_both.index)]=['Outaouais', 0]\n",
    "regs_both.loc[len(regs_both.index)]=['Nord-du-Quebec', 0]\n",
    "regs_both.loc[len(regs_both.index)]=['Laurentides', 0]\n",
    "regs_both.loc[len(regs_both.index)]=['Lanaudiere', 0]\n",
    "regs_both.loc[len(regs_both.index)]=['Chaudiere-Appalaches', 0]\n",
    "regs_both.loc[len(regs_both.index)]=['Cote-Nord', 0]\n",
    "regs_both.loc[len(regs_both.index)]=['Bas-Saint-Laurent', 0]\n",
    "regs_both.loc[len(regs_both.index)]=['Centre-du-Quebec', 0]\n",
    "regs_both.loc[len(regs_both.index)]=['Laval', 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60a5a830-5662-432f-a647-2fbeca16008e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load in quebec administrative region geographic data\n",
    "# region labels with accents removed so they could be read\n",
    "geoname_data = os.path.join(data_path, f\"quebec.geojson\")\n",
    "\n",
    "with open(geoname_data) as f:\n",
    "    var_geojson = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a1d9b3a-c881-4c19-b1bf-4938dc51346f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| label: fig3cell\n",
    "\n",
    "colorscale = [[0.0, \"#FFFFC0\"], [0.5, \"#CBC7C0\"], [1.0, \"#FFFFF0\"]]\n",
    "\n",
    "# filled grey map of administrative regions\n",
    "trace1 = go.Choropleth(geojson=var_geojson,\n",
    "                      showscale=False,\n",
    "                      colorscale=colorscale,\n",
    "                      zmin=0, zmax=1,\n",
    "                      z=[0.5]*len(var_geojson['features']),\n",
    "                      locations=regs_all['Region'],\n",
    "                      featureidkey='properties.name',\n",
    "                      hoverinfo='skip')\n",
    "\n",
    "\n",
    "# sized markers for number of studies by region\n",
    "maxval=regs_all['count'].max()\n",
    "minval=regs_all['count'].min()\n",
    "trace2 = go.Scattergeo(mode='markers',\n",
    "                       marker=dict(size=np.power(regs_all['count'],0.5)*10, sizemode='area',\n",
    "                                   opacity=0.8,\n",
    "                                   cmin=minval,\n",
    "                                   cmax=maxval,\n",
    "                                   color=regs_all['count'],\n",
    "                                   colorbar_title='Number of studies'),\n",
    "                       geojson=var_geojson,\n",
    "                       locations=regs_all['Region'],\n",
    "                       featureidkey='properties.name',\n",
    "                       customdata=regs_all[['Region','count']],\n",
    "                       hovertemplate=\n",
    "                       \"<b>%{customdata[0]}</b><br>\" +\n",
    "                       \"<b>Studies:</b> %{customdata[1]}<extra></extra>\")\n",
    "\n",
    "maxval=regs_pet['count'].max()\n",
    "minval=regs_pet['count'].min()\n",
    "trace3 = go.Scattergeo(mode='markers',\n",
    "                       marker=dict(size=np.power(regs_pet['count'],0.5)*30, sizemode='area',\n",
    "                                   opacity=0.8,\n",
    "                                   cmin=minval,\n",
    "                                   cmax=maxval,\n",
    "                                   color=regs_pet['count'],\n",
    "                                   colorbar_title='Number of studies'),\n",
    "                       geojson=var_geojson,\n",
    "                       locations=regs_pet['Region'],\n",
    "                       featureidkey='properties.name',\n",
    "                       customdata=regs_pet[['Region','count']],\n",
    "                       visible=False,\n",
    "                       hovertemplate=\n",
    "                       \"<b>%{customdata[0]}</b><br>\" +\n",
    "                       \"<b>Studies:</b> %{customdata[1]}<extra></extra>\")\n",
    "\n",
    "maxval=regs_mri['count'].max()\n",
    "minval=regs_mri['count'].min()\n",
    "trace4 = go.Scattergeo(mode='markers',\n",
    "                       marker=dict(size=np.power(regs_mri['count'],0.5)*10, sizemode='area',\n",
    "                                   opacity=0.8,\n",
    "                                   cmin=minval,\n",
    "                                   cmax=maxval,\n",
    "                                   color=regs_mri['count'],\n",
    "                                   colorbar_title='Number of studies'),\n",
    "                       geojson=var_geojson,\n",
    "                       locations=regs_mri['Region'],\n",
    "                       featureidkey='properties.name',\n",
    "                       customdata=regs_mri[['Region','count']],\n",
    "                       visible=False,\n",
    "                       hovertemplate=\n",
    "                       \"<b>%{customdata[0]}</b><br>\" +\n",
    "                       \"<b>Studies:</b> %{customdata[1]}<extra></extra>\")\n",
    "\n",
    "maxval=regs_both['count'].max()\n",
    "minval=regs_both['count'].min()\n",
    "trace5 = go.Scattergeo(mode='markers',\n",
    "                       marker=dict(size=np.power(regs_both['count'],0.5)*15, sizemode='area',\n",
    "                                   opacity=0.8,\n",
    "                                   cmin=minval,\n",
    "                                   cmax=maxval,\n",
    "                                   color=regs_both['count'],\n",
    "                                   colorbar_title='Number of studies'),\n",
    "                       geojson=var_geojson,\n",
    "                       locations=regs_both['Region'],\n",
    "                       featureidkey='properties.name',\n",
    "                       customdata=regs_both[['Region','count']],\n",
    "                       visible=False,\n",
    "                       hovertemplate=\n",
    "                       \"<b>%{customdata[0]}</b><br>\" +\n",
    "                       \"<b>Studies:</b> %{customdata[1]}<extra></extra>\")\n",
    "\n",
    "# remove all default layout elements, only interested in quebec\n",
    "layout = go.Layout(\n",
    "    geo=dict(showland=False,\n",
    "             showcountries=False,\n",
    "             showocean=False,\n",
    "             showrivers=False,\n",
    "             showlakes=False,\n",
    "             showcoastlines=False)\n",
    ")\n",
    "\n",
    "fig = go.Figure(data = [trace1, trace2, trace3, trace4, trace5], layout=layout)\n",
    "#conic conformal projection recommended by Statistics Canada https://www150.statcan.gc.ca/n1/pub/92-195-x/2011001/other-autre/mapproj-projcarte/m-c-eng.htm\n",
    "fig.update_geos(fitbounds=\"locations\", projection_type='conic conformal') \n",
    "fig.update_layout(\n",
    "    updatemenus=[\n",
    "        dict(\n",
    "            buttons=list([\n",
    "                dict(label=\"All studies\",\n",
    "                     method=\"update\",\n",
    "                     args=[{\"visible\": [True, True, False, False, False]}]),\n",
    "                dict(label=\"PET studies\",\n",
    "                     method=\"update\",\n",
    "                     args=[{\"visible\": [True, False, True, False, False]}]),\n",
    "                dict(label=\"MRI studies\",\n",
    "                     method=\"update\",\n",
    "                     args=[{\"visible\": [True, False, False, True, False]}]),\n",
    "                dict(label=\"PET+MRI studies\",\n",
    "                     method=\"update\",\n",
    "                     args=[{\"visible\": [True, False, False, False, True]}]),\n",
    "            ]),\n",
    "        )\n",
    "    ]\n",
    ")\n",
    "\n",
    "fig.update_layout(width=750, height=600)\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b06aa56a-b708-4d2d-857e-6046f4fda500",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| label: fig4cell\n",
    "\n",
    "df = df.rename(columns = dict({'Average Age (Years)' : 'Age'}))\n",
    "\n",
    "# studies with no age reported.\n",
    "noagedf = df[df['Age'].isna()]\n",
    "\n",
    "# drop all nans, empties, nonnumerics (entries with > or other symbols)\n",
    "Agedf = df.copy()\n",
    "Agedf['Age'] = pd.to_numeric(Agedf['Age'], errors='coerce')\n",
    "Agedf = Agedf.dropna(subset=['Age'])\n",
    "\n",
    "\n",
    "bins = np.arange(0,90,5)\n",
    "\n",
    "agebincounts = []\n",
    "agecounts = []\n",
    "labels = []\n",
    "for i,x in enumerate(bins):\n",
    "    agebincounts.append( len(Agedf[(Agedf['Age'] >= x) & (Agedf['Age'] < (x+5))].index) )\n",
    "    agecounts.append(Agedf.loc[(Agedf['Age'] >= x) & (Agedf['Age'] < (x+5)), 'Total participants'].sum())\n",
    "    labels.append( f'{x}-{x+5}' )\n",
    "\n",
    "unreported_studies = len(noagedf.index)\n",
    "unreported_participants = noagedf['Total participants'].sum()\n",
    "\n",
    "ages1 = pd.DataFrame({'Range':labels, 'count':agebincounts})\n",
    "ages2 = pd.DataFrame({'Range':labels, 'count':agecounts})\n",
    "\n",
    "ages1.loc[len(ages1.index)] = ['Unreported', unreported_studies]\n",
    "ages2.loc[len(ages2.index)] = ['Unreported', unreported_participants]\n",
    "\n",
    "colors = [\"royalblue\"]*18 + [\"dimgrey\"]\n",
    "\n",
    "bar1 = go.Bar(\n",
    "    x = ages1['Range'],\n",
    "    y = ages1['count'],\n",
    "    marker_color=colors,\n",
    "    customdata=ages1['count']/ages1['count'].sum()*100,\n",
    "    hovertemplate=\"Percent of studies: %{customdata:.3f}%<extra></extra>\",\n",
    "    yaxis='y'\n",
    ")\n",
    "\n",
    "bar2 = go.Bar(\n",
    "    x=ages2['Range'],\n",
    "    y=ages2['count'],\n",
    "    marker_color=colors,\n",
    "    customdata=ages2['count']/ages2['count'].sum()*100,\n",
    "    hovertemplate=\"Percent of participants: %{customdata:.3f}%<extra></extra>\",\n",
    "    visible=False,\n",
    "    yaxis='y'\n",
    ")\n",
    "\n",
    "bar1p = go.Bar(\n",
    "    x = ages1['Range'],\n",
    "    y = ages1['count']/ages1['count'].sum()*100,\n",
    "    marker_color=colors,\n",
    "    hovertemplate=\"Percent of studies: %{value:.3f}%<extra></extra>\",\n",
    "    yaxis='y2',\n",
    "    visible=True\n",
    ")\n",
    "\n",
    "bar2p = go.Bar(\n",
    "    x = ages2['Range'],\n",
    "    y = ages2['count']/ages2['count'].sum()*100,\n",
    "    marker_color=colors,\n",
    "    hovertemplate=\"Percent of participants: %{value:.3f}%<extra></extra>\",\n",
    "    yaxis='y2',\n",
    "    visible=False\n",
    ")\n",
    "\n",
    "fig=go.Figure(data=[bar1,bar2,bar1p,bar2p],\n",
    "             layout={\n",
    "                 'yaxis' : {'title':'Number of studies', 'showgrid':False},\n",
    "                 'yaxis2' : {'title': 'Percent of studies', 'overlaying': 'y', 'side': 'right'}\n",
    "             })\n",
    "\n",
    "fig.update_layout(xaxis_tickangle=-45,\n",
    "                 barmode='group', width=750, height=550, showlegend=False, plot_bgcolor='rgba(0.2, 0.2, 0.2, 0.2)',\n",
    "                 xaxis=dict(title=\"Average age\"),\n",
    "                 margin=dict(t=140, b=0, l=0, r=0))\n",
    "\n",
    "fig.update_layout(\n",
    "    updatemenus=[\n",
    "        dict(\n",
    "            buttons=list([\n",
    "                dict(label=\"By number of studies\",\n",
    "                     method=\"update\",\n",
    "                     args=[{\"visible\": [True, False, True, False]},\n",
    "                           {'yaxis' : {'title':'Number of studies', 'showgrid':False},\n",
    "                            'yaxis2' : {'title': 'Percent of studies', 'overlaying': 'y', 'side': 'right'}}]),\n",
    "                dict(label=\"By number of participants\",\n",
    "                     method=\"update\",\n",
    "                     args=[{\"visible\": [False, True, False, True]},\n",
    "                           {'yaxis' : {'title':'Number of participants', 'showgrid':False},\n",
    "                            'yaxis2' : {'title': 'Percent of participants', 'overlaying': 'y', 'side': 'right'}}])\n",
    "            ]),\n",
    "            xanchor='left',\n",
    "            y=1.12,\n",
    "            pad={\"l\": 50, \"t\": 0}\n",
    "        )]\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da4ce4b4-1eaa-4d6e-8abf-1f16b6157ab1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| label: fig5cell\n",
    "\n",
    "# Sex analysis \n",
    "# male and female totals reported in most studies\n",
    "male = df_full['Male'].sum()\n",
    "female = df_full['Female'].sum()\n",
    "# majority of unreported sex is studies which report no sex at all, however some is from studies which only partially report sex\n",
    "unreported = (df_full.loc[:,'Total participants'] - df_full.loc[:,['Male','Female']].sum(axis=1)).sum()\n",
    "\n",
    "plt.style.use('ggplot')\n",
    "sns.set_palette(\"colorblind\")\n",
    "labels = ['Male', 'Female', 'Unreported']\n",
    "sizes = [male, female, unreported]\n",
    "fig, ax = plt.subplots()\n",
    "ax.pie(sizes, labels=labels,autopct='%1.1f%%',explode=[0,0,.15])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99214aaf-7fcc-40f9-abb2-cc2eee0a3c38",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| label: fig6cell\n",
    "\n",
    "# drop all rows with no age information, and eliminate nonnumerics (e.g. entries where age is listed as >0.69)\n",
    "Agedf2 = df[df['Age'].notna()].copy()\n",
    "Agedf2['Age'] = pd.to_numeric(Agedf2['Age'], errors='coerce')\n",
    "Agedf2 = Agedf2.dropna(subset=['Age'])\n",
    "\n",
    "# drop rows with no male or female information\n",
    "Agedf2 = Agedf2[~((Agedf2['Female'].isna()) & Agedf2['Male'].isna())]\n",
    "# Here we are down to 1085 studies from 1553\n",
    "\n",
    "bins = np.arange(0,90,5)\n",
    "agecounts_male = []\n",
    "agecounts_female = []\n",
    "labels = []\n",
    "for i,x in enumerate(bins):\n",
    "    agecounts_male.append( Agedf2.loc[(Agedf2['Age'] >= x) & (Agedf2['Age'] < (x+5)), 'Male'].sum() )\n",
    "    agecounts_female.append( Agedf2.loc[(Agedf2['Age'] >= x) & (Agedf2['Age'] < (x+5)), 'Female'].sum() )\n",
    "    labels.append( f'{x}-{x+5}' )\n",
    "\n",
    "\n",
    "labels.reverse()\n",
    "agecounts_male.reverse()\n",
    "agecounts_female.reverse()\n",
    "pyramid_df = pd.DataFrame(data={'age range':labels, 'male':agecounts_male, 'female':agecounts_female})\n",
    "pyramid_df['male'] = pyramid_df['male'] * -1\n",
    "\n",
    "ax1 = sns.barplot(x='male', y='age range', data=pyramid_df, palette=\"Reds\", hue=labels)\n",
    "ax2 = sns.barplot(x='female', y='age range', data=pyramid_df, palette=\"Blues\", hue=labels)\n",
    "plt.xlabel(\"             Male    Female\")\n",
    "plt.xticks(ticks=[-4000, -2000, 0, 2000, 4000], labels=[4000, 2000, 0, 2000, 4000])\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1425eb42-97dd-4dfb-8521-bf1b42f019cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| label: fig7cell\n",
    "\n",
    "# relational age/sex plot with percentages\n",
    "# make percentage column \n",
    "df['Male'] = df['Male'].fillna(0)\n",
    "df['Female'] = df['Female'].fillna(0)\n",
    "ser = df['Female'] / df.loc[:,['Female','Male']].sum(axis=1) *100\n",
    "try:\n",
    "    df.insert(7, \"Percentage female\", ser)\n",
    "except ValueError:\n",
    "    pass\n",
    "\n",
    "# Joint distribution of average age of participants and sex as a percentage of participants\n",
    "df_rel = df[~(df['Percentage female'].isnull())].copy()\n",
    "df_rel['Age'] = pd.to_numeric(df_rel['Age'], errors='coerce')\n",
    "df_rel = df_rel.dropna(subset=['Age'])\n",
    "\n",
    "norm = mplc.LogNorm(vmin=10, vmax=300, clip=True)\n",
    "\n",
    "fig = make_subplots(1,1,horizontal_spacing=0,specs=[[{\"secondary_y\": True}]])\n",
    "\n",
    "for step in np.arange(1, 51, 1):\n",
    "    fig.add_trace(\n",
    "        go.Scatter(\n",
    "            visible=False,\n",
    "            name=str(step),\n",
    "            mode='markers',\n",
    "            customdata=df_rel.loc[df_rel['Total participants']>=step,'Total participants'],\n",
    "            hovertemplate=\"Mean age: %{x}<br>Percent female: %{y:.2f}%<br>Number of participants: %{customdata}<extra></extra>\",\n",
    "            marker=dict(\n",
    "                color=norm(df_rel.loc[df_rel['Total participants']>=step,'Total participants']),\n",
    "                colorscale='magma',\n",
    "                cmin=0,\n",
    "                cmax=1,\n",
    "                colorbar=dict(thickness=25,\n",
    "                              len=.9,\n",
    "                              orientation='h',\n",
    "                              tickvals=[norm(10),norm(30),norm(50),norm(100),norm(300)],\n",
    "                              ticktext=['<10',30,50,100,'>300'],\n",
    "                              ticks=\"outside\",\n",
    "                              title=\"Number of participants\",\n",
    "                              tickformatstops=[dict(dtickrange=[0,1])],\n",
    "                              y=-0.24\n",
    "                             )\n",
    "            ),\n",
    "            x=df_rel.loc[df_rel['Total participants']>=step,'Age'],\n",
    "            y=df_rel.loc[df_rel['Total participants']>=step,'Percentage female']),\n",
    "        secondary_y=False\n",
    "    )\n",
    "\n",
    "fig.add_trace(\n",
    "    go.Scatter(visible=False),\n",
    "    secondary_y=True\n",
    ")\n",
    "\n",
    "fig.data[0].visible = True\n",
    "\n",
    "steps = []\n",
    "for i in range(len(fig.data)-1):\n",
    "    step = dict(\n",
    "        method=\"update\",\n",
    "        label=i+1,\n",
    "        args=[{\"visible\": [False] * len(fig.data)}],\n",
    "    )\n",
    "    step[\"args\"][0][\"visible\"][i] = True\n",
    "    steps.append(step)\n",
    "\n",
    "sliders = [dict(\n",
    "    active=0,\n",
    "    currentvalue={\"prefix\": \"Minimum number of participants: \"},\n",
    "    pad={\"t\": 50},\n",
    "    steps=steps,\n",
    "    y=-.12,\n",
    "    len=.95\n",
    ")]\n",
    "\n",
    "fig.update_layout(\n",
    "    sliders=sliders,\n",
    "    height=800,\n",
    "    xaxis=dict(range=[-5,95],title='Average age (years)'),\n",
    "    yaxis=dict(title=\"Percentage of female participants\",range=[-6,106]),\n",
    "    plot_bgcolor='rgba(0.2, 0.2, 0.2, 0.2)',\n",
    "    width=1100,\n",
    ")\n",
    "\n",
    "fig.update_yaxes(title_text=\"Percentage of male participants\", range=[-6,106], tickmode='array', tickvals=[0,20,40,60,80,100], ticktext=[100,80,60,40,20,0], secondary_y=True)\n",
    "\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7518c2d7-2f29-4596-95a7-344af09874fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| label: fig8cell\n",
    "\n",
    "labels = df.columns[8:17].to_list()\n",
    "parents = ['Non-white','Non-white','Non-white','Reported','Non-white','Non-white','Non-white','Reported','','Reported','']\n",
    "ethnum = df[df.columns[8:17]].sum()\n",
    "tot = ethnum.sum()\n",
    "ethnum['Non-white'] = 0\n",
    "ethnum['Reported'] = 0 \n",
    "ethdata = ethnum/tot*100\n",
    "labels.append('Non-white')\n",
    "labels.append('Reported')\n",
    "\n",
    "values = ethdata[labels].to_list()\n",
    "values = [f'{(round(v,3))}' for v in values]\n",
    "textbox = [\"Percent of total: \" + str(x) + \"%\" for x in values]\n",
    "textbox[0] = textbox[0] + \"<br>Percent of reported: 0.49%<br>Total participants: 12\"\n",
    "textbox[1] = textbox[1] + \"<br>Percent of reported: 1.35%<br>Total participants: 33\"\n",
    "textbox[2] = textbox[2] + \"<br>Percent of reported: 2.58%<br>Total participants: 63\"\n",
    "textbox[3] = textbox[3] + \"<br>Percent of reported: 0.98%<br>Total participants: 24\"\n",
    "textbox[4] = textbox[4] + \"<br>Percent of reported: 0.20%<br>Total participants: 5\"\n",
    "textbox[5] = textbox[5] + \"<br>Percent of reported: 0.12%<br>Total participants: 3\"\n",
    "textbox[6] = textbox[6] + \"<br>Percent of reported: 0.08%<br>Total participants: 2\"\n",
    "textbox[7] = textbox[7] + \"<br>Percent of reported: 94.18%<br>Total participants: 2299\"\n",
    "textbox[8] = textbox[8] + \"<br>Total participants: 60114\"\n",
    "\n",
    "fig=go.Figure(go.Treemap(\n",
    "    labels=labels,\n",
    "    parents=parents,\n",
    "    values = values,\n",
    "    hoverinfo='skip',\n",
    "    text=textbox\n",
    "))\n",
    "\n",
    "fig.update_layout(height=500,width=750,margin = dict(t=0, l=0, r=50, b=0))\n",
    "fig.show()\n",
    "\n",
    "ethdata = df[df.columns[8:17]].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d98795e8-85ed-4837-b840-15c2671d490a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| label: fig9cell\n",
    "\n",
    "# grouping by year of publication and summing male and female participants\n",
    "df_grouped = df_full[['Year of Publication','Male','Female']].copy().fillna(0)\n",
    "df_grouped['Male'] = pd.to_numeric(df_grouped['Male'], errors='coerce').fillna(0)\n",
    "df_grouped['Female'] = pd.to_numeric(df_grouped['Female'], errors='coerce').fillna(0)\n",
    "df_ageyear = df_grouped.groupby('Year of Publication').sum()\n",
    "df_ageyear = df_ageyear.loc[df_ageyear.index[0:32]]\n",
    "s1 = df_ageyear.loc[:,['Female','Male']].sum(axis=1)\n",
    "s2 = df_ageyear['Female']\n",
    "s3 = df_ageyear['Female'] / df_ageyear.loc[:,['Female','Male']].sum(axis=1) *100\n",
    "df_ageyear['Male'] = s1\n",
    "df_ageyear['Female'] = s2\n",
    "\n",
    "\n",
    "ax1 = sns.set_style(style=None, rc=None)\n",
    "fig, ax1 = plt.subplots(figsize=(12,6))\n",
    "\n",
    "b1 = sns.barplot(x='Year of Publication', y='Male', data=df_ageyear, color='darkblue', ax=ax1)\n",
    "b2 = sns.barplot(x='Year of Publication', y='Female', data=df_ageyear, color='lightblue', ax=ax1)\n",
    "top = mpatches.Patch(color='darkblue', label='Male participants')\n",
    "bottom = mpatches.Patch(color='lightblue', label='Female participants')\n",
    "plt.xticks(rotation=-60)\n",
    "plt.ylabel(\"Number of participants\")\n",
    "ax1.grid(False)\n",
    "ax1.set_ylim(0,5000)\n",
    "ax1.set_yticks([0,500,1000,1500,2000,2500,3000,3500,4000,4500,5000])\n",
    "\n",
    "df_ageyear['Percent Female'] = s3\n",
    "\n",
    "ax2 = plt.twinx()\n",
    "rl = sns.lineplot(x=np.arange(32), y='Percent Female', ax=ax2, marker='o', color='firebrick', data=df_ageyear, label='Percent female',legend=False)\n",
    "\n",
    "ax2.set_ylabel('Percent female')\n",
    "ax2.set_ylim(0,100)\n",
    "ax2.set_yticks([0,10,20,30,40,50,60,70,80,90,100])\n",
    "\n",
    "ax2.grid(which='minor')\n",
    "\n",
    "handle, label = ax2.get_legend_handles_labels()\n",
    "plt.legend(handles=[top,bottom,handle[0]],loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35a40e4a-bce7-4943-9569-6c16f880e231",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| label: fig10cell\n",
    "\n",
    "filtage = filtered[['Geographical location','Average Age (Years)']].copy().dropna()\n",
    "filtage['Average Age (Years)'] = pd.to_numeric(filtage['Average Age (Years)'], errors='coerce').dropna()\n",
    "filtage = filtage.dropna()\n",
    "\n",
    "colors = plotly.colors.PLOTLY_SCALES[\"Viridis\"]\n",
    "\n",
    "fig = go.Figure()\n",
    "\n",
    "for i in range(0,3):\n",
    "    fig.add_trace(go.Violin(y=filtage['Geographical location'][(filtage['Geographical location'] == pd.unique(filtage['Geographical location'])[i])],\n",
    "                            x=filtage['Average Age (Years)'][(filtage['Geographical location'] == pd.unique(filtage['Geographical location'])[i])],\n",
    "                            side='positive',\n",
    "                            pointpos=-.3,\n",
    "                            line_color='black',\n",
    "                            orientation='h',\n",
    "                            fillcolor=colors[i*4+4][1],\n",
    "                            marker=dict(size=3),\n",
    "                            spanmode='hard',\n",
    "                            box=dict(width=.5),\n",
    "                            showlegend=False,\n",
    "                            width=.9)\n",
    "             )\n",
    "\n",
    "fig.update_traces(box_visible=True,\n",
    "                  points='all', \n",
    "                  jitter=0.1,  \n",
    "                  scalemode='count')\n",
    "fig.update_layout(\n",
    "    height=500,\n",
    "    width=750,\n",
    "    xaxis=dict(title='Average age (years)'),\n",
    "    plot_bgcolor='rgba(0.2, 0.2, 0.2, 0.2)',\n",
    "    violingap=0, violingroupgap=0, violinmode='overlay')\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52c141a1-1834-4246-b4b3-5f6110a8d65c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| label: fig11cell\n",
    "\n",
    "df_paths2 = df_paths.loc[df_paths['Average age'].notna()]\n",
    "\n",
    "colors = plotly.colors.PLOTLY_SCALES[\"Viridis\"]\n",
    "\n",
    "fig = go.Figure()\n",
    "\n",
    "for i in range(0,len(pd.unique(df_paths2['Pathology']))):\n",
    "    fig.add_trace(go.Violin(y=df_paths2['Pathology'][(df_paths2['Pathology'] == pd.unique(df_paths2['Pathology'])[i])],\n",
    "                            x=df_paths2['Average age'][(df_paths2['Pathology'] == pd.unique(df_paths2['Pathology'])[i])],\n",
    "                            side='positive',\n",
    "                            pointpos=-.3,\n",
    "                            line_color='black',\n",
    "                            orientation='h',\n",
    "                            fillcolor=colors[i][1],\n",
    "                            marker=dict(size=3),\n",
    "                            spanmode='hard',\n",
    "                            box=dict(width=.5),\n",
    "                            showlegend=False)\n",
    "             )\n",
    "\n",
    "fig.update_traces(box_visible=True,\n",
    "                  points='all', \n",
    "                  jitter=0.1,  \n",
    "                  scalemode='count')\n",
    "fig.update_layout(\n",
    "    height=1000, width=1000,\n",
    "    xaxis=dict(title='Average age (years)'),\n",
    "    plot_bgcolor='rgba(0.2, 0.2, 0.2, 0.2)',\n",
    "    violingap=0, violingroupgap=0, violinmode='overlay')\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e57012e7-0b00-4c59-ae0a-13a4070791aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| label: fig12cell\n",
    "\n",
    "df_full['Unreported sex'] = df_full.loc[:,'Total participants'] - df_full.loc[:,['Male','Female']].sum(axis=1)\n",
    "df_full['Reported sex'] = df_full.loc[:,['Male','Female']].sum(axis=1)\n",
    "\n",
    "df_full.loc[df_full['Average age'].notna(),'Unreported age'] = 0\n",
    "df_full.loc[df_full['Average age'].isna(),'Unreported age'] = df_full.loc[df_full['Average age'].isna(),'Total participants']\n",
    "df_full.loc[df_full['Average age'].notna(),'Reported age'] = df_full.loc[df_full['Average age'].notna(),'Total participants']\n",
    "df_full.loc[df_full['Average age'].isna(),'Reported age'] = 0\n",
    "\n",
    "df_full.loc[df_full['Ethnicity']!='Ethnic distribution not specified','Unreported ethnicity'] = 0\n",
    "df_full.loc[df_full['Ethnicity']!='Ethnic distribution not specified','Reported ethnicity'] = df_full.loc[df_full['Ethnicity']!='Ethnic distribution not specified','Total participants']\n",
    "df_full.loc[df_full['Ethnicity']=='Ethnic distribution not specified','Unreported ethnicity'] = df_full.loc[df_full['Ethnicity']=='Ethnic distribution not specified','Total participants']\n",
    "df_full.loc[df_full['Ethnicity']=='Ethnic distribution not specified','Reported ethnicity'] = 0\n",
    "\n",
    "\n",
    "df_grouped = df_full[['Year of Publication','Unreported sex','Reported sex','Unreported age','Reported age','Unreported ethnicity','Reported ethnicity','Total participants']].copy().fillna(0)\n",
    "dfrep = df_grouped.groupby('Year of Publication').sum()\n",
    "dfrep = dfrep.loc[dfrep.index[0:32]]\n",
    "s1 = dfrep['Total participants']\n",
    "s2 = dfrep['Reported sex'] / dfrep['Total participants'] * 100\n",
    "s3 = dfrep['Reported age'] / dfrep['Total participants'] * 100\n",
    "s4 = dfrep['Reported ethnicity'] / dfrep['Total participants'] * 100\n",
    "\n",
    "plt.style.use('ggplot')\n",
    "ax1 = sns.set_style(style=None, rc=None)\n",
    "fig, ax1 = plt.subplots(figsize=(12,6))\n",
    "\n",
    "\n",
    "sns.set_palette(\"colorblind\")\n",
    "\n",
    "b1 = sns.barplot(x='Year of Publication', y='Total participants', data=dfrep, color='darkblue', ax=ax1)\n",
    "top = mpatches.Patch(color='darkblue', label='Total participants')\n",
    "plt.xticks(rotation=-60)\n",
    "plt.ylabel(\"Number of participants\")\n",
    "ax1.grid(False)\n",
    "\n",
    "dfrep['Percent sex reported'] = s2\n",
    "dfrep['Percent age reported'] = s3\n",
    "dfrep['Percent ethnicity reported'] = s4\n",
    "\n",
    "ax2 = plt.twinx()\n",
    "rl = sns.lineplot(x=np.arange(32), y='Percent sex reported', ax=ax2, marker='o', data=dfrep,  label='Percent sex reported',legend=False)\n",
    "r2 = sns.lineplot(x=np.arange(32), y='Percent age reported', ax=ax2, marker='o', data=dfrep,  label='Percent age reported',legend=False)\n",
    "r3 = sns.lineplot(x=np.arange(32), y='Percent ethnicity reported', ax=ax2, marker='o', data=dfrep, label='Percent ethnicity reported',legend=False)\n",
    "\n",
    "ax2.set_ylabel('Percent reported')\n",
    "ax2.set_ylim(0,100)\n",
    "ax2.set_yticks([0,10,20,30,40,50,60,70,80,90,100])\n",
    "ax2.grid(True)\n",
    "\n",
    "handle, label = ax2.get_legend_handles_labels()\n",
    "plt.legend(handles=[top,handle[0],handle[1],handle[2]],loc='center left',bbox_to_anchor=(0,.35))\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
