preprocessing_onlin-v3_mgf.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0c60b9c7-bba8-4e7f-91d8-7935ef85da08",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from tqdm import tqdm\n",
    "import tensorflow as tf\n",
    "import json\n",
    "import tensorflow_text as text\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "from matchms import set_matchms_logger_level\n",
    "import pandas as pd\n",
    "set_matchms_logger_level(\"ERROR\")\n",
    "from matchms.filtering import add_losses\n",
    "from matchms.filtering import add_parent_mass\n",
    "from matchms.filtering import default_filters\n",
    "from matchms.filtering import normalize_intensities\n",
    "from matchms.filtering import select_by_intensity\n",
    "from matchms.filtering import reduce_to_number_of_peaks\n",
    "from matchms.filtering import require_minimum_number_of_peaks\n",
    "from matchms.filtering import select_by_mz\n",
    "from matchms.importing import load_from_mgf\n",
    "from matchms.exporting import save_as_mgf\n",
    "from matchms.importing import load_from_msp\n",
    "\n",
    "\n",
    "from matchms.filtering import repair_inchi_inchikey_smiles\n",
    "from matchms.filtering import derive_inchikey_from_inchi\n",
    "from matchms.filtering import derive_smiles_from_inchi\n",
    "from matchms.filtering import derive_inchi_from_smiles\n",
    "from matchms.filtering import harmonize_undefined_inchi\n",
    "from matchms.filtering import harmonize_undefined_inchikey\n",
    "from matchms.filtering import harmonize_undefined_smiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "30ed8f9b-215b-4fac-8d09-64317fb965e3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python 3.9.15\n"
     ]
    }
   ],
   "source": [
    "!python -V"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1a88907a-9f26-4877-9356-1fe77bfed534",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 7.12 s\n",
      "Wall time: 7.25 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "#from spec2vec.model_building import train_new_word2vec_model\n",
    "\n",
    "def spectrum_processing(s):\n",
    "    \"\"\"This is how one would typically design a desired pre- and post-\n",
    "    processing pipeline.\"\"\"\n",
    "    s = default_filters(s)\n",
    "    s = add_parent_mass(s)\n",
    "    s = normalize_intensities(s)\n",
    "    s = select_by_intensity(s, intensity_from=0.01)\n",
    "    s = reduce_to_number_of_peaks(s, n_required=5, n_max=250)\n",
    "    s = select_by_mz(s, mz_from=15, mz_to=2000)\n",
    "    s = add_losses(s, loss_mz_from=15.0, loss_mz_to=350.0)\n",
    "    s = require_minimum_number_of_peaks(s, n_required=5)\n",
    "    return s\n",
    "\n",
    "\n",
    "\n",
    "def metadata_processing(spectrum):\n",
    "    spectrum = default_filters(spectrum)\n",
    "    spectrum = repair_inchi_inchikey_smiles(spectrum)\n",
    "    spectrum = derive_inchi_from_smiles(spectrum)\n",
    "    spectrum = derive_smiles_from_inchi(spectrum)\n",
    "    spectrum = derive_inchikey_from_inchi(spectrum)\n",
    "    spectrum = harmonize_undefined_smiles(spectrum)\n",
    "    spectrum = harmonize_undefined_inchi(spectrum)\n",
    "    spectrum = harmonize_undefined_inchikey(spectrum)\n",
    "    return spectrum\n",
    "# Load data from MGF file and apply filters\n",
    "\n",
    "import os\n",
    "from matchms.importing import load_from_mgf\n",
    "path_data = \"C:/Users/delser/mass2smiles\"  # enter path to downloaded mgf file\n",
    "file_mgf = os.path.join(path_data, \n",
    "                        \"casmi_candidates_pos_casmi_id.mgf\")\n",
    "spectrums = list(load_from_mgf(file_mgf))\n",
    "\n",
    "spectrums = [metadata_processing(s) for s in spectrums]\n",
    "spectrums = [spectrum_processing(s) for s in spectrums]\n",
    "#spectrums = [spectrum_processing(s) for s in load_from_mgf(\"/Users/delser/Desktop/PhD/Phytochemistry/NP-Databases/CFM-4_DB/TOTAL_COMPOUNDS_DB.energies_merged_name.mgf\")]\n",
    "#spectrums = [spectrum_processing(s) for s in load_from_mgf(\"/Users/delser/Desktop/PhD/Phytochemistry/FBMN/alltissues/altissues15072021-py.mgf\")]\n",
    "# Omit spectrums that didn't qualify for analysis\n",
    "spectrums = [s for s in spectrums if s is not None]\n",
    "\n",
    "\n",
    "# Create spectrum documents\n",
    "#reference_documents = [SpectrumDocument(s, n_decimals=2) for s in spectrums]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "89dca0b7-9f27-441c-95b4-3f074766ead8",
   "metadata": {},
   "outputs": [],
   "source": [
    "precs = []\n",
    "IDs = []\n",
    "mzs=[]\n",
    "ints=[]\n",
    "loss_mzs=[]\n",
    "loss_ints=[]\n",
    "\n",
    "\n",
    "for spec in spectrums: \n",
    "    IDs.append(spec.get(\"feature_id\"))\n",
    "    precs.append(spec.get(\"precursor_mz\"))\n",
    "    mzs.append(list(spec.peaks.mz))\n",
    "    ints.append(list(spec.peaks.intensities))\n",
    "    loss_mzs.append(list(spec.losses.mz))\n",
    "    loss_ints.append(list(spec.losses.intensities))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "4321c672-2b43-41fc-930d-3665db0c2608",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{56.604: 0.020625,\n",
       " 57.0337: 0.03625,\n",
       " 61.0286: 0.10625,\n",
       " 69.0336: 0.51875,\n",
       " 71.0492: 0.2,\n",
       " 74.5668: 0.018125,\n",
       " 81.0336: 0.075,\n",
       " 83.0491: 0.3,\n",
       " 85.0283: 0.425,\n",
       " 87.044: 0.1625,\n",
       " 95.0492: 0.024375,\n",
       " 97.0286: 0.125,\n",
       " 99.0442: 0.034375,\n",
       " 109.0281: 0.0325,\n",
       " 111.0441: 0.1625,\n",
       " 115.0384: 0.02625,\n",
       " 127.0392: 0.026875,\n",
       " 129.0546: 0.1125,\n",
       " 232.1556: 0.020625,\n",
       " 299.0545: 1.0,\n",
       " 299.0909: 0.036875,\n",
       " 311.0572: 0.01875,\n",
       " 355.1155: 0.15,\n",
       " 391.0793: 0.0225,\n",
       " 407.7533: 0.02,\n",
       " 498.4384: 0.02125,\n",
       " 206.79970000000003: 0.02125,\n",
       " 297.4848: 0.02,\n",
       " 314.15880000000004: 0.0225}"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = dict(zip(mzs[3]+loss_mzs[3], ints[3]+loss_ints[3]))\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "5cf3fe57-c57d-4913-8e1b-d1da1696486b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{56.604: 0.020625,\n",
       " 57.0337: 0.03625,\n",
       " 61.0286: 0.10625,\n",
       " 69.0336: 0.51875,\n",
       " 71.0492: 0.2,\n",
       " 74.5668: 0.018125,\n",
       " 81.0336: 0.075,\n",
       " 83.0491: 0.3,\n",
       " 85.0283: 0.425,\n",
       " 87.044: 0.1625,\n",
       " 95.0492: 0.024375,\n",
       " 97.0286: 0.125,\n",
       " 99.0442: 0.034375,\n",
       " 109.0281: 0.0325,\n",
       " 111.0441: 0.1625,\n",
       " 115.0384: 0.02625,\n",
       " 127.0392: 0.026875,\n",
       " 129.0546: 0.1125,\n",
       " 206.79970000000003: 0.02125,\n",
       " 232.1556: 0.020625,\n",
       " 297.4848: 0.02,\n",
       " 299.0545: 1.0,\n",
       " 299.0909: 0.036875,\n",
       " 311.0572: 0.01875,\n",
       " 314.15880000000004: 0.0225,\n",
       " 355.1155: 0.15,\n",
       " 391.0793: 0.0225,\n",
       " 407.7533: 0.02,\n",
       " 498.4384: 0.02125}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res=dict(sorted(res.items()))\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "777cd1bc-3b0e-4c29-a327-2c1fd755e258",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(list(res.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "7fe78582-fb81-4480-aea3-850216adf45c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_values([0.020625, 0.03625, 0.10625, 0.51875, 0.2, 0.018125, 0.075, 0.3, 0.425, 0.1625, 0.024375, 0.125, 0.034375, 0.0325, 0.1625, 0.02625, 0.026875, 0.1125, 0.02125, 0.020625, 0.02, 1.0, 0.036875, 0.01875, 0.0225, 0.15, 0.0225, 0.02, 0.02125])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res.values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2d4707a7-6f95-4fd3-89bc-23ac60ab019a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature_id</th>\n",
       "      <th>precursor_mz</th>\n",
       "      <th>mzs</th>\n",
       "      <th>intensities</th>\n",
       "      <th>loss_mzs</th>\n",
       "      <th>loss_intensities</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>398</td>\n",
       "      <td>235.1691</td>\n",
       "      <td>[53.0389, 53.0509, 55.0181, 55.0545, 55.5835, ...</td>\n",
       "      <td>[0.07166666666666667, 0.020833333333333332, 0....</td>\n",
       "      <td>[19.8476, 19.87299999999999, 56.06339999999997...</td>\n",
       "      <td>[0.075, 0.07083333333333333, 0.031666666666666...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>398</td>\n",
       "      <td>235.1691</td>\n",
       "      <td>[53.0389, 53.0509, 55.0181, 55.0545, 55.5835, ...</td>\n",
       "      <td>[0.07166666666666667, 0.020833333333333332, 0....</td>\n",
       "      <td>[19.8476, 19.87299999999999, 56.06339999999997...</td>\n",
       "      <td>[0.075, 0.07083333333333333, 0.031666666666666...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>159</td>\n",
       "      <td>485.2164</td>\n",
       "      <td>[66.3952, 68.416, 81.3048, 100.9429, 121.0081,...</td>\n",
       "      <td>[0.05, 0.04565217391304348, 0.0521739130434782...</td>\n",
       "      <td>[18.01230000000004, 96.17940000000004, 102.068...</td>\n",
       "      <td>[1.0, 0.058695652173913045, 0.0630434782608695...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>169</td>\n",
       "      <td>705.2381</td>\n",
       "      <td>[56.604, 57.0337, 61.0286, 69.0336, 71.0492, 7...</td>\n",
       "      <td>[0.020625, 0.03625, 0.10625, 0.51875, 0.2, 0.0...</td>\n",
       "      <td>[206.79970000000003, 297.4848, 314.15880000000...</td>\n",
       "      <td>[0.02125, 0.02, 0.0225]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>423</td>\n",
       "      <td>441.2264</td>\n",
       "      <td>[62.099, 62.3016, 65.6795, 71.5638, 95.0857, 1...</td>\n",
       "      <td>[0.07096774193548387, 0.08709677419354839, 0.0...</td>\n",
       "      <td>[70.07659999999998, 88.08980000000003, 130.173...</td>\n",
       "      <td>[1.0, 0.14516129032258066, 0.12903225806451613...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>231</th>\n",
       "      <td>439</td>\n",
       "      <td>336.0647</td>\n",
       "      <td>[55.173, 55.6729, 57.7501, 63.2992, 64.3001, 7...</td>\n",
       "      <td>[0.010476190476190476, 0.010952380952380953, 0...</td>\n",
       "      <td>[17.859800000000007, 72.62130000000002, 119.40...</td>\n",
       "      <td>[0.04428571428571428, 0.010952380952380953, 0....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>232</th>\n",
       "      <td>322</td>\n",
       "      <td>532.3113</td>\n",
       "      <td>[53.6431, 61.0821, 67.0543, 69.0334, 81.0699, ...</td>\n",
       "      <td>[0.0125, 0.012, 0.016, 0.08, 0.23, 0.285, 0.07...</td>\n",
       "      <td>[197.09179999999998, 207.04439999999994, 215.1...</td>\n",
       "      <td>[0.0245, 0.015, 0.205, 0.135, 0.28, 0.0205, 0....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>233</th>\n",
       "      <td>300</td>\n",
       "      <td>478.3159</td>\n",
       "      <td>[63.7647, 71.5594, 93.5641, 108.7938, 119.8244...</td>\n",
       "      <td>[0.10476190476190476, 0.1, 0.11428571428571428...</td>\n",
       "      <td>[16.459400000000016, 35.03750000000002, 53.050...</td>\n",
       "      <td>[0.10952380952380952, 0.13333333333333333, 0.1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>234</th>\n",
       "      <td>254</td>\n",
       "      <td>329.2320</td>\n",
       "      <td>[53.0389, 55.0182, 55.0546, 67.008, 67.0544, 6...</td>\n",
       "      <td>[0.0456140350877193, 0.05087719298245614, 0.28...</td>\n",
       "      <td>[89.70510000000002, 112.67550000000003, 145.15...</td>\n",
       "      <td>[0.042105263157894736, 0.2631578947368421, 0.0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235</th>\n",
       "      <td>237</td>\n",
       "      <td>278.1172</td>\n",
       "      <td>[50.8153, 74.535, 78.0275, 93.2556, 97.6804, 1...</td>\n",
       "      <td>[0.01818181818181818, 0.01818181818181818, 0.0...</td>\n",
       "      <td>[67.65480000000002, 83.67750000000004, 86.4763...</td>\n",
       "      <td>[0.02, 0.019090909090909092, 0.018181818181818...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>236 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    feature_id  precursor_mz  \\\n",
       "0          398      235.1691   \n",
       "1          398      235.1691   \n",
       "2          159      485.2164   \n",
       "3          169      705.2381   \n",
       "4          423      441.2264   \n",
       "..         ...           ...   \n",
       "231        439      336.0647   \n",
       "232        322      532.3113   \n",
       "233        300      478.3159   \n",
       "234        254      329.2320   \n",
       "235        237      278.1172   \n",
       "\n",
       "                                                   mzs  \\\n",
       "0    [53.0389, 53.0509, 55.0181, 55.0545, 55.5835, ...   \n",
       "1    [53.0389, 53.0509, 55.0181, 55.0545, 55.5835, ...   \n",
       "2    [66.3952, 68.416, 81.3048, 100.9429, 121.0081,...   \n",
       "3    [56.604, 57.0337, 61.0286, 69.0336, 71.0492, 7...   \n",
       "4    [62.099, 62.3016, 65.6795, 71.5638, 95.0857, 1...   \n",
       "..                                                 ...   \n",
       "231  [55.173, 55.6729, 57.7501, 63.2992, 64.3001, 7...   \n",
       "232  [53.6431, 61.0821, 67.0543, 69.0334, 81.0699, ...   \n",
       "233  [63.7647, 71.5594, 93.5641, 108.7938, 119.8244...   \n",
       "234  [53.0389, 55.0182, 55.0546, 67.008, 67.0544, 6...   \n",
       "235  [50.8153, 74.535, 78.0275, 93.2556, 97.6804, 1...   \n",
       "\n",
       "                                           intensities  \\\n",
       "0    [0.07166666666666667, 0.020833333333333332, 0....   \n",
       "1    [0.07166666666666667, 0.020833333333333332, 0....   \n",
       "2    [0.05, 0.04565217391304348, 0.0521739130434782...   \n",
       "3    [0.020625, 0.03625, 0.10625, 0.51875, 0.2, 0.0...   \n",
       "4    [0.07096774193548387, 0.08709677419354839, 0.0...   \n",
       "..                                                 ...   \n",
       "231  [0.010476190476190476, 0.010952380952380953, 0...   \n",
       "232  [0.0125, 0.012, 0.016, 0.08, 0.23, 0.285, 0.07...   \n",
       "233  [0.10476190476190476, 0.1, 0.11428571428571428...   \n",
       "234  [0.0456140350877193, 0.05087719298245614, 0.28...   \n",
       "235  [0.01818181818181818, 0.01818181818181818, 0.0...   \n",
       "\n",
       "                                              loss_mzs  \\\n",
       "0    [19.8476, 19.87299999999999, 56.06339999999997...   \n",
       "1    [19.8476, 19.87299999999999, 56.06339999999997...   \n",
       "2    [18.01230000000004, 96.17940000000004, 102.068...   \n",
       "3    [206.79970000000003, 297.4848, 314.15880000000...   \n",
       "4    [70.07659999999998, 88.08980000000003, 130.173...   \n",
       "..                                                 ...   \n",
       "231  [17.859800000000007, 72.62130000000002, 119.40...   \n",
       "232  [197.09179999999998, 207.04439999999994, 215.1...   \n",
       "233  [16.459400000000016, 35.03750000000002, 53.050...   \n",
       "234  [89.70510000000002, 112.67550000000003, 145.15...   \n",
       "235  [67.65480000000002, 83.67750000000004, 86.4763...   \n",
       "\n",
       "                                      loss_intensities  \n",
       "0    [0.075, 0.07083333333333333, 0.031666666666666...  \n",
       "1    [0.075, 0.07083333333333333, 0.031666666666666...  \n",
       "2    [1.0, 0.058695652173913045, 0.0630434782608695...  \n",
       "3                              [0.02125, 0.02, 0.0225]  \n",
       "4    [1.0, 0.14516129032258066, 0.12903225806451613...  \n",
       "..                                                 ...  \n",
       "231  [0.04428571428571428, 0.010952380952380953, 0....  \n",
       "232  [0.0245, 0.015, 0.205, 0.135, 0.28, 0.0205, 0....  \n",
       "233  [0.10952380952380952, 0.13333333333333333, 0.1...  \n",
       "234  [0.042105263157894736, 0.2631578947368421, 0.0...  \n",
       "235  [0.02, 0.019090909090909092, 0.018181818181818...  \n",
       "\n",
       "[236 rows x 6 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metadata = pd.DataFrame(list(zip(IDs, precs,mzs,ints,loss_mzs,loss_ints)), columns=[\"feature_id\", \"precursor_mz\",\"mzs\",\"intensities\",\"loss_mzs\",\"loss_intensities\" ])\n",
    "metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "538b75bb-ec4d-4bdd-82f6-687a89cd5276",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature_id</th>\n",
       "      <th>precursor_mz</th>\n",
       "      <th>mzs</th>\n",
       "      <th>intensities</th>\n",
       "      <th>loss_mzs</th>\n",
       "      <th>loss_intensities</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>398</td>\n",
       "      <td>235.1691</td>\n",
       "      <td>[53.0389, 53.0509, 55.0181, 55.0545, 55.5835, ...</td>\n",
       "      <td>[0.07166666666666667, 0.020833333333333332, 0....</td>\n",
       "      <td>[19.8476, 19.87299999999999, 56.06339999999997...</td>\n",
       "      <td>[0.075, 0.07083333333333333, 0.031666666666666...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>398</td>\n",
       "      <td>235.1691</td>\n",
       "      <td>[53.0389, 53.0509, 55.0181, 55.0545, 55.5835, ...</td>\n",
       "      <td>[0.07166666666666667, 0.020833333333333332, 0....</td>\n",
       "      <td>[19.8476, 19.87299999999999, 56.06339999999997...</td>\n",
       "      <td>[0.075, 0.07083333333333333, 0.031666666666666...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>159</td>\n",
       "      <td>485.2164</td>\n",
       "      <td>[66.3952, 68.416, 81.3048, 100.9429, 121.0081,...</td>\n",
       "      <td>[0.05, 0.04565217391304348, 0.0521739130434782...</td>\n",
       "      <td>[18.01230000000004, 96.17940000000004, 102.068...</td>\n",
       "      <td>[1.0, 0.058695652173913045, 0.0630434782608695...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>169</td>\n",
       "      <td>705.2381</td>\n",
       "      <td>[56.604, 57.0337, 61.0286, 69.0336, 71.0492, 7...</td>\n",
       "      <td>[0.020625, 0.03625, 0.10625, 0.51875, 0.2, 0.0...</td>\n",
       "      <td>[206.79970000000003, 297.4848, 314.15880000000...</td>\n",
       "      <td>[0.02125, 0.02, 0.0225]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>423</td>\n",
       "      <td>441.2264</td>\n",
       "      <td>[62.099, 62.3016, 65.6795, 71.5638, 95.0857, 1...</td>\n",
       "      <td>[0.07096774193548387, 0.08709677419354839, 0.0...</td>\n",
       "      <td>[70.07659999999998, 88.08980000000003, 130.173...</td>\n",
       "      <td>[1.0, 0.14516129032258066, 0.12903225806451613...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>231</th>\n",
       "      <td>439</td>\n",
       "      <td>336.0647</td>\n",
       "      <td>[55.173, 55.6729, 57.7501, 63.2992, 64.3001, 7...</td>\n",
       "      <td>[0.010476190476190476, 0.010952380952380953, 0...</td>\n",
       "      <td>[17.859800000000007, 72.62130000000002, 119.40...</td>\n",
       "      <td>[0.04428571428571428, 0.010952380952380953, 0....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>232</th>\n",
       "      <td>322</td>\n",
       "      <td>532.3113</td>\n",
       "      <td>[53.6431, 61.0821, 67.0543, 69.0334, 81.0699, ...</td>\n",
       "      <td>[0.0125, 0.012, 0.016, 0.08, 0.23, 0.285, 0.07...</td>\n",
       "      <td>[197.09179999999998, 207.04439999999994, 215.1...</td>\n",
       "      <td>[0.0245, 0.015, 0.205, 0.135, 0.28, 0.0205, 0....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>233</th>\n",
       "      <td>300</td>\n",
       "      <td>478.3159</td>\n",
       "      <td>[63.7647, 71.5594, 93.5641, 108.7938, 119.8244...</td>\n",
       "      <td>[0.10476190476190476, 0.1, 0.11428571428571428...</td>\n",
       "      <td>[16.459400000000016, 35.03750000000002, 53.050...</td>\n",
       "      <td>[0.10952380952380952, 0.13333333333333333, 0.1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>234</th>\n",
       "      <td>254</td>\n",
       "      <td>329.2320</td>\n",
       "      <td>[53.0389, 55.0182, 55.0546, 67.008, 67.0544, 6...</td>\n",
       "      <td>[0.0456140350877193, 0.05087719298245614, 0.28...</td>\n",
       "      <td>[89.70510000000002, 112.67550000000003, 145.15...</td>\n",
       "      <td>[0.042105263157894736, 0.2631578947368421, 0.0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235</th>\n",
       "      <td>237</td>\n",
       "      <td>278.1172</td>\n",
       "      <td>[50.8153, 74.535, 78.0275, 93.2556, 97.6804, 1...</td>\n",
       "      <td>[0.01818181818181818, 0.01818181818181818, 0.0...</td>\n",
       "      <td>[67.65480000000002, 83.67750000000004, 86.4763...</td>\n",
       "      <td>[0.02, 0.019090909090909092, 0.018181818181818...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>236 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    feature_id  precursor_mz  \\\n",
       "0          398      235.1691   \n",
       "1          398      235.1691   \n",
       "2          159      485.2164   \n",
       "3          169      705.2381   \n",
       "4          423      441.2264   \n",
       "..         ...           ...   \n",
       "231        439      336.0647   \n",
       "232        322      532.3113   \n",
       "233        300      478.3159   \n",
       "234        254      329.2320   \n",
       "235        237      278.1172   \n",
       "\n",
       "                                                   mzs  \\\n",
       "0    [53.0389, 53.0509, 55.0181, 55.0545, 55.5835, ...   \n",
       "1    [53.0389, 53.0509, 55.0181, 55.0545, 55.5835, ...   \n",
       "2    [66.3952, 68.416, 81.3048, 100.9429, 121.0081,...   \n",
       "3    [56.604, 57.0337, 61.0286, 69.0336, 71.0492, 7...   \n",
       "4    [62.099, 62.3016, 65.6795, 71.5638, 95.0857, 1...   \n",
       "..                                                 ...   \n",
       "231  [55.173, 55.6729, 57.7501, 63.2992, 64.3001, 7...   \n",
       "232  [53.6431, 61.0821, 67.0543, 69.0334, 81.0699, ...   \n",
       "233  [63.7647, 71.5594, 93.5641, 108.7938, 119.8244...   \n",
       "234  [53.0389, 55.0182, 55.0546, 67.008, 67.0544, 6...   \n",
       "235  [50.8153, 74.535, 78.0275, 93.2556, 97.6804, 1...   \n",
       "\n",
       "                                           intensities  \\\n",
       "0    [0.07166666666666667, 0.020833333333333332, 0....   \n",
       "1    [0.07166666666666667, 0.020833333333333332, 0....   \n",
       "2    [0.05, 0.04565217391304348, 0.0521739130434782...   \n",
       "3    [0.020625, 0.03625, 0.10625, 0.51875, 0.2, 0.0...   \n",
       "4    [0.07096774193548387, 0.08709677419354839, 0.0...   \n",
       "..                                                 ...   \n",
       "231  [0.010476190476190476, 0.010952380952380953, 0...   \n",
       "232  [0.0125, 0.012, 0.016, 0.08, 0.23, 0.285, 0.07...   \n",
       "233  [0.10476190476190476, 0.1, 0.11428571428571428...   \n",
       "234  [0.0456140350877193, 0.05087719298245614, 0.28...   \n",
       "235  [0.01818181818181818, 0.01818181818181818, 0.0...   \n",
       "\n",
       "                                              loss_mzs  \\\n",
       "0    [19.8476, 19.87299999999999, 56.06339999999997...   \n",
       "1    [19.8476, 19.87299999999999, 56.06339999999997...   \n",
       "2    [18.01230000000004, 96.17940000000004, 102.068...   \n",
       "3    [206.79970000000003, 297.4848, 314.15880000000...   \n",
       "4    [70.07659999999998, 88.08980000000003, 130.173...   \n",
       "..                                                 ...   \n",
       "231  [17.859800000000007, 72.62130000000002, 119.40...   \n",
       "232  [197.09179999999998, 207.04439999999994, 215.1...   \n",
       "233  [16.459400000000016, 35.03750000000002, 53.050...   \n",
       "234  [89.70510000000002, 112.67550000000003, 145.15...   \n",
       "235  [67.65480000000002, 83.67750000000004, 86.4763...   \n",
       "\n",
       "                                      loss_intensities  \n",
       "0    [0.075, 0.07083333333333333, 0.031666666666666...  \n",
       "1    [0.075, 0.07083333333333333, 0.031666666666666...  \n",
       "2    [1.0, 0.058695652173913045, 0.0630434782608695...  \n",
       "3                              [0.02125, 0.02, 0.0225]  \n",
       "4    [1.0, 0.14516129032258066, 0.12903225806451613...  \n",
       "..                                                 ...  \n",
       "231  [0.04428571428571428, 0.010952380952380953, 0....  \n",
       "232  [0.0245, 0.015, 0.205, 0.135, 0.28, 0.0205, 0....  \n",
       "233  [0.10952380952380952, 0.13333333333333333, 0.1...  \n",
       "234  [0.042105263157894736, 0.2631578947368421, 0.0...  \n",
       "235  [0.02, 0.019090909090909092, 0.018181818181818...  \n",
       "\n",
       "[236 rows x 6 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train=metadata.dropna()\n",
    "df_train"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17d5d4e0-789e-415e-98c0-7b357c3262bc",
   "metadata": {},
   "source": [
    "df_train=df_train.loc[df_train.loss_mzs.apply(str) != \"[]\"]\n",
    "df_train"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9428407-e8b2-450e-82ca-2d7a5541cdf7",
   "metadata": {},
   "source": [
    "df_wrong=metadata.loc[set(metadata.index) - set(df_train.index.values.tolist())]\n",
    "df_wrong"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ac599deb-d5cc-4bec-824b-52a3ae976d8a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.to_csv('casmi_ids.tsv',sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "c9d0f5a9-2e05-4143-bf21-a62d075b273b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>spectrum_id</th>\n",
       "      <th>precursor_mz</th>\n",
       "      <th>mzs</th>\n",
       "      <th>intensities</th>\n",
       "      <th>loss_mzs</th>\n",
       "      <th>loss_intensities</th>\n",
       "      <th>smiles_preprocessed</th>\n",
       "      <th>num_of_sugars</th>\n",
       "      <th>Number of aliphatic carboxylic acids</th>\n",
       "      <th>...</th>\n",
       "      <th>Number of phenols</th>\n",
       "      <th>Number of phosphoric acid groups</th>\n",
       "      <th>Number of phosphoric ester groups</th>\n",
       "      <th>Number of piperdine rings</th>\n",
       "      <th>Number of primary amides</th>\n",
       "      <th>Number of pyridine rings</th>\n",
       "      <th>Number of quaternary nitrogens</th>\n",
       "      <th>Number of thioether</th>\n",
       "      <th>Number of thiazole rings</th>\n",
       "      <th>Number of unbranched alkanes of at least 4 members (excludes halogenated alkanes)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>719.2538</td>\n",
       "      <td>[53.0387, 55.0179, 55.0545, 57.0338, 59.0492, ...</td>\n",
       "      <td>[0.017272727272727273, 0.01818181818181818, 0....</td>\n",
       "      <td>[314.15709999999996]</td>\n",
       "      <td>[0.01818181818181818]</td>\n",
       "      <td>CC1C(C(C(C(O1)OC2=C(OC3=C(C(=CC(=C3C2=O)O)O)CC...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>499.2298</td>\n",
       "      <td>[67.0543, 69.0698, 81.0699, 83.0492, 83.0853, ...</td>\n",
       "      <td>[0.010357142857142856, 0.04642857142857143, 0....</td>\n",
       "      <td>[60.02260000000001, 118.02660000000003, 160.03...</td>\n",
       "      <td>[0.060714285714285714, 0.39285714285714285, 0....</td>\n",
       "      <td>CC1CC2(C(C1O)C=C(C(CC3C(C3(C)C)C=C(C2=O)C)OC(=...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>1102.5777</td>\n",
       "      <td>[81.0334, 83.0491, 85.0224, 85.0284, 85.0333, ...</td>\n",
       "      <td>[0.14210526315789473, 0.2236842105263158, 0.01...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>CC1C(C(C(C(O1)OC2C(OC(C(C2O)O)OCC3C(C(C(C(O3)O...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>472.2082</td>\n",
       "      <td>[72.2736, 145.0759, 148.0868, 149.0707, 172.06...</td>\n",
       "      <td>[0.010697674418604652, 0.03488372093023256, 0....</td>\n",
       "      <td>[135.07919999999996, 148.0632, 165.08959999999...</td>\n",
       "      <td>[0.053488372093023255, 0.10465116279069768, 0....</td>\n",
       "      <td>CN1C2=C(C=C(C=C2)C(=O)N(CCC(=O)O)C3=CC=CC=N3)N...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>657.3116</td>\n",
       "      <td>[55.0542, 55.9818, 57.0334, 60.2456, 69.0335, ...</td>\n",
       "      <td>[0.018260869565217393, 0.013043478260869565, 0...</td>\n",
       "      <td>[158.094, 210.1282, 228.1352, 246.1485, 280.16...</td>\n",
       "      <td>[0.02217391304347826, 0.06086956521739131, 0.1...</td>\n",
       "      <td>CCCC(=O)OCC(C(C(CN1C2=C(C=C(C(=C2)C)C)N=C3C1=N...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237</th>\n",
       "      <td>237</td>\n",
       "      <td>490</td>\n",
       "      <td>268.1541</td>\n",
       "      <td>[86.0599, 109.065, 109.1013, 121.101, 123.1168...</td>\n",
       "      <td>[0.10689655172413794, 0.04482758620689655, 0.0...</td>\n",
       "      <td>[18.01030000000003, 36.02120000000002, 46.0054...</td>\n",
       "      <td>[0.21724137931034482, 0.06206896551724138, 1.0...</td>\n",
       "      <td>CC1CCC2C1C(=O)OC(C2C)NC(=O)C3C(O3)C</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>238</th>\n",
       "      <td>238</td>\n",
       "      <td>491</td>\n",
       "      <td>411.3254</td>\n",
       "      <td>[55.0544, 57.0701, 67.0544, 69.0699, 81.0699, ...</td>\n",
       "      <td>[0.0215625, 0.053125, 0.015, 0.34375, 0.046875...</td>\n",
       "      <td>[18.01060000000001, 88.08850000000001, 142.135...</td>\n",
       "      <td>[0.153125, 0.02125, 0.01875, 0.02625, 0.028437...</td>\n",
       "      <td>CC(C)C(C)C=CC(C)C1CCC2C1(CCC3=C2C(=O)C=C4C3(CC...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>239</th>\n",
       "      <td>239</td>\n",
       "      <td>492</td>\n",
       "      <td>430.2432</td>\n",
       "      <td>[50.4192, 52.9216, 74.1355, 81.0699, 86.4846, ...</td>\n",
       "      <td>[0.05555555555555555, 0.058333333333333334, 0....</td>\n",
       "      <td>[98.63150000000002, 179.0794, 197.0896, 214.55...</td>\n",
       "      <td>[0.06666666666666667, 0.2777777777777778, 1.0,...</td>\n",
       "      <td>CC1CCC(C2(C1=CC(CC2)C(=C)C(=O)O)C)OC3C(C(C(C(O...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240</th>\n",
       "      <td>240</td>\n",
       "      <td>495</td>\n",
       "      <td>578.2076</td>\n",
       "      <td>[54.5814, 57.0338, 61.0286, 69.0335, 81.0331, ...</td>\n",
       "      <td>[0.01, 0.010769230769230769, 0.038461538461538...</td>\n",
       "      <td>[160.76359999999994]</td>\n",
       "      <td>[0.011923076923076923]</td>\n",
       "      <td>COC1=C2C(=CC(=C1OC3C(C(C(C(O3)CO)O)O)OC4C(C(C(...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241</th>\n",
       "      <td>241</td>\n",
       "      <td>500</td>\n",
       "      <td>243.1014</td>\n",
       "      <td>[71.9563, 104.546, 105.0698, 107.0491, 107.056...</td>\n",
       "      <td>[0.015833333333333335, 0.018333333333333333, 0...</td>\n",
       "      <td>[24.813600000000008, 42.197700000000026, 94.04...</td>\n",
       "      <td>[0.025833333333333333, 0.0175, 0.065, 1.0, 0.6...</td>\n",
       "      <td>C1C(COC2=C1C=CC(=C2)O)C3=CC=C(C=C3)O</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>242 rows × 68 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Unnamed: 0  spectrum_id  precursor_mz  \\\n",
       "0             0            3      719.2538   \n",
       "1             1            5      499.2298   \n",
       "2             2            6     1102.5777   \n",
       "3             3           10      472.2082   \n",
       "4             4           11      657.3116   \n",
       "..          ...          ...           ...   \n",
       "237         237          490      268.1541   \n",
       "238         238          491      411.3254   \n",
       "239         239          492      430.2432   \n",
       "240         240          495      578.2076   \n",
       "241         241          500      243.1014   \n",
       "\n",
       "                                                   mzs  \\\n",
       "0    [53.0387, 55.0179, 55.0545, 57.0338, 59.0492, ...   \n",
       "1    [67.0543, 69.0698, 81.0699, 83.0492, 83.0853, ...   \n",
       "2    [81.0334, 83.0491, 85.0224, 85.0284, 85.0333, ...   \n",
       "3    [72.2736, 145.0759, 148.0868, 149.0707, 172.06...   \n",
       "4    [55.0542, 55.9818, 57.0334, 60.2456, 69.0335, ...   \n",
       "..                                                 ...   \n",
       "237  [86.0599, 109.065, 109.1013, 121.101, 123.1168...   \n",
       "238  [55.0544, 57.0701, 67.0544, 69.0699, 81.0699, ...   \n",
       "239  [50.4192, 52.9216, 74.1355, 81.0699, 86.4846, ...   \n",
       "240  [54.5814, 57.0338, 61.0286, 69.0335, 81.0331, ...   \n",
       "241  [71.9563, 104.546, 105.0698, 107.0491, 107.056...   \n",
       "\n",
       "                                           intensities  \\\n",
       "0    [0.017272727272727273, 0.01818181818181818, 0....   \n",
       "1    [0.010357142857142856, 0.04642857142857143, 0....   \n",
       "2    [0.14210526315789473, 0.2236842105263158, 0.01...   \n",
       "3    [0.010697674418604652, 0.03488372093023256, 0....   \n",
       "4    [0.018260869565217393, 0.013043478260869565, 0...   \n",
       "..                                                 ...   \n",
       "237  [0.10689655172413794, 0.04482758620689655, 0.0...   \n",
       "238  [0.0215625, 0.053125, 0.015, 0.34375, 0.046875...   \n",
       "239  [0.05555555555555555, 0.058333333333333334, 0....   \n",
       "240  [0.01, 0.010769230769230769, 0.038461538461538...   \n",
       "241  [0.015833333333333335, 0.018333333333333333, 0...   \n",
       "\n",
       "                                              loss_mzs  \\\n",
       "0                                 [314.15709999999996]   \n",
       "1    [60.02260000000001, 118.02660000000003, 160.03...   \n",
       "2                                                   []   \n",
       "3    [135.07919999999996, 148.0632, 165.08959999999...   \n",
       "4    [158.094, 210.1282, 228.1352, 246.1485, 280.16...   \n",
       "..                                                 ...   \n",
       "237  [18.01030000000003, 36.02120000000002, 46.0054...   \n",
       "238  [18.01060000000001, 88.08850000000001, 142.135...   \n",
       "239  [98.63150000000002, 179.0794, 197.0896, 214.55...   \n",
       "240                               [160.76359999999994]   \n",
       "241  [24.813600000000008, 42.197700000000026, 94.04...   \n",
       "\n",
       "                                      loss_intensities  \\\n",
       "0                                [0.01818181818181818]   \n",
       "1    [0.060714285714285714, 0.39285714285714285, 0....   \n",
       "2                                                   []   \n",
       "3    [0.053488372093023255, 0.10465116279069768, 0....   \n",
       "4    [0.02217391304347826, 0.06086956521739131, 0.1...   \n",
       "..                                                 ...   \n",
       "237  [0.21724137931034482, 0.06206896551724138, 1.0...   \n",
       "238  [0.153125, 0.02125, 0.01875, 0.02625, 0.028437...   \n",
       "239  [0.06666666666666667, 0.2777777777777778, 1.0,...   \n",
       "240                             [0.011923076923076923]   \n",
       "241  [0.025833333333333333, 0.0175, 0.065, 1.0, 0.6...   \n",
       "\n",
       "                                   smiles_preprocessed  num_of_sugars  \\\n",
       "0    CC1C(C(C(C(O1)OC2=C(OC3=C(C(=CC(=C3C2=O)O)O)CC...            2.0   \n",
       "1    CC1CC2(C(C1O)C=C(C(CC3C(C3(C)C)C=C(C2=O)C)OC(=...            0.0   \n",
       "2    CC1C(C(C(C(O1)OC2C(OC(C(C2O)O)OCC3C(C(C(C(O3)O...            4.0   \n",
       "3    CN1C2=C(C=C(C=C2)C(=O)N(CCC(=O)O)C3=CC=CC=N3)N...            0.0   \n",
       "4    CCCC(=O)OCC(C(C(CN1C2=C(C=C(C(=C2)C)C)N=C3C1=N...            1.0   \n",
       "..                                                 ...            ...   \n",
       "237                CC1CCC2C1C(=O)OC(C2C)NC(=O)C3C(O3)C            0.0   \n",
       "238  CC(C)C(C)C=CC(C)C1CCC2C1(CCC3=C2C(=O)C=C4C3(CC...            0.0   \n",
       "239  CC1CCC(C2(C1=CC(CC2)C(=C)C(=O)O)C)OC3C(C(C(C(O...            1.0   \n",
       "240  COC1=C2C(=CC(=C1OC3C(C(C(C(O3)CO)O)O)OC4C(C(C(...            2.0   \n",
       "241               C1C(COC2=C1C=CC(=C2)O)C3=CC=C(C=C3)O            0.0   \n",
       "\n",
       "     Number of aliphatic carboxylic acids  ...  Number of phenols  \\\n",
       "0                                       0  ...                  2   \n",
       "1                                       1  ...                  0   \n",
       "2                                       0  ...                  0   \n",
       "3                                       1  ...                  0   \n",
       "4                                       0  ...                  0   \n",
       "..                                    ...  ...                ...   \n",
       "237                                     0  ...                  0   \n",
       "238                                     0  ...                  0   \n",
       "239                                     1  ...                  0   \n",
       "240                                     1  ...                  0   \n",
       "241                                     0  ...                  2   \n",
       "\n",
       "     Number of phosphoric acid groups  Number of phosphoric ester groups  \\\n",
       "0                                   0                                  0   \n",
       "1                                   0                                  0   \n",
       "2                                   0                                  0   \n",
       "3                                   0                                  0   \n",
       "4                                   0                                  0   \n",
       "..                                ...                                ...   \n",
       "237                                 0                                  0   \n",
       "238                                 0                                  0   \n",
       "239                                 0                                  0   \n",
       "240                                 0                                  0   \n",
       "241                                 0                                  0   \n",
       "\n",
       "     Number of piperdine rings  Number of primary amides  \\\n",
       "0                            0                         0   \n",
       "1                            0                         0   \n",
       "2                            0                         0   \n",
       "3                            0                         0   \n",
       "4                            0                         0   \n",
       "..                         ...                       ...   \n",
       "237                          0                         0   \n",
       "238                          0                         0   \n",
       "239                          0                         0   \n",
       "240                          0                         0   \n",
       "241                          0                         0   \n",
       "\n",
       "     Number of pyridine rings  Number of quaternary nitrogens  \\\n",
       "0                           0                               0   \n",
       "1                           0                               0   \n",
       "2                           0                               0   \n",
       "3                           1                               0   \n",
       "4                           0                               0   \n",
       "..                        ...                             ...   \n",
       "237                         0                               0   \n",
       "238                         0                               0   \n",
       "239                         0                               0   \n",
       "240                         0                               0   \n",
       "241                         0                               0   \n",
       "\n",
       "     Number of thioether  Number of thiazole rings  \\\n",
       "0                      0                         0   \n",
       "1                      0                         0   \n",
       "2                      0                         0   \n",
       "3                      0                         0   \n",
       "4                      0                         0   \n",
       "..                   ...                       ...   \n",
       "237                    0                         0   \n",
       "238                    0                         0   \n",
       "239                    0                         0   \n",
       "240                    0                         0   \n",
       "241                    0                         0   \n",
       "\n",
       "     Number of unbranched alkanes of at least 4 members (excludes halogenated alkanes)  \n",
       "0                                                    0                                  \n",
       "1                                                    0                                  \n",
       "2                                                    0                                  \n",
       "3                                                    0                                  \n",
       "4                                                    0                                  \n",
       "..                                                 ...                                  \n",
       "237                                                  0                                  \n",
       "238                                                  0                                  \n",
       "239                                                  0                                  \n",
       "240                                                  0                                  \n",
       "241                                                  0                                  \n",
       "\n",
       "[242 rows x 68 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = pd.read_csv(\"casmi_func_groups_2201.tsv\",sep=\"\\t\")\n",
    "#data_df = pd.read_csv(\"/Users/delser/mass2smiles/retrain/nist/all_HRMS_validation_16122022_cddd_refine.tsv\",sep=\"\\t\")\n",
    "df_train=df_train.dropna()\n",
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0d79b609-3512-4b19-950c-be76089d8a96",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_wrong.to_csv('loss_fail_matchms.tsv',sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "62e7f44e-b7fc-4154-a48c-c6f12b02ddb9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def positional_encoding(max_position, d_model, min_freq=1e-6):\n",
    "    position = np.arange(max_position)\n",
    "    freqs = min_freq**(2*(np.arange(d_model)//2)/d_model)\n",
    "    pos_enc = position.reshape(-1,1)*freqs.reshape(1,-1)\n",
    "    pos_enc[:, ::2] = np.cos(pos_enc[:, ::2])\n",
    "    pos_enc[:, 1::2] = np.sin(pos_enc[:, 1::2])\n",
    "    return pos_enc\n",
    "\n",
    "def trun_n_d(n,d):\n",
    "    return (  n if not n.find('.')+1 else n[:n.find('.')+d+1]  )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "23408430-22d9-4190-a930-54743b8a48ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "P=positional_encoding(200000,256, min_freq=1e2)\n",
    "#np.save('positions_512_1e2.npy',P)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "84cb2056-f65f-4a64-8139-ddae05cefe63",
   "metadata": {},
   "outputs": [],
   "source": [
    "#matchms mgf encoding\n",
    "\n",
    "def prepro_specs_train(df):\n",
    "    valid=[]\n",
    "    precs=df['precursor_mz'].to_list()\n",
    "    mzs=df['mzs'].to_list()\n",
    "    ints=df['intensities'].to_list()\n",
    "    loss_mzs=df['loss_mzs'].to_list()\n",
    "    loss_ints=df['loss_intensities'].to_list()\n",
    "    for one_pre,one_mzs,one_ints,one_loss,one_loss_ints in tqdm(zip(precs,mzs,ints,loss_mzs,loss_ints)):\n",
    "        mz_list=[round(float(trun_n_d(str(one_pre),2))*100)] # add precursor mz\n",
    "        intes_list=[2.0] # add precursor int\n",
    "        res = dict(zip(one_mzs+one_loss, one_ints+one_loss_ints))  # order by mzs\n",
    "        res=dict(sorted(res.items()))\n",
    "        for m,i in zip(list(res.keys()), list(res.values())): # change this from mgf from matchms\n",
    "            mz=round(float(trun_n_d(str(m),2))*100)\n",
    "            mz_list.append(mz)\n",
    "            intens=round(i,4)\n",
    "            intes_list.append(intens)\n",
    "        int_mzs=[intes_list,mz_list]   \n",
    "        valid.append(int_mzs) # put intesities at first\n",
    "    return tf.ragged.constant(valid)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "c1230761-5a62-431e-906b-a403f2127558",
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepro_specs_train(df):\n",
    "    valid=[]\n",
    "    precs=df['precursor_mz'].to_list()\n",
    "    mzs=df['mzs'].to_list()\n",
    "    ints=df['intensities'].to_list()\n",
    "    loss_mzs=df['loss_mzs'].to_list()\n",
    "    loss_ints=df['loss_intensities'].to_list()\n",
    "    for one_pre,one_mzs,one_ints,one_loss,one_loss_ints in tqdm(zip(precs,mzs,ints,loss_mzs,loss_ints)):\n",
    "        mz_list=[round(float(trun_n_d(str(one_pre),2))*100)] # add precursor mz\n",
    "        intes_list=[2.0] # add precursor int\n",
    "        res = dict(zip(json.loads(one_mzs)+json.loads(one_loss), json.loads(one_ints)+json.loads(one_loss_ints)))  # order by mzs\n",
    "        res=dict(sorted(res.items()))\n",
    "        for m,i in zip(list(res.keys()), list(res.values())): # change this from mgf from matchms\n",
    "            mz=round(float(trun_n_d(str(m),2))*100)\n",
    "            mz_list.append(mz)\n",
    "            intens=round(i,4)\n",
    "            intes_list.append(intens)\n",
    "        int_mzs=[intes_list,mz_list]   \n",
    "        valid.append(int_mzs) # put intesities at first\n",
    "    return tf.ragged.constant(valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "e63172d5-e297-4760-91c1-2c4e1eae540e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "242it [00:00, 1234.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 516 ms\n",
      "Wall time: 475 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train=prepro_specs_train(df_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c7c0941a-7433-4d4c-b78e-572fe1ba72a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([5, None, None])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.gather(train, [0, 1, 2, 3, 4]).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "54d9db5e-77e0-4718-8565-b9ddd4f45063",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([32, None, None])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train[0:32].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "57acb8e7-faaa-42d5-a7ca-89b969e20a0f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "253"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "length=[i[0].shape[0] for i in train]\n",
    "max(length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "38425b96-bac0-4aac-8390-77598f6d39f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "dimn=256\n",
    "def encoding(rag_tensor,P,dimn):\n",
    "    to_pad=[]\n",
    "    for sample in rag_tensor:\n",
    "        all_dim=[sample[0].numpy().tolist()]\n",
    "        pos_enc=[P[int(i)-1] for i in sample[1].numpy().tolist()]\n",
    "        for dim in range(dimn):\n",
    "            dim_n=[i[dim] for i in pos_enc]\n",
    "            all_dim.append(dim_n)\n",
    "        to_pad.append(all_dim)\n",
    "    to_pad=[tf.keras.preprocessing.sequence.pad_sequences(i,maxlen=501,dtype='float32',padding='post',truncating='post',value=10) for i in to_pad]\n",
    "    to_pad=np.stack((to_pad))\n",
    "    to_pad=np.swapaxes(to_pad, 1, -1)\n",
    "    return to_pad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "6aa1bde2-2aa0-4508-bbe9-614fefa5c73b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.0717    ,  0.5332156 , -0.8459794 ,  0.01143887, -0.99993455,\n",
       "        0.28827846, -0.95754665,  0.9954703 ,  0.09507313, -0.95233387,\n",
       "        0.3050577 ,  0.59969896,  0.8002257 ,  0.57598245,  0.8174621 ,\n",
       "       -0.996885  , -0.07886912, -0.17545353,  0.9844877 , -0.9999982 ,\n",
       "       -0.00188863,  0.1089685 ,  0.9940452 , -0.98186713, -0.18957031,\n",
       "       -0.95267904,  0.30397803,  0.9439646 ,  0.33004668, -0.80919385,\n",
       "        0.58754176, -0.9424226 , -0.3344243 , -0.86887133, -0.49503803,\n",
       "       -0.94475156, -0.32778722, -0.9721955 , -0.2341707 , -0.7721371 ,\n",
       "       -0.63545597,  0.5649307 , -0.82513833, -0.4526213 ,  0.89170283,\n",
       "        0.68732804,  0.72634715, -0.56023455,  0.82833403,  0.93247634,\n",
       "        0.36123106, -0.654053  ,  0.75644875, -0.6053066 ,  0.7959924 ,\n",
       "        0.69717646,  0.7168996 ,  0.22841789, -0.9735632 , -0.92402524,\n",
       "        0.38233152,  0.37131587,  0.9285066 ,  0.55711097,  0.8304381 ,\n",
       "       -0.9599877 ,  0.28004214,  0.25998947,  0.96561146, -0.96464807,\n",
       "       -0.26354155, -0.84793615, -0.5300983 , -0.9946696 ,  0.10311342,\n",
       "       -0.6722058 ,  0.7403644 , -0.7557188 ,  0.6548962 , -0.3556158 ,\n",
       "       -0.93463224, -0.9336576 ,  0.3581669 ,  0.2960882 , -0.9551606 ,\n",
       "        0.90296847, -0.42970684,  0.9975964 ,  0.06929295,  0.16034408,\n",
       "        0.9870612 ,  0.04580148, -0.99895054,  0.33682257, -0.94156814,\n",
       "       -0.90871364, -0.41742012, -0.05195929,  0.9986492 ,  0.8410124 ,\n",
       "        0.5410158 , -0.2589479 ,  0.9658913 ,  0.83173   ,  0.5551804 ,\n",
       "       -0.07052226, -0.9975102 , -0.99015146,  0.1400004 , -0.904356  ,\n",
       "       -0.42677885, -0.22271346,  0.974884  ,  0.82213503, -0.5692925 ,\n",
       "       -0.22886883, -0.9734573 ,  0.99570584,  0.09257355,  0.73236376,\n",
       "        0.6809136 , -0.5831401 ,  0.8123716 ,  0.97060597,  0.24067426,\n",
       "       -0.9765509 , -0.21528676,  0.3335899 ,  0.94271827, -0.7904735 ,\n",
       "        0.61249626, -0.99346447, -0.1141419 ,  0.9764814 , -0.21560162,\n",
       "        0.5799171 ,  0.8146755 , -0.9991347 , -0.04159042, -0.9890586 ,\n",
       "        0.1475232 , -0.989487  , -0.14462198,  0.7041925 ,  0.7100091 ,\n",
       "        0.19167444, -0.98145854,  0.99982166,  0.01888514, -0.46084157,\n",
       "        0.8874824 , -0.33013698,  0.943933  , -0.9985991 , -0.05291281,\n",
       "       -0.97547275, -0.2201203 ,  0.99999547,  0.00300953, -0.9642097 ,\n",
       "       -0.26514086,  0.4496711 , -0.8931942 , -0.98045963, -0.19672027,\n",
       "       -0.7123622 ,  0.701812  ,  0.8237492 ,  0.56695443, -0.96114033,\n",
       "        0.2760603 ,  0.01536015,  0.99988204, -0.27245298,  0.9621691 ,\n",
       "       -0.94391733,  0.3301819 , -0.85027707, -0.52633536, -0.6628971 ,\n",
       "       -0.7487105 , -0.9762616 , -0.21659462, -0.13901651,  0.99029005,\n",
       "        0.66964203, -0.742684  , -0.7097949 ,  0.7044084 ,  0.7883553 ,\n",
       "       -0.6152202 , -0.930816  , -0.36548817, -0.9543821 , -0.2985879 ,\n",
       "        0.8720212 ,  0.4894681 , -0.8894411 ,  0.45704985,  0.7674    ,\n",
       "       -0.6411686 ,  0.5122702 , -0.8588243 , -0.46367675, -0.88600445,\n",
       "        0.65450615, -0.75605667,  0.88037896, -0.47427094, -0.86710006,\n",
       "        0.49813405,  0.9908141 , -0.135231  , -0.26741374,  0.9635818 ,\n",
       "        0.7690229 ,  0.63922125,  0.03958566,  0.9992162 , -0.89686114,\n",
       "       -0.44231218,  0.95714754,  0.28960076, -0.3952544 , -0.9185717 ,\n",
       "       -0.86790514, -0.49672997,  0.8436555 , -0.53688484,  0.86669165,\n",
       "       -0.49884427,  0.9220753 , -0.3870106 , -0.7610874 ,  0.64864933,\n",
       "        0.16541438, -0.9862242 , -0.29848945, -0.95441294,  0.6250325 ,\n",
       "       -0.78059876,  0.9744055 ,  0.22479734,  0.673075  ,  0.7395742 ,\n",
       "        0.6001858 ,  0.7998606 ,  0.35591003,  0.93452024, -0.9983589 ,\n",
       "        0.05726733, -0.75771755,  0.65258265, -0.563865  ,  0.82586694,\n",
       "        0.9665189 , -0.2565955 ], dtype=float32)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xtrain[0][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "adbccd44-0de9-4492-8a39-ac30cd2f29fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 7.39 s\n",
      "Wall time: 7.39 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "xtrain=encoding(train,P,dimn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "56740753-1960-4e35-9cb9-08951cd8aa42",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 3.28 s\n",
      "Wall time: 3.29 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "np.save('casmi_specs.npy',xtrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73610487-9ad9-4bdc-a6c6-18f7f72298df",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}