JigsawMultilingualToxicComments[Ensemble].ipynb

{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from sklearn import *\nimport numpy as np\nimport pandas as pd\nimport glob\n\ndata = {k.split('/')[-1][:-4]:k for k in glob.glob('/kaggle/input/**/**.csv')}\ntrain = pd.read_csv(data['jigsaw-toxic-comment-train'], usecols=['id', 'comment_text', 'toxic'])\nval = pd.read_csv(data['validation'], usecols=['comment_text', 'toxic'])\ntest = pd.read_csv(data['test'], usecols=['id', 'content'])\ntest.columns = ['id', 'comment_text']\ntest['toxic'] = 0.5\n\nsub2 = pd.read_csv('../input/ensemble/submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndef f_experience(c, s):\n    it = {'memory':10,\n        'influence':0.5,\n        'inference':0.5,\n        'interest':0.9,\n        'sentiment':1e-10,\n        'harmony':0.5}\n    \n    exp = {}\n    \n    for i in range(len(c)):\n        words = set([w for w in str(c[i]).lower().split(' ')])\n        for w in words:\n            try:\n                exp[w]['influence'] = exp[w]['influence'][1:] + [s[i]] #need to normalize\n                exp[w]['inference'] += 1\n                exp[w]['interest'] = exp[w]['interest'][1:] + [(exp[w]['interest'][it['memory']-1] + (s[i] * it['interest']))/2]\n                exp[w]['sentiment'] += s[i]\n                #exp[w]['harmony']\n            except:\n                m = [0. for m_ in range(it['memory'])]\n                exp[w] = {}\n                exp[w]['influence'] = m[1:] + [s[i]]\n                exp[w]['inference'] = 1\n                exp[w]['interest'] = m[1:] + [s[i] * it['interest'] / 2]\n                exp[w]['sentiment'] = s[i]\n                #exp[w]['harmony'] = 0\n                \n    for w in exp:\n        exp[w]['sentiment'] /= exp[w]['inference'] + it['sentiment']\n        exp[w]['inference'] /= len(c) * it['inference']\n\n    return exp\n\nexp = f_experience(train['comment_text'].values, train['toxic'].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndef features(df):\n    df['len'] = df['comment_text'].map(len)\n    df['wlen'] = df['comment_text'].map(lambda x: len(str(x).split(' ')))\n    \n    df['influence_sum'] = df['comment_text'].map(lambda x: np.sum([np.mean(exp[w]['influence']) if w in exp else 0 for w in str(x).lower().split(' ')]))\n    df['influence_mean'] = df['comment_text'].map(lambda x: np.mean([np.mean(exp[w]['influence']) if w in exp else 0 for w in str(x).lower().split(' ')]))\n    \n    df['inference_sum'] = df['comment_text'].map(lambda x: np.sum([exp[w]['inference'] if w in exp else 0 for w in str(x).lower().split(' ')]))\n    df['inference_mean'] = df['comment_text'].map(lambda x: np.mean([exp[w]['inference'] if w in exp else 0 for w in str(x).lower().split(' ')]))\n    \n    df['interest_sum'] = df['comment_text'].map(lambda x: np.sum([np.mean(exp[w]['interest']) if w in exp else 0 for w in str(x).lower().split(' ')]))\n    df['interest_mean'] = df['comment_text'].map(lambda x: np.mean([np.mean(exp[w]['interest']) if w in exp else 0 for w in str(x).lower().split(' ')]))\n    \n    df['sentiment_sum'] = df['comment_text'].map(lambda x: np.sum([exp[w]['sentiment'] if w in exp else 0.5 for w in str(x).lower().split(' ')]))\n    df['sentiment_mean'] = df['comment_text'].map(lambda x: np.mean([exp[w]['sentiment'] if w in exp else 0.5 for w in str(x).lower().split(' ')]))\n    return df\n\nval = features(val)\ntest= features(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"col = [c for c in val if c not in ['id', 'comment_text', 'toxic']]\nx1, x2, y1, y2 = model_selection.train_test_split(val[col], val['toxic'], test_size=0.3, random_state=20)\n\nmodel = ensemble.ExtraTreesClassifier(n_estimators=1000, max_depth=7, n_jobs=-1, random_state=20)\nmodel.fit(x1, y1)\nprint(metrics.roc_auc_score(y2, model.predict_proba(x2)[:,1].clip(0.,1.)))\n\nmodel.fit(val[col], val['toxic'])\ntest['toxic'] = model.predict_proba(test[col])[:,1].clip(0.,1.)\nsub1 = test[['id', 'toxic']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub1.rename(columns={'toxic':'toxic1'}, inplace=True)\nsub2.rename(columns={'toxic':'toxic2'}, inplace=True)\nsub3 = pd.merge(sub1, sub2, how='left', on='id')\n\nsub3['toxic'] = (sub3['toxic1'] * 0.1) + (sub3['toxic2'] * 0.9) #blend 1\nsub3['toxic'] = (sub3['toxic2'] * 0.51) + (sub3['toxic'] * 0.49) #blend 2\n\nsub3[['id', 'toxic']].to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Is it toxic :)\ntest = pd.DataFrame(['Howling with Wolf on Lügenpresse'], columns=['comment_text'])\ntest['id'] = test.index\ntest= features(test)\ntest['toxic'] = model.predict_proba(test[col])[:,1].clip(0.,1.)\ntest[['id', 'comment_text', 'toxic']].head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Ｈ𝐀𝑷𝑷𝓎 🇰𝗮𝘨𝘨🇱𝖎Ｎɢ 💯\n========"}],"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}