new words

1 files changed, 46 insertions, 424 deletions

diff --git a/generators/datageneration.ipynb b/generators/datageneration.ipynb
index 85c6e02..9bb1407 100644
--- a/generators/datageneration.ipynb
+++ b/generators/datageneration.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 38,
    "id": "e1b17564-0abb-41c5-8cf4-7200b014550f",
    "metadata": {},
    "outputs": [
@@ -25,18 +25,9 @@
     "nltk.download('wordnet')\n",
     "from nltk.corpus import words\n",
     "nltk.download('words')\n",
-    "ww = words.words()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "id": "8fe45bc7-a41a-49db-9067-700254f388c0",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def format(s):\n",
-    "    return ' '.join(s.split('_'))"
+    "ww = words.words()\n",
+    "import pandas as pd \n",
+    "import string"
    ]
   },
   {
@@ -46,16 +37,26 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "def format(s):\n",
+    "    return ' '.join(s.split('_'))\n",
     "from collections import defaultdict\n",
     "lexnames = set()\n",
     "lexname_blacklist = {\n",
     "    'noun.plant',\n",
     "    'noun.animal',\n",
-    "    'noun.person'\n",
+    "    'noun.person',\n",
+    "    'noun.state',\n",
+    "    'noun.body',\n",
+    "    #'noun.location',\n",
+    "    #'noun.group',\n",
     "}\n",
+    "def is_proper(w):\n",
+    "    return w[0] in string.ascii_uppercase\n",
     "def groups_for_pos(pos='as'):\n",
     "    dsynonyms = []\n",
     "    for word in wn.words():\n",
+    "        if is_proper(word):\n",
+    "            continue\n",
     "        synsets = wn.synsets(word)\n",
     "        synonymss = wn.synonyms(word)\n",
     "        syns = set()\n",
@@ -63,20 +64,11 @@
     "            if synset.lexname() in lexname_blacklist:\n",
     "                continue\n",
     "            if synset.pos() in pos:\n",
-    "                syns |= set(synonyms)\n",
+    "                syns |= set(syn for syn in synonyms if not is_proper(syn))\n",
     "        if len(syns) >= 4:\n",
     "            clues = [format(clue) for clue in syns]\n",
     "            clues.append(format(word))\n",
     "            dsynonyms.append(dict(answer=word, hint=f'synonyms for {format(word)}', clues=clues))\n",
-    "            \"\"\"    \n",
-    "            ok = True\n",
-    "            for clue in clues:\n",
-    "                if clue in dsynonyms:\n",
-    "                    ok = False\n",
-    "            if ok:\n",
-    "                clues.append(format(word))\n",
-    "                dsynonyms[word] = dict(group=f'synonyms for {format(word)}', clues=clues)\n",
-    "            \"\"\"\n",
     "    return dsynonyms"
    ]
   },
@@ -95,218 +87,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
-   "id": "7e552fc8-03b2-4b8f-b6f6-072d580702bc",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "(3976, 7141, 19563, 490)"
-      ]
-     },
-     "execution_count": 30,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "len(dadj), len(dverb), len(dnoun), len(dadverb)"
-   ]
-  },
-  {
-   "cell_type": "code",
    "execution_count": 62,
-   "id": "8d780210-38a4-4f8d-ae2e-b4631cb06368",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "(3976, 7141, 13824, 490)"
-      ]
-     },
-     "execution_count": 62,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "len(dadj), len(dverb), len(dnoun), len(dadverb)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 33,
-   "id": "48233554-2634-4a5e-9013-4e45c6f7d3d9",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# flag button for reporting, definitions in webcollectionscollections"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "1464f8df-180a-4334-b123-d76303140a03",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "f4eb0f95-901d-43d8-8ba7-6103b3a0f6be",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 57,
-   "id": "bdcb3a19-a5cf-48e9-ab6f-aaeed50e2c31",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{'adj.all',\n",
-       " 'adj.pert',\n",
-       " 'adj.ppl',\n",
-       " 'adv.all',\n",
-       " 'noun.Tops',\n",
-       " 'noun.act',\n",
-       " 'noun.animal',\n",
-       " 'noun.artifact',\n",
-       " 'noun.attribute',\n",
-       " 'noun.body',\n",
-       " 'noun.cognition',\n",
-       " 'noun.communication',\n",
-       " 'noun.event',\n",
-       " 'noun.feeling',\n",
-       " 'noun.food',\n",
-       " 'noun.group',\n",
-       " 'noun.location',\n",
-       " 'noun.motive',\n",
-       " 'noun.object',\n",
-       " 'noun.person',\n",
-       " 'noun.phenomenon',\n",
-       " 'noun.plant',\n",
-       " 'noun.possession',\n",
-       " 'noun.process',\n",
-       " 'noun.quantity',\n",
-       " 'noun.relation',\n",
-       " 'noun.shape',\n",
-       " 'noun.state',\n",
-       " 'noun.substance',\n",
-       " 'noun.time',\n",
-       " 'verb.body',\n",
-       " 'verb.change',\n",
-       " 'verb.cognition',\n",
-       " 'verb.communication',\n",
-       " 'verb.competition',\n",
-       " 'verb.consumption',\n",
-       " 'verb.contact',\n",
-       " 'verb.creation',\n",
-       " 'verb.emotion',\n",
-       " 'verb.motion',\n",
-       " 'verb.perception',\n",
-       " 'verb.possession',\n",
-       " 'verb.social',\n",
-       " 'verb.stative',\n",
-       " 'verb.weather'}"
-      ]
-     },
-     "execution_count": 57,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 50,
-   "id": "d3a0f1e9-99e5-4aa2-bd72-d1d200bf2b40",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "'noun.plant'"
-      ]
-     },
-     "execution_count": 50,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "syn.lexname()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "64ad6047-a09b-41c3-85b6-65476d8dba0e",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "88a4aeab-8123-435e-9b44-3e568102f0b1",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "b6459fdf-fe4e-4dcf-81ef-1688de01be95",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "id": "e588bdf3-d648-48b3-ab6b-027a07194292",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import pandas as pd"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "id": "b27aa837-73d2-4b10-826b-990e12a3f7e2",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df = pd.read_csv('LADECv1-2019.csv', index_col=0)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "id": "176e2790-560c-4daf-b436-a1771611c4bf",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df = df.drop(df[df.correctParse == 'no'].index)\n",
-    "df = df.drop(df[df.isCommonstim == 0].index)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
    "id": "64ccaf3d-9743-49ed-b10b-d7b3e70e0235",
    "metadata": {},
    "outputs": [],
    "source": [
+    "df = pd.read_csv('LADECv1-2019.csv', index_col=0)\n",
+    "df = df.drop(df[df.correctParse == 'no'].index)\n",
+    "df = df.drop(df[df.isCommonstim == 0].index)\n",
     "prefixes = df.groupby('c1').groups\n",
     "suffixes = df.groupby('c2').groups\n",
     "dprefix = []\n",
@@ -321,15 +109,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "86c69c9f-bc6a-4dd1-9eb3-ab37ad766586",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 63,
    "id": "def43999-d789-4e5c-bb27-4fd29074c875",
    "metadata": {},
    "outputs": [],
@@ -346,85 +126,6 @@
     "        pass\n",
     "        #print(a, b, r)\n",
     "    return r >= .85\n",
-    "\"\"\"\n",
-    "Skip to Main\n",
-    "datageneration\n",
-    "Last Checkpoint: 11 days ago\n",
-    "[Python 3 (ipykernel)]\n",
-    "import json\n",
-    "import nltk\n",
-    "from nltk.corpus import wordnet as wn\n",
-    "from nltk.stem.wordnet import WordNetLemmatizer\n",
-    "nltk.download('wordnet')\n",
-    "from nltk.corpus import words\n",
-    "nltk.download('words')\n",
-    "ww = words.words()\n",
-    "\n",
-    "[nltk_data] Downloading package wordnet to /home/sy/nltk_data...\n",
-    "[nltk_data]   Package wordnet is already up-to-date!\n",
-    "[nltk_data] Downloading package words to /home/sy/nltk_data...\n",
-    "[nltk_data]   Package words is already up-to-date!\n",
-    "\n",
-    "def format(s):\n",
-    "    return ' '.join(s.split('_'))\n",
-    "from collections import defaultdict\n",
-    "\n",
-    "def groups_for_pos(pos='as'):\n",
-    "    dsynonyms = []\n",
-    "    for word in wn.words():\n",
-    "        synsets = wn.synsets(word)\n",
-    "        synonymss = wn.synonyms(word)\n",
-    "        syns = set()\n",
-    "        for synset, synonyms in zip(synsets, synonymss):\n",
-    "            if synset.pos() in pos: # 'as'\n",
-    "                syns |= set(synonyms)\n",
-    "        if len(syns) >= 4:\n",
-    "            clues = [format(clue) for clue in syns]\n",
-    "            \n",
-    "            clues.append(format(word))\n",
-    "            dsynonyms.append(dict(answer=word, hint=f'synonyms for {format(word)}', clues=clues))\n",
-    "            \n",
-    "            ok = True\n",
-    "            for clue in clues:\n",
-    "                if clue in dsynonyms:\n",
-    "                    ok = False\n",
-    "            if ok:\n",
-    "                clues.append(format(word))\n",
-    "                dsynonyms[word] = dict(group=f'synonyms for {format(word)}', clues=clues)\n",
-    "            \n",
-    "    return dsynonyms\n",
-    "\n",
-    "dadj = groups_for_pos('as')\n",
-    "dverb = groups_for_pos('v')\n",
-    "dnoun = groups_for_pos('n')\n",
-    "dadverb = groups_for_pos('r')\n",
-    "len(dadj), len(dverb), len(dnoun), len(dadverb)\n",
-    "\n",
-    "(3976, 7141, 19563, 490)\n",
-    "\n",
-    "\n",
-    "# flag button for reporting\n",
-    "\n",
-    "\n",
-    "import pandas as pd\n",
-    "df = pd.read_csv('LADECv1-2019.csv', index_col=0)\n",
-    "df = df.drop(df[df.correctParse == 'no'].index)\n",
-    "df = df.drop(df[df.isCommonstim == 0].index)\n",
-    "prefixes = df.groupby('c1').groups\n",
-    "suffixes = df.groupby('c2').groups\n",
-    "dprefix = []\n",
-    "for prefix, ids in prefixes.items():\n",
-    "    if len(ids) >= 4:\n",
-    "        dprefix.append(dict(answer=prefix, hint=f'{prefix} _', clues=list(df.loc[list(ids)].c2)))\n",
-    "dsuffix = []\n",
-    "for suffix, ids in suffixes.items():\n",
-    "    if len(ids) >= 4:\n",
-    "        dsuffix.append(dict(answer=suffix, hint=f'_ {suffix}', clues=list(df.loc[list(ids)].c1)))\n",
-    "\n",
-    ":\n",
-    "        return True\n",
-    "    # Then, print everything between .8 and .9 to see whats the best cutoff\n",
-    "\"\"\"\n",
     "def filter_duplicates(group):\n",
     "    if not group:\n",
     "        return []\n",
@@ -440,21 +141,12 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
-   "id": "6a3c04eb-79a6-47f5-846e-93258db65921",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "blacklist = ['man', 'men', 'woman', 'women']"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 63,
-   "id": "dfb38b21-3dc4-495a-8805-446b2e9e8483",
+   "execution_count": 64,
+   "id": "1c8175f2-817e-45ab-af0b-5ea7ee7a5dc9",
    "metadata": {},
    "outputs": [],
    "source": [
+    "blacklist = ['man', 'men', 'woman', 'women']\n",
     "def process_groups(groups):\n",
     "    new = []\n",
     "    for group in groups:\n",
@@ -464,16 +156,7 @@
     "        if len(clues) < 4:\n",
     "            continue\n",
     "        new.append(dict(answer=group['answer'], hint=group['hint'], clues=clues))\n",
-    "    return new"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 64,
-   "id": "1c8175f2-817e-45ab-af0b-5ea7ee7a5dc9",
-   "metadata": {},
-   "outputs": [],
-   "source": [
+    "    return new\n",
     "corpus = [\n",
     "    dict(name='suffixes', groups=process_groups(dsuffix), portion=.9),\n",
     "    dict(name='prefixes', groups=process_groups(dprefix), portion=.8),\n",
@@ -487,80 +170,61 @@
   {
    "cell_type": "code",
    "execution_count": 65,
-   "id": "b35092c5-17db-4257-bf45-83e8c3973da4",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "[Synset('basil_thyme.n.01')]"
-      ]
-     },
-     "execution_count": 65,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "wn.synsets('satureja_acinos')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 66,
-   "id": "8025664c-e116-481a-9609-d58200f773ec",
+   "id": "1f2cca38-5d1a-421e-9bcc-09d68880c6f1",
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Can lemmatize/singularize and compare levenshtein based on that, but don't use lemma itself for the group as it may bed wrong"
+    "with open('../static/corpus.js', 'w') as f:\n",
+    "    f.write('var corpus = ')\n",
+    "    json.dump(corpus, f)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 67,
-   "id": "77256802-04ef-4908-9b39-e9381f6abac5",
+   "execution_count": 66,
+   "id": "589f6645-3a52-40ad-9899-717ea7614d00",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "(13824, 3976, 490, 7141, 226, 196)"
+       "'noun.group'"
       ]
      },
-     "execution_count": 67,
+     "execution_count": 66,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "len(dnoun), len(dadj), len(dadverb), len(dverb), len(dprefix), len(dsuffix)"
+    "wn.synsets('Islamic_jihad')[0].lexname()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 68,
-   "id": "66256c50-fcbc-42ad-a17b-057fd0d7dea1",
+   "execution_count": 67,
+   "id": "8faeb5ee-e1ff-4571-91bf-178cdc7d29f7",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "25853"
+       "'ABCDEFGHIJKLMNOPQRSTUVWXYZ'"
       ]
      },
-     "execution_count": 68,
+     "execution_count": 67,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "sum((len(dnoun), len(dadj), len(dadverb), len(dverb), len(dprefix), len(dsuffix)))"
+    "    string.ascii_uppercase"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "635b6a6c-c084-4584-a63f-cde4221e0ad9",
+   "id": "b8d12e1a-757f-4d87-9374-e7eb656f30d4",
    "metadata": {},
    "outputs": [],
    "source": []
@@ -568,73 +232,31 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "f48e2eec-2ab1-4398-a862-172398c413a0",
+   "id": "68f254d5-7a96-488a-b1a8-6a82dce44271",
    "metadata": {},
    "outputs": [],
    "source": []
   },
   {
    "cell_type": "code",
-   "execution_count": 70,
-   "id": "1f2cca38-5d1a-421e-9bcc-09d68880c6f1",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "with open('../static/corpus.js', 'w') as f:\n",
-    "    f.write('var corpus = ')\n",
-    "    json.dump(corpus, f)"
-   ]
-  },
-  {
-   "cell_type": "code",
    "execution_count": null,
-   "id": "551ce71f-0d75-4e41-8387-808db1e5e20f",
+   "id": "091e0422-99be-4b20-a20d-17ac296990b6",
    "metadata": {},
    "outputs": [],
    "source": []
   },
   {
    "cell_type": "code",
-   "execution_count": 38,
-   "id": "19589357-f1ca-4d10-8574-3639bd05173f",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "26388"
-      ]
-     },
-     "execution_count": 38,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 39,
-   "id": "dd927be9-a77c-4606-984a-b3cf555b2618",
+   "execution_count": null,
+   "id": "ff46e8c4-b806-43f6-8cf9-d4260bad1ba8",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "195"
-      ]
-     },
-     "execution_count": 39,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": []
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "f598354d-3f52-4952-a8c0-69c480ebe8b1",
+   "id": "668b25f4-6992-41a3-881c-3a6a72ba0d77",
    "metadata": {},
    "outputs": [],
    "source": []
@@ -642,7 +264,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "77cb04f6-846e-454b-98e1-4feb575d2332",
+   "id": "1a0ae159-2a40-4ca5-915a-6ca40df3080f",
    "metadata": {},
    "outputs": [],
    "source": []