{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Non alphanums {'>', '\\t', '[', '\\r', '\\x13', '\\x1d', '<', '^', '\\x04', '\\x08', '\\x17', '\\x1b', '\\\\', '\\x19', '`', '\\x1f', '$', '\\x0b', '\\x14', '\\x12', '\\x18', ')', '=', '?', ' ', \"'\", '\\x1c', '|', ';', '/', '\\x1e', '\\n', '\\x07', '\\x10', '\\x03', '\\x02', '#', '\\x0c', '@', '\\x16', '_', '}', '.', '-', '(', '!', '+', '\\x06', ']', '{', ':', '\\x01', '\\x11', '\\x0f', '\\x05', ',', '~', '\\x15', '\\x00', '\"', '%', '\\x0e', '*', '&', '\\x1a'}\n",
      "Separators b'([\\\\>\\\\\\t\\\\[\\\\\\r\\\\\\x13\\\\\\x1d\\\\<\\\\^\\\\\\x04\\\\\\x08\\\\\\x17\\\\\\x1b\\\\\\\\\\\\\\x19\\\\`\\\\\\x1f\\\\$\\\\\\x0b\\\\\\x14\\\\\\x12\\\\\\x18\\\\)\\\\=\\\\?\\\\ \\\\\\'\\\\\\x1c\\\\|\\\\;\\\\/\\\\\\x1e\\\\\\n\\\\\\x07\\\\\\x10\\\\\\x03\\\\\\x02\\\\#\\\\\\x0c\\\\@\\\\\\x16\\\\_\\\\}\\\\.\\\\-\\\\(\\\\!\\\\+\\\\\\x06\\\\]\\\\{\\\\:\\\\\\x01\\\\\\x11\\\\\\x0f\\\\\\x05\\\\,\\\\~\\\\\\x15\\\\\\x00\\\\\"\\\\%\\\\\\x0e\\\\*\\\\&\\\\\\x1a])'\n"
     ]
    }
   ],
   "source": [
    "import string\n",
    "\n",
    "non_alphanums = set(chr(x) for x in range(127)) - set(string.ascii_letters) - set(string.digits)\n",
    "print(\"Non alphanums\", non_alphanums)\n",
    "separators = '([' + ''.join('\\\\' + x for x in non_alphanums) + '])'\n",
    "separators = separators.encode('ascii')\n",
    "print(\"Separators\", separators)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[b'dlg',\n",
       " b'=',\n",
       " b'Resource',\n",
       " b'.',\n",
       " b'loadfromresfile',\n",
       " b'(',\n",
       " b'filename',\n",
       " b',',\n",
       " b'win',\n",
       " b',',\n",
       " b'QuoteDialog',\n",
       " b'.',\n",
       " b'MyQuoteDialog',\n",
       " b',',\n",
       " b\"'\",\n",
       " b'QuoteDialog',\n",
       " b\"'\",\n",
       " b',',\n",
       " b'win',\n",
       " b')']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "program_line = b\"\"\"dlg = Resource.loadfromresfile(filename, win, QuoteDialog.MyQuoteDialog, 'QuoteDialog', win)\"\"\"\n",
    "tokens = [t for x in re.split(separators, program_line) if (t := x.strip())]\n",
    "tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking base dir: HTML document\n",
      "Checking base dir: C source\n",
      "Checking base dir: C++ source\n",
      "Checking base dir: PHP script\n",
      "Checking base dir: ReStructuredText file\n",
      "Checking base dir: Python script\n",
      "Checking base dir: Ruby script\n",
      "Checking base dir: Java source\n",
      "Checking base dir: Objective-C source\n",
      "Checking base dir: Perl5 module source\n",
      "Checking base dir: XML 1.0 document\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from collections import Counter\n",
    "\n",
    "DUMP_BASE = '/home/facundo/devel/ml/dump'\n",
    "\n",
    "# directories with 1000 files of each code type, excluding \"just text\" (ascii, utf8, etc)\n",
    "CODE_TYPES = [\n",
    "    'HTML document',\n",
    "    'C source',\n",
    "    'C++ source',\n",
    "    'PHP script',\n",
    "    'ReStructuredText file',\n",
    "    'Python script',\n",
    "    'Ruby script',\n",
    "    'Java source',\n",
    "    'Objective-C source',\n",
    "    'Perl5 module source',\n",
    "    'XML 1.0 document',\n",
    "]\n",
    "\n",
    "# let's collect ALL tokens present in all the program files\n",
    "tokens = Counter()\n",
    "for basedir in CODE_TYPES:\n",
    "    print(\"Checking base dir:\", basedir)\n",
    "    for dirpath, dirnames, filenames in os.walk(os.path.join(DUMP_BASE, basedir)):\n",
    "        for fname in filenames:\n",
    "            fpath = os.path.join(dirpath, fname)\n",
    "            with open(fpath, 'rb') as fh:\n",
    "                tokens.update(t for x in re.split(separators, fh.read()) if (t := x.strip()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total different tokens 213400\n",
      "  870525 b'.'\n",
      "  756849 b'_'\n",
      "  730609 b'('\n",
      "  717433 b'='\n",
      "  699725 b')'\n",
      "  688556 b'/'\n",
      "  661461 b'\"'\n",
      "  640989 b','\n",
      "  625121 b'-'\n",
      "  594091 b'>'\n"
     ]
    }
   ],
   "source": [
    "different_tokens = len(tokens)\n",
    "print(\"Total different tokens\", different_tokens)\n",
    "for name, quant in tokens.most_common(10):\n",
    "    print(\"{:8d} {}\".format(quant, name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total tokens found: 20101050\n",
      "Total representative: 3324\n",
      "Last ten...\n",
      "     289 b'enables'\n",
      "     289 b'smaller'\n",
      "     289 b'Creates'\n",
      "     289 b'cross'\n",
      "     289 b'GLFW'\n",
      "     289 b'Os'\n",
      "     289 b'usb'\n",
      "     288 b'stylesheets'\n",
      "     288 b'ad'\n",
      "     288 b'WIDTH'\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "total_tokens = sum(tokens.values())\n",
    "print(\"Total tokens found:\", total_tokens)\n",
    "\n",
    "most = total_tokens * 0.9\n",
    "tot = 0\n",
    "representative_data = []\n",
    "for name, quant in tokens.most_common():\n",
    "    representative_data.append((name, quant))\n",
    "    tot += quant\n",
    "    if tot > most:\n",
    "        break\n",
    "\n",
    "print(\"Total representative:\", len(representative_data))\n",
    "print(\"Last ten...\")\n",
    "for name, quant in representative_data[-10:]:\n",
    "    print(\"{:8d} {}\".format(quant, name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting data from base dir HTML document\n",
      "Collecting data from base dir C source\n",
      "Collecting data from base dir C++ source\n",
      "Collecting data from base dir PHP script\n",
      "Collecting data from base dir ReStructuredText file\n",
      "Collecting data from base dir Python script\n",
      "Collecting data from base dir Ruby script\n",
      "Collecting data from base dir Java source\n",
      "Collecting data from base dir Objective-C source\n",
      "Collecting data from base dir Perl5 module source\n",
      "Collecting data from base dir XML 1.0 document\n",
      "Src data samples: 11000\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "# real \"ML data\": a list of (code_type, features) (one pair for each file)\n",
    "#  code_type: the *position* of the code type corresponding to the file (needs to be an int)\n",
    "#  features: a list of values, each value corresponds to how many of the tokens of that position the file has\n",
    "\n",
    "representative_tokens = [name for name, _ in representative_data]\n",
    "\n",
    "all_src_data = []\n",
    "for idx, basedir in enumerate(CODE_TYPES):\n",
    "    print(\"Collecting data from base dir\", basedir)\n",
    "    for dirpath, dirnames, filenames in os.walk(os.path.join(DUMP_BASE, basedir)):\n",
    "        for fname in filenames:\n",
    "            fpath = os.path.join(dirpath, fname)\n",
    "            with open(fpath, 'rb') as fh:\n",
    "                fcontent = fh.read()\n",
    "            \n",
    "            file_tokens = Counter(t for x in re.split(separators, fcontent) if (t := x.strip()))\n",
    "            token_quantities = [file_tokens.get(t, 0) for t in representative_tokens]\n",
    "\n",
    "            all_src_data.append((idx, token_quantities))\n",
    "\n",
    "print(\"Src data samples:\", len(all_src_data))\n",
    "\n",
    "# shuffle, as currently is too much \"per directory\"\n",
    "random.shuffle(all_src_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras import Model, layers\n",
    "import numpy as np\n",
    "\n",
    "# representation of our model\n",
    "num_classes = len(CODE_TYPES)\n",
    "num_features = len(representative_tokens)\n",
    "\n",
    "# 1st and 2nd layer number of neurons (these numbers are just chamuyo)\n",
    "n_hidden_1 = 128  \n",
    "n_hidden_2 = 256\n",
    "\n",
    "# training parameters (more chamuyo)\n",
    "learning_rate = 0.1\n",
    "\n",
    "batch_size = 256\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Example code types: (2, 0, 4, 4, 5)\n",
      "Example token quants: [246, 786, 246, 61, 246] [32, 32, 5, 22, 5]\n",
      "Token quants shape: (11000, 3324)\n",
      "Example normalized quants: [0.3129771  1.         0.3129771  0.07760815 0.3129771 ] [0.9411765  0.9411765  0.14705883 0.64705884 0.14705883]\n"
     ]
    }
   ],
   "source": [
    "# separate the source data and into two pairing lists\n",
    "code_types, token_quantities = zip(*all_src_data)\n",
    "print(\"Example code types:\", code_types[:5])\n",
    "print(\"Example token quants:\", token_quantities[0][:5], token_quantities[117][:5])\n",
    "\n",
    "# convert features to float\n",
    "float_quantities = np.array(token_quantities, np.float32)\n",
    "print(\"Token quants shape:\", float_quantities.shape)\n",
    "\n",
    "# normalize EACH ONE to [0, 1]\n",
    "for quants in float_quantities:\n",
    "    quants /= max(quants)\n",
    "print(\"Example normalized quants:\", float_quantities[0][:5], float_quantities[117][:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Real training set: 89.70%\n"
     ]
    }
   ],
   "source": [
    "# let's prepare the teching sets, input and output for training first (90% of cases) and\n",
    "# then testing what's learned (the remaining 10%); \n",
    "input_training = []\n",
    "output_training = []\n",
    "input_testing = []\n",
    "output_testing = []\n",
    "for token_distribution, code_type in zip(float_quantities, code_types):\n",
    "    if random.random() < .1:\n",
    "        input_testing.append(token_distribution)\n",
    "        output_testing.append(code_type)\n",
    "    else:\n",
    "        input_training.append(token_distribution)\n",
    "        output_training.append(code_type)\n",
    "print(\"Real training set: {:.2f}%\".format(100 * len(input_training) / len(float_quantities)))        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras import Model\n",
    "\n",
    "class NeuralNet(Model):\n",
    "    \"\"\"Chamuyo al cuadrado.\"\"\"\n",
    "    \n",
    "    def __init__(self):\n",
    "        super(NeuralNet, self).__init__()\n",
    "        self.fc1 = layers.Dense(n_hidden_1, activation=tf.nn.sigmoid)  # se puede cambiar a relu\n",
    "        self.fc2 = layers.Dense(n_hidden_2, activation=tf.nn.sigmoid)  # se puede cambiar a relu\n",
    "        self.out = layers.Dense(num_classes)\n",
    "\n",
    "    def call(self, x, is_training=False):\n",
    "        x = self.fc1(x)\n",
    "        x = self.fc2(x)\n",
    "        x = self.out(x)\n",
    "        if not is_training:\n",
    "            # tf cross entropy expect logits without softmax, so only\n",
    "            # apply softmax when not training.\n",
    "            x = tf.nn.softmax(x)\n",
    "        return x\n",
    "\n",
    "neural_net = NeuralNet()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Note that this will apply 'softmax' to the logits.\n",
    "def cross_entropy_loss(x, y):\n",
    "    # Convert labels to int 64 for tf cross-entropy function.\n",
    "    y = tf.cast(y, tf.int64)\n",
    "    # Apply softmax to logits and compute cross-entropy.\n",
    "    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=x)\n",
    "    # Average loss across the batch.\n",
    "    return tf.reduce_mean(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Stochastic gradient descent optimizer.\n",
    "optimizer = tf.optimizers.SGD(learning_rate)\n",
    "\n",
    "# Optimización. \n",
    "def run_optimization(x, y):\n",
    "    # Funciones para calcular el gradiente\n",
    "    with tf.GradientTape() as g:\n",
    "        # Algoritmo de forward\n",
    "        pred = neural_net(x, is_training=True)\n",
    "        # Computa la función de costo o pérdida utilizando entropía cruzada\n",
    "        loss = cross_entropy_loss(pred, y)\n",
    "        \n",
    "    # Actualiza las variables de entrenamiento.\n",
    "    trainable_variables = neural_net.trainable_variables\n",
    "\n",
    "    # Computa los gradientes\n",
    "    gradients = g.gradient(loss, trainable_variables)\n",
    "    \n",
    "    # Actualiza los nuevos parámetros W (pesos) y b (bias).\n",
    "    optimizer.apply_gradients(zip(gradients, trainable_variables))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Accuracy metric.\n",
    "def accuracy(y_pred, y_true):\n",
    "    # Predicted class is the index of highest score in prediction vector (i.e. argmax).\n",
    "    correct_prediction = tf.equal(tf.argmax(y_pred, 1), tf.cast(y_true, tf.int64))\n",
    "    return tf.reduce_mean(tf.cast(correct_prediction, tf.float32), axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 100: loss=2.379924, accuracy=0.125000\n",
      "Step 200: loss=2.372284, accuracy=0.128906\n",
      "Step 300: loss=2.366430, accuracy=0.140625\n",
      "Step 400: loss=2.360768, accuracy=0.121094\n",
      "Step 500: loss=2.347394, accuracy=0.109375\n",
      "Step 600: loss=2.349795, accuracy=0.199219\n",
      "Step 700: loss=2.312950, accuracy=0.218750\n",
      "Step 800: loss=2.284508, accuracy=0.289062\n",
      "Step 900: loss=2.173395, accuracy=0.335938\n",
      "Step 1000: loss=2.115966, accuracy=0.300781\n",
      "Step 1100: loss=1.977837, accuracy=0.441406\n",
      "Step 1200: loss=1.860783, accuracy=0.425781\n",
      "Step 1300: loss=1.866206, accuracy=0.425781\n",
      "Step 1400: loss=1.773057, accuracy=0.402344\n",
      "Step 1500: loss=1.736271, accuracy=0.546875\n",
      "Step 1600: loss=1.626320, accuracy=0.578125\n",
      "Step 1700: loss=1.537970, accuracy=0.539062\n",
      "Step 1800: loss=1.369012, accuracy=0.609375\n",
      "Step 1900: loss=1.286771, accuracy=0.625000\n",
      "Step 2000: loss=1.270916, accuracy=0.640625\n"
     ]
    }
   ],
   "source": [
    "train_data = tf.data.Dataset.from_tensor_slices((input_training, output_training))\n",
    "\n",
    "# NOTE: this doesn't only selectes, it completely transform the structures\n",
    "#   from  <TensorSliceDataset shapes: ((3324,), ()), types: (tf.float32, tf.int32)>\n",
    "#     to  <PrefetchDataset shapes: ((None, 3324), (None,)), types: (tf.float32, tf.int32)>\n",
    "train_data = train_data.repeat().shuffle(5000).batch(batch_size).prefetch(1)\n",
    "\n",
    "display_step = 100\n",
    "training_steps = 2000\n",
    "\n",
    "# Run training for the given number of steps.\n",
    "for step, (input_batch, output_batch) in enumerate(train_data.take(training_steps), 1):\n",
    "    # Run the optimization to update W and b values.\n",
    "    run_optimization(input_batch, output_batch)\n",
    "    \n",
    "    if step % display_step == 0:\n",
    "        pred = neural_net(input_batch, is_training=True)\n",
    "        loss = cross_entropy_loss(pred, output_batch)\n",
    "        acc = accuracy(pred, output_batch)\n",
    "        print(\"Step {}: loss={:f}, accuracy={:f}\".format(step, loss, acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.578994\n"
     ]
    }
   ],
   "source": [
    "# Test model on validation set\n",
    "input_testing = np.array(input_testing)\n",
    "pred = neural_net(input_testing, is_training=False)\n",
    "print(\"Test Accuracy: {:f}\".format(accuracy(pred, output_testing)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# If we consider the \"more probable\" prediction on each item, how well it goes?\n",
    "all_guesses = neural_net(input_testing, is_training=False)\n",
    "guesses_ok = [0] * len(CODE_TYPES)\n",
    "guesses_bad = [0] * len(CODE_TYPES)\n",
    "\n",
    "for guess, real in zip(all_guesses, output_testing):\n",
    "    # guess is an array of len(CODE_TYPES) with a float in each position showing\n",
    "    # which one is the most probable to be real, so we need to get position \n",
    "    # for the max one and check if it matches with the real real :)\n",
    "    position_for_max = np.where(guess == np.amax(guess))[0][0]\n",
    "    if position_for_max == real:\n",
    "        guesses_ok[real] += 1\n",
    "    else:\n",
    "        guesses_bad[real] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 22392 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "values_ok = [guesses_ok[i] for i in range(len(CODE_TYPES))]\n",
    "values_bad = [guesses_bad[i] for i in range(len(CODE_TYPES))]\n",
    "x_positions = np.arange(len(CODE_TYPES))\n",
    "\n",
    "p1 = plt.bar(x_positions, guesses_ok)\n",
    "p2 = plt.bar(x_positions, guesses_bad, bottom=guesses_ok)\n",
    "\n",
    "plt.ylabel('Tipo de código')\n",
    "plt.title('Aciertos o no, por tipo de código')\n",
    "plt.xticks(x_positions, CODE_TYPES, rotation=90)\n",
    "#plt.yticks(np.arange(0, 81, 10))\n",
    "plt.legend((p1[0], p2[0]), ('Correcto', 'Incorrecto'))\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
