Mejoras en cerebro

Pipfile

@@ -14,4 +14,4 @@ nltk = "*"
 spacy = "*"
 
 [requires]
-python_version = "3.9"
+python_version = "3.8"

TODO.md

@@ -1,26 +1,33 @@
 # TODO
 
-## 1. Main modulo
+## Main modulo
 1. Workers para
     1. [x] Revisar Email
     1. [ ] Revisar WhatsApp
     1. [ ] Procesar Texto
 
-## 2. Modulo de Revision de Email
+## Modulo de Revision de Email
 Para revisar si hay mails nuevos, validando que sea de las personas registradas.
 Si no está registrada se avisa a administrador para saber que hacer.
 Limpieza de Inbox y Spam.
 - [x] Revisar lista de emails
 - [x] Revisar si fue revisado
 - [x] Revisar si corresponde a un "Jefe"
+- [x] Confirmar con cerebro si es un comando
 
-## 3. Modulo de WhatsApp
+## Modulo de WhatsApp
 Respuestas a mensajes.
 
-## 4. Modulo de Recordatorio
+## AI para procesar textos
+Spacy permite procesar texto
+Hay que poder "educar" el modelo
+Hay que poder agregar datos
+Hay que poder agregar los correos y mensajes que no se conocen a listado "desconocido"
+
+## Modulo de Recordatorio
 Crear recordatorios y mandar correo o WhatsApp en el momento del recordatorio.
 
-## 5. Modulo de Registro de Eventos
+## Modulo de Registro de Eventos
 Ir llevando un registro de eventos.
 
 ### Otros

config/email.json

@@ -4,7 +4,7 @@
     "port": 993,
     "user": {
       "name": "secretary@incoviba.cl",
-      "password": "quzshqzyfcnydevp"
+      "password": "vgvjuwzwizktdpka"
     },
     "ssl": true
   },
@@ -13,7 +13,7 @@
     "port": 495,
     "user": {
       "name": "secretary@incoviba.cl",
-      "password": "quzshqzyfcnydevp"
+      "password": "vgvjuwzwizktdpka"
     },
     "ssl": true
   },

src/brain/brain.py

@@ -1,12 +1,29 @@
 import os
 import spacy
+from src.instrucciones import Instrucciones
 
 
 class Brain:
     def __init__(self, data_folder):
+        self.folder = data_folder
         self.filename = os.path.join(data_folder, 'brain.json')
-        self.nlp = spacy.load('es_core_news_sm')
+        self.nlp = None
+
+        self.load_nlp(data_folder)
+
+    def load_nlp(self, data_folder):
+        folder = os.path.join(data_folder, 'model')
+        self.nlp = spacy.load(folder)
+
+    def save_model(self):
+        folder = os.path.join(self.folder, 'model')
+        self.nlp.to_disk(folder)
 
     def get_command(self, phrase):
+        doc = self.nlp(phrase)
+        command = max(doc.cats, key=doc.cats.get)
+        return command
+
+    def get_response(self, command, phrase):
         doc = self.nlp(phrase)
         return doc

src/brain/build_model.py

@@ -0,0 +1,15 @@
+import os
+import spacy
+from src.instrucciones import Instrucciones
+
+
+def load_model(commands):
+    nlp = spacy.load('es_core_news_sm')
+    if 'textcat' not in nlp.pipe_names:
+        textcat = nlp.create_pipe('textcat')
+        nlp.add_pipe(textcat)
+        textcat.add_label('test')
+        for c in commands.instrucciones:
+            textcat.add_label(c.instruccion)
+        optimizer = nlp.begin_training()
+    return nlp

src/communication/message.py

@@ -1,8 +1,9 @@
 class Message:
-    def __init__(self, mtype, sender, datetime, text, original):
+    def __init__(self, mtype, sender, datetime, text, original, subject):
         self.type = mtype
         self.sender = sender
         self.datetime = datetime
         self.text = text
         self.checked = False
         self.original = original
+        self.subject = subject

src/email/definitions.py

@@ -1,4 +1,5 @@
-import email
+from email.parser import BytesParser
+from email.policy import default as DefaultPolicy
 
 
 class Email:
@@ -10,8 +11,7 @@ class Email:
         status, raw_data = imap.uid('fetch', self.uid, '(RFC822)')
         if status != 'OK':
             raise Exception('Could not recover message {0}'.format(self.uid))
-
+        self.message = BytesParser(policy=DefaultPolicy).parsebytes(text=raw_data[0][1])
-        self.message = email.message_from_bytes(raw_data[0][1])
 
     def delete(self, imap):
         status, result = imap.uid('STORE', self.uid, '+FLAGS', '(\\Deleted)')

src/email/supervisor.py

@@ -95,9 +95,7 @@ class Email(Thread):
     def run(self) -> None:
         self.start_workers()
         worker = Thread(target=exit_thread, args=(self.params['events']['stop'], self.params['logging']))
-        self.add_worker(worker)
         worker.start()
-        self.worker_status.append(True)
 
         while not self.params['events']['stop'].is_set():
             if not self.check_workers():
@@ -106,3 +104,4 @@ class Email(Thread):
         self.params['logging'].log('Waiting for workers', type(self))
         self.params['events']['log_stop'].set()
         self.join_workers()
+        worker.join()

src/email/workers.py

@@ -9,6 +9,7 @@ import email.utils
 from src.communication import Message
 import json
 from src.functions import dump_queue
+from pprint import pprint
 
 
 class Obtenedor(Worker):
@@ -70,7 +71,7 @@ class Obtenedor(Worker):
             for part in email_part.get_payload():
                 output += self.build_message(part)
         else:
-            html = email_part.get_payload(decode=True)
+            html = email_part.get_payload(decode=True).decode('utf-8')
             bs = BeautifulSoup(html, 'html.parser')
             if bs.body:
                 html = bs.body.get_text()
@@ -94,8 +95,9 @@ class Obtenedor(Worker):
                     if self.is_revisado(em.uid):
                         continue
                     sender = em.message['from']
-                    text = ' '.join([em.message['subject'] + '.'] + self.build_message(em.message))
+                    # text = ' '.join([em.message['subject'] + '.'] + self.build_message(em.message))
-                    msg = Message('email', text=text, original=em, sender=sender,
+                    text = self.build_message(em.message)
+                    msg = Message('email', text=text, original=em, sender=sender, subject=str(em.message['subject']),
                                   datetime=email.utils.parsedate_to_datetime(em.message['Date']))
                     self.queue.put(msg)
                     self.add_revisado(em.uid)
@@ -132,7 +134,7 @@ class Validador(Worker):
         return self.bosses.is_boss(sender)
 
     def validar_instrucciones(self, message):
-        return self.instrucciones.is_valid(message.original.message['subject'])
+        return self.instrucciones.is_valid(message.subject)
 
     def validar(self, message):
         if self.validar_bosses(message.sender):
@@ -245,7 +247,7 @@ class Borrador(Worker):
                 return
 
             for uid in self.borrar:
-                print(uid)
+                print('Borrar ', uid)
                 # status, ids = imap.uid('store', uid, '+FLAGS', b'\\Deleted')
                 # if status != 'OK':
                 #    continue
@@ -281,6 +283,9 @@ class Procesador(Worker):
         self.frec = configs.get('supervisor.wait')
         self.brain = params['brain']
 
+    def cleanup(self):
+        self.brain.save_model()
+
     def run(self) -> None:
         self.start_turn()
         while not self.stop.is_set():
@@ -288,7 +293,10 @@ class Procesador(Worker):
                 em = self.queue.get(timeout=self.frec)
             except queue.Empty:
                 continue
-            print(em.text)
+            command = self.brain.get_command(em.subject)
-            command = self.brain.get_command(em.text)
+            pprint((em.subject, command))
-            [print(ent) for ent in command.ents]
+            for t in em.text:
+                contenido = self.brain.get_response(command, t)
+
+        self.cleanup()
         self.end_turn()

src/instrucciones.py

@@ -10,14 +10,15 @@ class Command:
 class Commands:
     def __init__(self, data_folder):
         self.filename = os.path.join(data_folder, 'commands.json')
-        data = []
-        try:
-            with open(self.filename, 'r') as f:
-                data = json.load(f)
-        except FileNotFoundError:
-            pass
 
         self.commands = []
+        self.load_commands(self.filename)
+
+    def load_commands(self, filename):
+        data = []
+        if os.path.isfile(filename):
+            with open(filename, 'r') as f:
+                data = json.load(f)
         for c in data:
             cmd = Command()
             cmd.command = c
@@ -36,7 +37,6 @@ class Commands:
 class Instruccion:
     def __init__(self):
         self.instruccion = ''
-        self.aliases = []
         self.command = None
         self.params = {}
 
@@ -44,22 +44,20 @@ class Instruccion:
 class Instrucciones:
     def __init__(self, data_folder):
         self.filename = os.path.join(data_folder, 'instrucciones.json')
-        data = []
-        try:
-            with open(self.filename, 'r') as f:
-                data = json.load(f)
-        except FileNotFoundError:
-            pass
-
         self.commands = Commands(data_folder)
 
         self.instrucciones = []
+        self.load_instrucciones(self.filename)
+        self.idx = 0
+
+    def load_instrucciones(self, filename):
+        data = []
+        if os.path.isfile(filename):
+            with open(filename, 'r') as f:
+                data = json.load(f)
         for d in data:
             i = Instruccion()
             i.instruccion = d['name']
-            if 'aliases' in d:
-                for a in d['aliases']:
-                    i.aliases.append(a)
             if 'params' in d:
                 for param, val in d['params'].items():
                     i.params[param] = val
@@ -73,8 +71,6 @@ class Instrucciones:
         for i, ins in enumerate(self.instrucciones):
             if instruccion == ins.instruccion:
                 return i
-            if instruccion in ins.aliases:
-                return i
 
     def find(self, instruccion):
         if not self.is_valid(instruccion):
@@ -85,23 +81,16 @@ class Instrucciones:
         for i in self.instrucciones:
             if instruccion == i.instruccion:
                 return True
-            if instruccion in i.aliases:
-                return True
         return False
 
     def add(self, instruccion, aliases: list = None):
         if self.is_valid(instruccion):
-            if aliases is not None:
-                i = self.get(instruccion)
-                self.instrucciones[i].aliases = aliases
             return
         ins = Instruccion()
         ins.instruccion = instruccion
-        if aliases is not None:
-            ins.aliases = aliases
         self.instrucciones.append(ins)
 
     def save(self):
-        data = [{'instruccion': i.instruccion, 'aliases': i.aliases} for i in self.instrucciones]
+        data = [{'instruccion': i.instruccion, 'params': i.params.items()} for i in self.instrucciones]
         with open(self.filename, 'w') as f:
             json.dump(data, f, indent=4)