Se actualizaron comandos para poder conseguir de mejor manera las métricas de los cablemodems. Se utiliza una librería Graze\ParallelProcess para forkear si se decide unos comandos.

app/config/parameters.yml.dist

@@ -68,3 +68,8 @@ parameters:
     
     # App Dummy URL
     env(HOST_DUMMY): www.flowdat.com
+
+    # stats.cm.method [getMetricsFork, getMetricsAMQP]
+    stats.cm.method: getMetricsAMQP
+    stats.cm.max.subprocess: 7
+    stats.cm.max.cms: 25

app/config/parameters.yml.docker

@@ -68,3 +68,7 @@ parameters:
       port: 8125
     endpoint.mysql.connection: 'http://jsonep_mysql:8000'
     endpoint.mongodb.connection: 'http://jsonep_mongo:8000'
+    # stats.cm.method [getMetricsFork, getMetricsAMQP]
+    stats.cm.method: getMetricsAMQP
+    stats.cm.max.subprocess: 7
+    stats.cm.max.cms: 25

composer.json

@@ -183,7 +183,7 @@
         "doctrine/doctrine-cache-bundle": "^1.2",
         "doctrine/doctrine-migrations-bundle": "^1.2",
         "doctrine/orm": "^2.5",
-        "jdorn/sql-formatter": "~1.2.17",
+        "graze/parallel-process": "^0.6.0",
         "ik/audit-bundle": "dev-master",
         "ik/auth-bundle": "dev-master",
         "ik/base-admin-bundle": "dev-master",
@@ -195,17 +195,18 @@
         "ik/owner-voter-bundle": "dev-master",
         "ik/redis-bundle": "dev-master",
         "ik/stats/base-stats-bundle": "dev-master",
+        "ik/stats/cmts-bundle": "dev-master",
         "ik/stats/fiberhome-bundle": "dev-master",
+        "ik/stats/fiberlink-bundle": "dev-master",
         "ik/stats/geoserver-bundle": "dev-master",
         "ik/stats/huawei-bundle": "dev-master",
-        "ik/stats/fiberlink-bundle": "dev-master",
+        "ik/stats/nas-bundle": "dev-master",
         "ik/stats/zte-bundle": "dev-master",
         "ik/statsd-bundle": "dev-master",
         "ik/webservice-bundle": "dev-master",
         "ik/workflow-bundle": "dev-master",
-        "ik/stats/nas-bundle": "dev-master",
-        "ik/stats/cmts-bundle": "dev-master",
         "incenteev/composer-parameter-handler": "^2.0",
+        "jdorn/sql-formatter": "~1.2.17",
         "jms/serializer": "^1.9.0",
         "jms/serializer-bundle": "^1.5.0",
         "kriswallsmith/assetic": "^1.4",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
         "This file is @generated automatically"
     ],
-    "content-hash": "725e16b6ddb290fa2a513fdfa3c5e876",
+    "content-hash": "459b2183499a5a4ee718beb040889f81",
     "packages": [
         {
             "name": "behat/transliterator",
@@ -1214,6 +1214,231 @@
             ],
             "time": "2015-01-21T20:55:10+00:00"
         },
+        {
+            "name": "graze/console-diff-renderer",
+            "version": "0.6.2",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/graze/console-diff-renderer.git",
+                "reference": "aafdaf504a96e6889f284bb15c75330318318df1"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/graze/console-diff-renderer/zipball/aafdaf504a96e6889f284bb15c75330318318df1",
+                "reference": "aafdaf504a96e6889f284bb15c75330318318df1",
+                "shasum": ""
+            },
+            "require": {
+                "php": "^5.6 | ^7.0",
+                "symfony/console": "^2.8 | ^3.2 | ^4.0"
+            },
+            "require-dev": {
+                "graze/standards": "^2.0",
+                "mockery/mockery": "^1",
+                "phpunit/phpunit": "^5.7.21 | ^6 | ^7",
+                "squizlabs/php_codesniffer": "^3"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Graze\\DiffRenderer\\": "src"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Harry Bragg",
+                    "email": "harry.bragg@graze.com",
+                    "role": "Developer"
+                },
+                {
+                    "name": "Graze Developers",
+                    "email": "developers@graze.com",
+                    "homepage": "http://www.graze.com",
+                    "role": "Development Team"
+                }
+            ],
+            "description": "only write output to a console that has changed",
+            "homepage": "https://github.com/graze/console-diff-renderer",
+            "keywords": [
+                "console-diff-renderer",
+                "graze"
+            ],
+            "time": "2018-06-22T14:35:55+00:00"
+        },
+        {
+            "name": "graze/data-structure",
+            "version": "2.1.0",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/graze/data-structure.git",
+                "reference": "24e0544b7828f65b1b93ce69ad702c9efb4a64d0"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/graze/data-structure/zipball/24e0544b7828f65b1b93ce69ad702c9efb4a64d0",
+                "reference": "24e0544b7828f65b1b93ce69ad702c9efb4a64d0",
+                "shasum": ""
+            },
+            "require": {
+                "graze/sort": "~2.0",
+                "php": ">=5.5|^7.0"
+            },
+            "require-dev": {
+                "graze/standards": "^2.0",
+                "phpunit/phpunit": "^4.2 | ^5.2",
+                "squizlabs/php_codesniffer": "^3"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Graze\\DataStructure\\": "src"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Graze tech team",
+                    "homepage": "https://github.com/graze/data-structure/graphs/contributors"
+                }
+            ],
+            "description": "Data collections and containers",
+            "homepage": "https://github.com/graze/data-structure",
+            "keywords": [
+                "array",
+                "collection",
+                "container",
+                "data",
+                "filter",
+                "map",
+                "parameters",
+                "reduce",
+                "structure"
+            ],
+            "time": "2017-11-29T09:06:31+00:00"
+        },
+        {
+            "name": "graze/parallel-process",
+            "version": "0.6",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/graze/parallel-process.git",
+                "reference": "71b580267b273124b16d4346a0f8413443329284"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/graze/parallel-process/zipball/71b580267b273124b16d4346a0f8413443329284",
+                "reference": "71b580267b273124b16d4346a0f8413443329284",
+                "shasum": ""
+            },
+            "require": {
+                "graze/console-diff-renderer": "^0.6.1",
+                "graze/data-structure": "^2.0",
+                "php": "^5.5 | ^7.0",
+                "symfony/process": "^2.8 | ^3.2 | ^4.0"
+            },
+            "require-dev": {
+                "graze/standards": "^2",
+                "mockery/mockery": "^1.0",
+                "phpunit/phpunit": "^5.7.21|^6|^7",
+                "squizlabs/php_codesniffer": "^3",
+                "symfony/console": "^3.1 | ^4"
+            },
+            "suggest": {
+                "symfony/console": "To use the Table to print current runs"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Graze\\ParallelProcess\\": "src"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Harry Bragg",
+                    "email": "harry.bragg@graze.com",
+                    "role": "Developer"
+                },
+                {
+                    "name": "Graze Developers",
+                    "email": "developers@graze.com",
+                    "homepage": "http://www.graze.com",
+                    "role": "Development Team"
+                }
+            ],
+            "description": "run a pool of processes simultaneously",
+            "homepage": "https://github.com/graze/parallel-process",
+            "keywords": [
+                "graze",
+                "parallel-process"
+            ],
+            "time": "2018-05-22T15:29:04+00:00"
+        },
+        {
+            "name": "graze/sort",
+            "version": "2.0.1",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/graze/sort.git",
+                "reference": "50f0896363f177f68be248d7bad9eb0c2f7f666c"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/graze/sort/zipball/50f0896363f177f68be248d7bad9eb0c2f7f666c",
+                "reference": "50f0896363f177f68be248d7bad9eb0c2f7f666c",
+                "shasum": ""
+            },
+            "require": {
+                "php": ">=5.3"
+            },
+            "require-dev": {
+                "adlawson/timezone": "~1.0",
+                "phpunit/phpunit": "~4.0"
+            },
+            "type": "library",
+            "autoload": {
+                "files": [
+                    "src/fn.php"
+                ],
+                "psr-4": {
+                    "Graze\\": "src"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Graze tech team",
+                    "homepage": "https://github.com/graze/sort/graphs/contributors"
+                }
+            ],
+            "description": "A collection of array sorting transforms and functions",
+            "homepage": "https://github.com/graze/sort",
+            "keywords": [
+                "array",
+                "collection",
+                "list",
+                "order",
+                "ordered",
+                "schwartzian",
+                "sort",
+                "sorting",
+                "transform"
+            ],
+            "time": "2014-09-23T17:01:23+00:00"
+        },
         {
             "name": "hwi/oauth-bundle",
             "version": "0.5.3",
@@ -6554,22 +6779,22 @@
         "ik/owner-voter-bundle": 20,
         "ik/redis-bundle": 20,
         "ik/stats/base-stats-bundle": 20,
+        "ik/stats/cmts-bundle": 20,
         "ik/stats/fiberhome-bundle": 20,
+        "ik/stats/fiberlink-bundle": 20,
         "ik/stats/geoserver-bundle": 20,
         "ik/stats/huawei-bundle": 20,
-        "ik/stats/fiberlink-bundle": 20,
+        "ik/stats/nas-bundle": 20,
         "ik/stats/zte-bundle": 20,
         "ik/statsd-bundle": 20,
         "ik/webservice-bundle": 20,
         "ik/workflow-bundle": 20,
-        "ik/stats/nas-bundle": 20,
-        "ik/stats/cmts-bundle": 20,
         "opensolutions/oss-snmp": 20,
         "sonata-project/admin-bundle": 20,
         "voryx/restgeneratorbundle": 20
     },
     "prefer-stable": false,
-    "prefer-lowest": true,
+    "prefer-lowest": false,
     "platform": {
         "php": ">=5.5.9"
     },

src/StatsBundle/Command/GenerateCrontabCommand.php

@@ -110,7 +110,7 @@ class GenerateCrontabCommand extends ContainerAwareCommand
                 
                 fwrite($handle, PHP_EOL."# OLT {$oltName} ({$deviceIp})".PHP_EOL);
                 
-                fwrite($handle, implode(PHP_EOL,$commands));
+                fwrite($handle, implode(PHP_EOL,$commands).PHP_EOL.PHP_EOL);
             }
             
             if($this->getContainer()->getParameter('geoserver_service')) {
@@ -122,7 +122,7 @@ class GenerateCrontabCommand extends ContainerAwareCommand
                 
                 fwrite($handle, implode(PHP_EOL,$commands));
                 
-                fwrite($handle, PHP_EOL);
+                fwrite($handle, PHP_EOL.PHP_EOL);
             }
 
             // APARTADO NAS
@@ -155,7 +155,7 @@ class GenerateCrontabCommand extends ContainerAwareCommand
                 
                 fwrite($handle, PHP_EOL.PHP_EOL."# NAS {$description} ({$deviceIp})".PHP_EOL);
                 fwrite($handle, PHP_EOL);
-                fwrite($handle, implode(PHP_EOL,$commands));
+                fwrite($handle, implode(PHP_EOL,$commands).PHP_EOL.PHP_EOL);
                 
             }
 
@@ -183,9 +183,12 @@ class GenerateCrontabCommand extends ContainerAwareCommand
                 
                 $library = $data['library'];
                 $snmpCommunity = $data['snmpCommunity'];
+                $docsVersion = $data['docsVersion'];
                 $description = $data['name'];
                 $deviceIp = $device->getIp();
                 $timeScan = $data['timeScan'];
+                $timeOctets = $data['timeCmtsOctets'];
+                (isset($data['timeCmStats']))? $timeCmStats = $data['timeCmStats'] : $timeCmStats = 10;
 
                 $params = "--cmts-ip={$deviceIp} --cmts-community={$snmpCommunity} --cmts-snmp-library={$library} --cmts-device-id={$deviceId} --cmts-server-id={$serverId}";
                 
@@ -196,14 +199,19 @@ class GenerateCrontabCommand extends ContainerAwareCommand
                 
                 $commands[] = "*/{$timeScan} * * * * {$pathConsole} cmts:interface:stats {$params}";
                 $commands[] = "*/{$timeScan} * * * * {$pathConsole} cmts:interface:description {$params}";
+
+                $params .= " --cmts-docs={$docsVersion}";
+
+                $commands[] = "*/{$timeOctets} * * * * {$pathConsole}  cmts:octets {$params}";
                 
                 $params = "--cmts-device-id={$deviceId} --cmts-server-id={$serverId}";
 
                 $commands[] = "*/{$timeScan} * * * * {$pathConsole} stats:interface {$params}";
+                $commands[] = "*/{$timeCmStats} * * * * {$pathConsole} stats:cm {$params}";
                 
                 fwrite($handle, PHP_EOL.PHP_EOL."# CMTS {$description} ({$deviceIp})".PHP_EOL);
                 fwrite($handle, PHP_EOL);
-                fwrite($handle, implode(PHP_EOL,$commands));
+                fwrite($handle, implode(PHP_EOL,$commands).PHP_EOL.PHP_EOL);
             }
         }
 

src/StatsBundle/Command/GenerateRemoteCrontabCommand.php

@@ -151,6 +151,8 @@ class GenerateRemoteCrontabCommand extends ContainerAwareCommand
                 $content .= PHP_EOL . "# OLT {$oltName} ({$deviceIp})" . PHP_EOL;
 
                 $content .= implode(PHP_EOL, $commands);
+
+                $content .= PHP_EOL . PHP_EOL;
             }
 
             if ($this->getContainer()->getParameter('geoserver_service')) {
@@ -164,7 +166,7 @@ class GenerateRemoteCrontabCommand extends ContainerAwareCommand
                 $commands[] = "*/10 * * * * {$pathConsole} {$amqpRemote} {$routing_key} stats:ponport:geo {$params}";
 
                 $content .= implode(PHP_EOL, $commands);
-                $content .= PHP_EOL;
+                $content .= PHP_EOL . PHP_EOL;
             }
 
             // APARTADO NAS
@@ -201,7 +203,8 @@ class GenerateRemoteCrontabCommand extends ContainerAwareCommand
                 $commands[] = "*/10 * * * * {$pathConsole} {$amqpRemote} {$routing_key} nas:onu:octets {$params}";
                 $content .= PHP_EOL . "# NAS {$description} ({$deviceIp})" . PHP_EOL;
 
-                $content .= implode(PHP_EOL, $commands) . PHP_EOL.PHP_EOL;
+                $content .= implode(PHP_EOL, $commands);
+                $content .= PHP_EOL . PHP_EOL;
             }
 
 
@@ -234,6 +237,8 @@ class GenerateRemoteCrontabCommand extends ContainerAwareCommand
                 $deviceIp = $device->getIp();
                 $timeScan = $data['timeScan'];
                 $timeOctets = $data['timeCmtsOctets'];
+                
+                (isset($data['timeCmStats']))? $timeCmStats = $data['timeCmStats'] : $timeCmStats = 10;
 
                 $params = "--cmts-ip={$deviceIp} --cmts-community={$snmpCommunity} --cmts-snmp-library={$library} --cmts-device-id={$deviceId} --cmts-server-id={$serverId}";
                 if ($amqp) 
@@ -266,18 +271,21 @@ class GenerateRemoteCrontabCommand extends ContainerAwareCommand
                 $commands[] = "*/{$timeScan} * * * * {$pathConsole} {$amqpRemote} {$routing_key} stats:interface {$params}";
                 
                 if ($amqp) {
-                    $params .= " --args=--generate-remote-commands:0";
+                    $params .= " --args=--generate-remote-commands:1";
                 } else {
-                    $params .= " --generate-remote-commands=0";
+                    $params .= " --generate-remote-commands=1";
                 }
-                $commands[] = "*/{$timeScan} * * * * {$pathConsole} {$amqpRemote} {$routing_key} stats:cm {$params}";
+                $commands[] = "*/{$timeCmStats} * * * * {$pathConsole} {$amqpRemote} {$routing_key} stats:cm {$params}";
                 
                 $content .= PHP_EOL . "# CMTS {$description} ({$deviceIp})" . PHP_EOL;
 
-                $content .= implode(PHP_EOL, $commands) . PHP_EOL.PHP_EOL;
+                $content .= implode(PHP_EOL, $commands);
+                $content .= PHP_EOL . PHP_EOL;
             }
 
         }
+
+        $content .= PHP_EOL."#END ".PHP_EOL.PHP_EOL;
         file_put_contents($fileCrontab, $content);
 
         print_r(shell_exec("/etc/init.d/cron reload"));

src/StatsBundle/Command/StatsCmCommand.php

@@ -12,6 +12,11 @@ use Symfony\Bundle\FrameworkBundle\Console\Application;
 use Symfony\Component\Console\Input\ArgvInput;
 use Symfony\Component\Console\Output\BufferedOutput;
 
+use Symfony\Component\Process\Process;
+use Symfony\Component\Process\Exception\ProcessFailedException;
+
+use Graze\ParallelProcess\Pool;
+
 class StatsCmCommand extends BaseCommand
 {
 
@@ -144,15 +149,38 @@ class StatsCmCommand extends BaseCommand
         }
             
         if($generateCommands) {
+            $method = 'getMetricsAMQP';
+            $maxSubprocess = 7;
+            $maxCms = 25;
+            
+            /* app/config/parameters.yml */
+            if($this->getContainer()->hasParameter('stats.cm.method')) $method = $this->getContainer()->getParameter('stats.cm.method');
+            if($this->getContainer()->hasParameter('stats.cm.max.subprocess')) $maxSubprocess = $this->getContainer()->getParameter('stats.cm.max.subprocess');
+            if($this->getContainer()->hasParameter('stats.cm.max.cms')) $maxCms = $this->getContainer()->getParameter('stats.cm.max.cms');
+            
             $inicio = microtime(true);
-            $this->generateCommandsAMQP($cmsCached, $cmtsDeviceId, $cmtsServerId);
-            $fin = microtime(true);
-            $time = $fin - $inicio;
-            $this->output->writeln("Generación comandos en AMQP: $time segundos");
+            if($method == 'getMetricsAMQP') {
+                $this->getMetricsAMQP($cmsCached, $cmtsDeviceId, $cmtsServerId, $maxCms);
+                $fin = microtime(true);
+                $time = $fin - $inicio;
+                $this->output->writeln("Generación comandos en AMQP: $time segundos");
+            } elseif($method == 'getMetricsFork') {
+                $this->getMetricsFork($cmsCached, $cmtsDeviceId, $cmtsServerId, $maxSubprocess, $maxCms);
+                $fin = microtime(true);
+                $time = $fin - $inicio;
+                $this->output->writeln("Procesamos los cablemodems al momento: $time segundos");
+            } else {
+                $this->output->writeln("$method no existe[getMetricsAMQP|getMetricsFork], no se consultaron métricas a los cablemodems.");
+            }
+            
+            
         }
 
     }
 
+    /*
+     *  Un comando amqp por cada cablemodem a consultar
+     */
     public function generateCommandsAMQP($cmsCached, $cmtsDeviceId, $cmtsServerId) 
     {
         foreach($cmsCached as $index => $info) {
@@ -175,6 +203,107 @@ class StatsCmCommand extends BaseCommand
 
     }
 
+    /*
+     *  Se generan comandos para el amqp con maxMacs cada uno > Podemos forkear en el amqp   
+     */
+    public function getMetricsAMQP($cmsCached, $cmtsDeviceId, $cmtsServerId, $maxCms = 20) 
+    {
+        $cmd_args = $base_args = array();
+        $base_args[0] = "--cmts-device-id:{$cmtsDeviceId}";
+        $base_args[1] = "--cmts-server-id:{$cmtsServerId}";
+        $base_args[2] = "--cm-community:public";
+        
+        $cmd = "cmts:cm:stats";
+
+        $cmd_args = $base_args;
+        $count = 1;
+
+        foreach($cmsCached as $index => $info) {
+            if($info['ip'] == '0.0.0.0') continue;
+
+            $ip = $info['ip'];
+            $mac = strtolower($info['mac']);
+            
+            $cmd_args[] = "--cm:{$ip},{$mac}";
+            if($count >= $maxCms) {
+                
+                $this->runCommand($cmd, $cmd_args);
+                $cmd_args = $base_args;
+                $count = 0;
+            } else {
+                $count++;
+            }
+        }
+
+        if(count($cmd_args > 3)) {
+            $this->runCommand($cmd, $cmd_args);
+        }
+
+    }
+
+    /*
+     *  Procesamos en el momento las stats de cm por el cmts:cm:stats > 7 forks y cada uno 25 cm
+     */
+    public function getMetricsFork($cmsCached, $cmtsDeviceId, $cmtsServerId, $maxSubprocess = 7, $maxCms = 25) 
+    {
+        $pool = new Pool();
+        
+        $exception = function(Process $process, $duration) {throw new ProcessFailedException($process);};
+        
+        $pool->setOnFailure($exception);
+        $pool->setMaxSimultaneous($maxSubprocess);
+
+        $cmd_args = array();
+        $cmd_args[0] = "--cmts-device-id:{$cmtsDeviceId}";
+        $cmd_args[1] = "--cmts-server-id:{$cmtsServerId}";
+        $cmd_args[2] = "--cm-community:public";
+        
+        $cm_args = array();
+        $count = 1;
+
+        $path = "/opt/stats/bin/console";
+        $cmd = "cmts:cm:stats";
+
+        /* $args = "--cmts-device-id={$cmtsDeviceId} --cmts-server-id={$cmtsServerId} --cm-community={$cmCommunity} --cm-snmp-library={$cmSnmpLibrary} --save-historic={$saveHistoric}"; */
+        $args = "--cmts-device-id={$cmtsDeviceId} --cmts-server-id={$cmtsServerId} --save-historic=1";
+
+        foreach($cmsCached as $index => $info) {
+            if($info['ip'] == '0.0.0.0') continue;
+
+            $ip = $info['ip'];
+            $mac = strtolower($info['mac']);
+            
+            $cm_args[] = "--cm={$ip},{$mac}";
+            if($count >= $maxCms) {
+                
+                $cablemodems = implode(" ",$cm_args);
+                $process = "{$path} {$cmd} {$args} {$cablemodems}";
+                
+                $pool->add(new Process($process));
+
+                $cm_args = array();
+                $count = 0;
+            } else {
+                $count++;
+            }
+        }
+
+        if($cm_args) {
+            
+            $cablemodems = implode(" ",$cm_args);
+            $process = "{$path} {$cmd} {$args} {$cablemodems}";
+            $pool->add(new Process($process));
+
+        }
+
+        try {
+            $pool->run(); // blocking that will run till it finishes
+        } catch(\ProcessFailedException $e) {
+            print_r($e);
+        }
+
+    }
+
     public function runCommand($name, $cmd_args = array())
     {
 

src/StatsBundle/Resources/translations/StatsBundle.es.yml

@@ -80,6 +80,7 @@ form:
     label_signal_cmts: SNR Ups
     label_microreflection_cmts: Microreflection Ups
     label_rx_power_olt: RX Power Port
+    label_time_cm_stats: Estadísticas y SLA de CMs
 
 show:
     label_name: Nombre