From d35bd068020224bda130110b28abc9fd20bad44a Mon Sep 17 00:00:00 2001 From: Nick McKinney Date: Thu, 2 Feb 2023 10:41:56 -0500 Subject: [PATCH] feat(inputs.opensearch_query): add opensearch_query input plugin (#12393) --- docs/LICENSE_OF_DEPENDENCIES.md | 1 + go.mod | 1 + go.sum | 14 + plugins/inputs/all/opensearch_query.go | 5 + plugins/inputs/opensearch_query/README.md | 258 +++++++ .../opensearch_query/aggregation.bucket.go | 42 ++ .../inputs/opensearch_query/aggregation.go | 66 ++ .../opensearch_query/aggregation.metric.go | 18 + .../opensearch_query/aggregation.response.go | 143 ++++ .../opensearch_query/opensearch_query.go | 269 +++++++ .../opensearch_query/opensearch_query_test.go | 695 ++++++++++++++++++ plugins/inputs/opensearch_query/query.go | 44 ++ plugins/inputs/opensearch_query/sample.conf | 59 ++ .../opensearch_query/testdata/nginx_logs | 500 +++++++++++++ 14 files changed, 2115 insertions(+) create mode 100644 plugins/inputs/all/opensearch_query.go create mode 100755 plugins/inputs/opensearch_query/README.md create mode 100644 plugins/inputs/opensearch_query/aggregation.bucket.go create mode 100644 plugins/inputs/opensearch_query/aggregation.go create mode 100644 plugins/inputs/opensearch_query/aggregation.metric.go create mode 100644 plugins/inputs/opensearch_query/aggregation.response.go create mode 100644 plugins/inputs/opensearch_query/opensearch_query.go create mode 100644 plugins/inputs/opensearch_query/opensearch_query_test.go create mode 100644 plugins/inputs/opensearch_query/query.go create mode 100644 plugins/inputs/opensearch_query/sample.conf create mode 100644 plugins/inputs/opensearch_query/testdata/nginx_logs diff --git a/docs/LICENSE_OF_DEPENDENCIES.md b/docs/LICENSE_OF_DEPENDENCIES.md index 76a96dfeb..d3a523ef1 100644 --- a/docs/LICENSE_OF_DEPENDENCIES.md +++ b/docs/LICENSE_OF_DEPENDENCIES.md @@ -257,6 +257,7 @@ following works: - github.com/opencontainers/go-digest [Apache License 2.0](https://github.com/opencontainers/go-digest/blob/master/LICENSE) - github.com/opencontainers/image-spec [Apache License 2.0](https://github.com/opencontainers/image-spec/blob/master/LICENSE) - github.com/opencontainers/runc [Apache License 2.0](https://github.com/opencontainers/runc/blob/main/LICENSE) +- github.com/opensearch-project/opensearch-go [Apache License 2.0](https://github.com/opensearch-project/opensearch-go/blob/main/LICENSE.txt) - github.com/opentracing/opentracing-go [Apache License 2.0](https://github.com/opentracing/opentracing-go/blob/master/LICENSE) - github.com/pborman/ansi [BSD 3-Clause "New" or "Revised" License](https://github.com/pborman/ansi/blob/master/LICENSE) - github.com/philhofer/fwd [MIT License](https://github.com/philhofer/fwd/blob/master/LICENSE.md) diff --git a/go.mod b/go.mod index 9b899faa1..b9b703835 100644 --- a/go.mod +++ b/go.mod @@ -128,6 +128,7 @@ require ( github.com/nsqio/go-nsq v1.1.0 github.com/olivere/elastic v6.2.37+incompatible github.com/openconfig/gnmi v0.0.0-20220920173703-480bf53a74d2 + github.com/opensearch-project/opensearch-go/v2 v2.1.0 github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5 github.com/openzipkin/zipkin-go v0.2.5 diff --git a/go.sum b/go.sum index 9b3dc980f..1894cc67e 100644 --- a/go.sum +++ b/go.sum @@ -362,6 +362,7 @@ github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/ github.com/aws/aws-sdk-go v1.38.3/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.38.68/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.40.45/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= +github.com/aws/aws-sdk-go v1.44.45/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.7.0/go.mod h1:tb9wi5s61kTDA5qCkcDbt3KRVV74GGslQkl/DRdX/P4= github.com/aws/aws-sdk-go-v2 v1.8.1/go.mod h1:xEFuWz+3TYdlPRuo+CqATbeDWIWyaT5uAPwPaWtgse0= @@ -370,6 +371,7 @@ github.com/aws/aws-sdk-go-v2 v1.9.1/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVj github.com/aws/aws-sdk-go-v2 v1.9.2/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= github.com/aws/aws-sdk-go-v2 v1.11.0/go.mod h1:SQfA+m2ltnu1cA0soUkj4dRSsmITiVQUJvBIZjzfPyQ= github.com/aws/aws-sdk-go-v2 v1.11.2/go.mod h1:SQfA+m2ltnu1cA0soUkj4dRSsmITiVQUJvBIZjzfPyQ= +github.com/aws/aws-sdk-go-v2 v1.16.6/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw= github.com/aws/aws-sdk-go-v2 v1.16.10/go.mod h1:WTACcleLz6VZTp7fak4EO5b9Q4foxbn+8PIz3PmyKlo= github.com/aws/aws-sdk-go-v2 v1.16.16/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k= github.com/aws/aws-sdk-go-v2 v1.17.1/go.mod h1:JLnGeGONAyi2lWXI1p0PCIOIy333JMVK1U7Hf0aRFLw= @@ -381,11 +383,13 @@ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8/go.mod h1:JTnlBSot9 github.com/aws/aws-sdk-go-v2/config v1.6.1/go.mod h1:t/y3UPu0XEDy0cEw6mvygaBQaPzWiYAxfP2SzgtvclA= github.com/aws/aws-sdk-go-v2/config v1.8.3/go.mod h1:4AEiLtAb8kLs7vgw2ZV3p2VZ1+hBavOc84hqxVNpCyw= github.com/aws/aws-sdk-go-v2/config v1.10.1/go.mod h1:auIv5pIIn3jIBHNRcVQcsczn6Pfa6Dyv80Fai0ueoJU= +github.com/aws/aws-sdk-go-v2/config v1.15.12/go.mod h1:oxRNnH11J580bxDEXyfTqfB3Auo2fxzhV052LD4HnyA= github.com/aws/aws-sdk-go-v2/config v1.17.8 h1:b9LGqNnOdg9vR4Q43tBTVWk4J6F+W774MSchvKJsqnE= github.com/aws/aws-sdk-go-v2/config v1.17.8/go.mod h1:UkCI3kb0sCdvtjiXYiU4Zx5h07BOpgBTtkPu/49r+kA= github.com/aws/aws-sdk-go-v2/credentials v1.3.3/go.mod h1:oVieKMT3m9BSfqhOfuQ+E0j/yN84ZAJ7Qv8Sfume/ak= github.com/aws/aws-sdk-go-v2/credentials v1.4.3/go.mod h1:FNNC6nQZQUuyhq5aE5c7ata8o9e4ECGmS4lAXC7o1mQ= github.com/aws/aws-sdk-go-v2/credentials v1.6.1/go.mod h1:QyvQk1IYTqBWSi1T6UgT/W8DMxBVa5pVuLFSRLLhGf8= +github.com/aws/aws-sdk-go-v2/credentials v1.12.7/go.mod h1:8b1nSHdDaKLho9VEK+K8WivifA/2K5pPm4sfI21NlQ8= github.com/aws/aws-sdk-go-v2/credentials v1.12.21/go.mod h1:O+4XyAt4e+oBAoIwNUYkRg3CVMscaIJdmZBOcPgJ8D8= github.com/aws/aws-sdk-go-v2/credentials v1.13.2 h1:F/v1w0XcFDZjL0bCdi9XWJenoPKjGbzljBhDKcryzEQ= github.com/aws/aws-sdk-go-v2/credentials v1.13.2/go.mod h1:eAT5aj/WJ2UDIA0IVNFc2byQLeD89SDEi4cjzH/MKoQ= @@ -394,6 +398,7 @@ github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.2.0/go.mod h1:UV github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.4.1/go.mod h1:+GTydg3uHmVlQdkRoetz6VHKbOMEYof70m19IpMLifc= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.6.0/go.mod h1:gqlclDEZp4aqJOancXK6TN24aKhT0W0Ae9MHk3wzTMM= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.0/go.mod h1:5E1J3/TTYy6z909QNR0QnXGBpfESYGDqd3O0zqONghU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.7/go.mod h1:81k6q0UUZj6AdQZ1E/VQ27cLrTUpJGraZR6/hVHRxjE= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17/go.mod h1:yIkQcCDYNsZfXpd5UX2Cy+sWA1jPgIhGTw9cOBzfVnQ= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 h1:E3PXZSI3F2bzyj6XxUXdTIfvp425HHhwKsFvmzBwHgs= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19/go.mod h1:VihW95zQpeKQWVPGkwT+2+WJNQV8UXFfMTWdU6VErL8= @@ -401,12 +406,14 @@ github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.7.1 h1:p9Dys1g2YdaqMalnp6AwCA github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.7.1/go.mod h1:wN/mvkow08GauDwJ70jnzJ1e+hE+Q3Q7TwpYLXOe9oI= github.com/aws/aws-sdk-go-v2/internal/configsources v1.0.4/go.mod h1:W5gGbtNXFpF9/ssYZTaItzG/B+j0bjTnwStiCP2AtWU= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.0/go.mod h1:NO3Q5ZTTQtO2xIg2+xTXYDiT7knSejfeDm7WGDaOo0U= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.13/go.mod h1:wLLesU+LdMZDM3U0PP9vZXJW39zmD/7L4nY2pSrYZ/g= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.17/go.mod h1:6qtGip7sJEyvgsLjphRZWF9qPe3xJf1mL/MM01E35Wc= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23/go.mod h1:2DFxAQ9pfIRy0imBCJv+vZ2X6RKxves6fbnEuSry6b4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25/go.mod h1:Zb29PYkf42vVYQY6pvSyJCJcFHlPIiY+YKdPtwnvMkY= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 h1:I3cakv2Uy1vNmmhRQmFptYDxOvBnwCdNwyw63N0RaRU= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27/go.mod h1:a1/UpzeyBBerajpnP5nGZa9mGzsBn5cOKxm6NWQsvoI= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.0/go.mod h1:anlUzBoEWglcUxUQwZA7HQOEVEnQALVZsizAapB2hq8= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.7/go.mod h1:93Uot80ddyVzSl//xEJreNKMhxntr71WtR3v/A1cRYk= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.11/go.mod h1:cYAfnB+9ZkmZWpQWmPDsuIGm4EA+6k2ZVtxKjw/XJBY= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17/go.mod h1:pRwaTYCJemADaqCbUAxltMoHKata7hmB5PjEXeu0kfg= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19/go.mod h1:6Q0546uHDp421okhmmGfbxzq2hBqbXFNpi4k+Q1JnQA= @@ -415,6 +422,7 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21/go.mod h1:+Gxn8jYn5k9 github.com/aws/aws-sdk-go-v2/internal/ini v1.2.1/go.mod h1:Pv3WenDjI0v2Jl7UaMFIIbPOBbhn33RmmAmGgkXDoqY= github.com/aws/aws-sdk-go-v2/internal/ini v1.2.4/go.mod h1:ZcBrrI3zBKlhGFNYWvju0I3TR93I7YIgAfy82Fh4lcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.0/go.mod h1:6oXGy4GLpypD3uCh8wcqztigGgmhLToMfjavgh+VySg= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.14/go.mod h1:R1HF8ZDdcRFfAGF+13En4LSHi2IrrNuPQCaxgWCeGyY= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24 h1:wj5Rwc05hvUSvKuOF29IYb9QrCLjU+rHAy/x/o0DK2c= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24/go.mod h1:jULHjqqjDlbyTa7pfM7WICATnOv+iOhjletM3N0Xbu8= github.com/aws/aws-sdk-go-v2/service/appconfig v1.4.2/go.mod h1:FZ3HkCe+b10uFZZkFdvf98LHW21k49W8o8J366lqVKY= @@ -441,6 +449,7 @@ github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.19/go.mod github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.3/go.mod h1:7gcsONBmFoCcKrAqrm95trrMd2+C/ReYKP7Vfu8yHHA= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.2/go.mod h1:72HRZDLMtmVQiLG2tLfQcaWLCssELvGl+Zf2WVxMmR8= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.0/go.mod h1:Mq6AEc+oEjCUlBuLiK5YwW4shSOAKCQ3tXN0sQeYoBA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.7/go.mod h1:HvVdEh/x4jsPBsjNvDy+MH3CDCPy4gTZEzFe2r4uJY8= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17/go.mod h1:4nYOrY41Lrbk2170/BGkcJKBhws9Pfn8MG3aGqjjeFI= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19/go.mod h1:02CP6iuYP+IVnBX5HULVdSAku/85eHB2Y9EsFhrkEwU= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 h1:5C6XgTViSb0bunmU57b3CT+MhxULqHH2721FVA+/kDM= @@ -455,6 +464,7 @@ github.com/aws/aws-sdk-go-v2/service/s3 v1.19.0/go.mod h1:Gwz3aVctJe6mUY9T//bcAL github.com/aws/aws-sdk-go-v2/service/sso v1.3.3/go.mod h1:Jgw5O+SK7MZ2Yi9Yvzb4PggAPYaFSliiQuWR0hNjexk= github.com/aws/aws-sdk-go-v2/service/sso v1.4.2/go.mod h1:NBvT9R1MEF+Ud6ApJKM0G+IkPchKS7p7c2YPKwHmBOk= github.com/aws/aws-sdk-go-v2/service/sso v1.6.0/go.mod h1:Q/l0ON1annSU+mc0JybDy1Gy6dnJxIcWjphO6qJPzvM= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.10/go.mod h1:UHxA35uPrCykRySBV5iSPZhZRlYnWSS2c/aaZVsoU94= github.com/aws/aws-sdk-go-v2/service/sso v1.11.23/go.mod h1:/w0eg9IhFGjGyyncHIQrXtU8wvNsTJOP0R6PPj0wf80= github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 h1:GFZitO48N/7EsFDt8fMa5iYdmWqkUDDB3Eje6z3kbG0= github.com/aws/aws-sdk-go-v2/service/sso v1.11.25/go.mod h1:IARHuzTXmj1C0KS35vboR0FeJ89OkEy1M9mWbK2ifCI= @@ -464,6 +474,7 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8/go.mod h1:er2JHN+kBY6FcMfcB github.com/aws/aws-sdk-go-v2/service/sts v1.6.2/go.mod h1:RBhoMJB8yFToaCnbe0jNq5Dcdy0jp6LhHqg55rjClkM= github.com/aws/aws-sdk-go-v2/service/sts v1.7.2/go.mod h1:8EzeIqfWt2wWT4rJVu3f21TfrhJ8AEMzVybRNSb/b4g= github.com/aws/aws-sdk-go-v2/service/sts v1.10.0/go.mod h1:jLKCFqS+1T4i7HDqCP9GM4Uk75YW1cS0o82LdxpMyOE= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.8/go.mod h1:50YdFq1WIuxA0AGrygvYGucnNYrG24WYzu5fNp7lMgY= github.com/aws/aws-sdk-go-v2/service/sts v1.16.19/go.mod h1:h4J3oPZQbxLhzGnk+j9dfYHi5qIOVJ5kczZd658/ydM= github.com/aws/aws-sdk-go-v2/service/sts v1.17.4 h1:YNncBj5dVYd05i4ZQ+YicOotSXo0ufc9P8kTioi13EM= github.com/aws/aws-sdk-go-v2/service/sts v1.17.4/go.mod h1:bXcN3koeVYiJcdDU89n3kCYILob7Y34AeLopUbZgLT4= @@ -473,6 +484,7 @@ github.com/aws/smithy-go v1.5.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAm github.com/aws/smithy-go v1.7.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/aws/smithy-go v1.9.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/aws/smithy-go v1.12.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.12.1/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.13.4/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= @@ -2096,6 +2108,8 @@ github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3 github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/opencontainers/selinux v1.10.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/opensearch-project/opensearch-go/v2 v2.1.0 h1:a2wQiSQQ3c9nXRCW28751ByCLIL6h8KHvV0LUxfw1SI= +github.com/opensearch-project/opensearch-go/v2 v2.1.0/go.mod h1:pAxdJshAIO69ugn8PCMXClyTetS3GVe/SgUYLvx8bE8= github.com/opentracing-contrib/go-grpc v0.0.0-20191001143057-db30781987df/go.mod h1:DYR5Eij8rJl8h7gblRrOZ8g0kW1umSpKqYIBTgeDtLo= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492 h1:lM6RxxfUMrYL/f8bWEUqdXrANWtrL7Nndbm9iFN0DlU= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= diff --git a/plugins/inputs/all/opensearch_query.go b/plugins/inputs/all/opensearch_query.go new file mode 100644 index 000000000..702903749 --- /dev/null +++ b/plugins/inputs/all/opensearch_query.go @@ -0,0 +1,5 @@ +//go:build !custom || inputs || inputs.opensearch_query + +package all + +import _ "github.com/influxdata/telegraf/plugins/inputs/opensearch_query" // register plugin diff --git a/plugins/inputs/opensearch_query/README.md b/plugins/inputs/opensearch_query/README.md new file mode 100755 index 000000000..77dba3970 --- /dev/null +++ b/plugins/inputs/opensearch_query/README.md @@ -0,0 +1,258 @@ +# OpenSearch Query Input Plugin + +This [OpenSearch](https://opensearch.org/) plugin queries endpoints +to derive metrics from data stored in an OpenSearch cluster. + +The following is supported: + +- return number of hits for a search query +- calculate the `avg`/`max`/`min`/`sum` for a numeric field, filtered by a query, + aggregated per tag +- `value_count` returns the number of documents for a particular field +- `stats` (returns `sum`, `min`, `max`, `avg`, and `value_count` in one query) +- extended_stats (`stats` plus stats such as sum of squares, variance, and standard + deviation) +- `percentiles` returns the 1st, 5th, 25th, 50th, 75th, 95th, and 99th percentiles + +## OpenSearch Support + +This plugins is tested against OpenSearch 2.5.0 and 1.3.7. + +## Global configuration options + +In addition to the plugin-specific configuration settings, plugins support +additional global and plugin configuration settings. These settings are used to +modify metrics, tags, and field or create aliases and configure ordering, etc. +See the [CONFIGURATION.md][CONFIGURATION.md] for more details. + +[CONFIGURATION.md]: ../../../docs/CONFIGURATION.md#plugins + +## Configuration + +```toml @sample.conf +# Derive metrics from aggregating OpenSearch query results +[[inputs.opensearch_query]] + ## OpenSearch cluster endpoint(s). Multiple urls can be specified as part + ## of the same cluster. Only one succesful call will be made per interval. + urls = [ "https://node1.os.example.com:9200" ] # required. + + ## OpenSearch client timeout, defaults to "5s". + # timeout = "5s" + + ## HTTP basic authentication details + # username = "admin" + # password = "admin" + + ## Skip TLS validation. Useful for local testing and self-signed certs. + # insecure_skip_verify = false + + [[inputs.opensearch_query.aggregation]] + ## measurement name for the results of the aggregation query + measurement_name = "measurement" + + ## OpenSearch index or index pattern to search + index = "index-*" + + ## The date/time field in the OpenSearch index (mandatory). + date_field = "@timestamp" + + ## If the field used for the date/time field in OpenSearch is also using + ## a custom date/time format it may be required to provide the format to + ## correctly parse the field. + ## + ## If using one of the built in OpenSearch formats this is not required. + ## https://opensearch.org/docs/2.4/opensearch/supported-field-types/date/#built-in-formats + # date_field_custom_format = "" + + ## Time window to query (eg. "1m" to query documents from last minute). + ## Normally should be set to same as collection interval + query_period = "1m" + + ## Lucene query to filter results + # filter_query = "*" + + ## Fields to aggregate values (must be numeric fields) + # metric_fields = ["metric"] + + ## Aggregation function to use on the metric fields + ## Must be set if 'metric_fields' is set + ## Valid values are: avg, sum, min, max, sum + # metric_function = "avg" + + ## Fields to be used as tags. Must be text, non-analyzed fields. Metric + ## aggregations are performed per tag + # tags = ["field.keyword", "field2.keyword"] + + ## Set to true to not ignore documents when the tag(s) above are missing + # include_missing_tag = false + + ## String value of the tag when the tag does not exist + ## Required when include_missing_tag is true + # missing_tag_value = "null" +``` + +### Required parameters + +- `measurement_name`: The target measurement to be stored the results of the + aggregation query. +- `index`: The index name to query on OpenSearch +- `query_period`: The time window to query (eg. "1m" to query documents from + last minute). Normally should be set to same as collection +- `date_field`: The date/time field in the OpenSearch index + +### Optional parameters + +- `date_field_custom_format`: Not needed if using one of the built in date/time + formats of OpenSearch, but may be required if using a custom date/time + format. The format syntax uses the [Joda date format][joda]. +- `filter_query`: Lucene query to filter the results (default: "\*") +- `metric_fields`: The list of fields to perform metric aggregation (these must + be indexed as numeric fields) +- `metric_funcion`: The single-value metric aggregation function to be performed + on the `metric_fields` defined. Currently supported aggregations are "avg", + "min", "max", "sum", "value_count", "stats", "extended_stats", "percentiles". + (see the [aggregation docs][agg] +- `tags`: The list of fields to be used as tags (these must be indexed as + non-analyzed fields). A "terms aggregation" will be done per tag defined +- `include_missing_tag`: Set to true to not ignore documents where the tag(s) + specified above does not exist. (If false, documents without the specified tag + field will be ignored in `doc_count` and in the metric aggregation) +- `missing_tag_value`: The value of the tag that will be set for documents in + which the tag field does not exist. Only used when `include_missing_tag` is + set to `true`. + +[joda]: https://opensearch.org/docs/2.4/opensearch/supported-field-types/date/#custom-formats +[agg]: https://opensearch.org/docs/2.4/opensearch/aggregations/ + +### Example configurations + +#### Search the average response time, per URI and per response status code + +```toml +[[inputs.opensearch_query.aggregation]] + measurement_name = "http_logs" + index = "my-index-*" + filter_query = "*" + metric_fields = ["response_time"] + metric_function = "avg" + tags = ["URI.keyword", "response.keyword"] + include_missing_tag = true + missing_tag_value = "null" + date_field = "@timestamp" + query_period = "1m" +``` + +#### Search the maximum response time per method and per URI + +```toml +[[inputs.opensearch_query.aggregation]] + measurement_name = "http_logs" + index = "my-index-*" + filter_query = "*" + metric_fields = ["response_time"] + metric_function = "max" + tags = ["method.keyword","URI.keyword"] + include_missing_tag = false + missing_tag_value = "null" + date_field = "@timestamp" + query_period = "1m" +``` + +#### Search number of documents matching a filter query in all indices + +```toml +[[inputs.opensearch_query.aggregation]] + measurement_name = "http_logs" + index = "*" + filter_query = "product_1 AND HEAD" + query_period = "1m" + date_field = "@timestamp" +``` + +#### Search number of documents matching a filter query, returning per response status code + +```toml +[[inputs.opensearch_query.aggregation]] + measurement_name = "http_logs" + index = "*" + filter_query = "downloads" + tags = ["response.keyword"] + include_missing_tag = false + date_field = "@timestamp" + query_period = "1m" +``` + +#### Search all documents and generate common statistics, returning per response status code + +```toml +[[inputs.opensearch_query.aggregation]] + measurement_name = "http_logs" + index = "*" + tags = ["response.keyword"] + include_missing_tag = false + date_field = "@timestamp" + query_period = "1m" +``` + +## Metrics + +All metrics derive from aggregating OpenSearch query results. Queries must +conform to appropriate OpenSearch +[Aggregations](https://opensearch.org/docs/latest/opensearch/aggregations/) +for more information. + +Metric names are composed of a combination of the field name, metric aggregation +function, and the result field name. + +For simple metrics, the result field name is `value`, and so getting the `avg` +on a field named `size` would produce the result `size_value_avg`. + +For functions with multiple metrics, we use the resulting field. For example, +the `stats` function returns five different results, so for a field `size`, +we would see five metric fields, named `size_stats_min`, +`size_stats_max`, `size_stats_sum`, `size_stats_avg`, and `size_stats_count`. + +Nested results will build on their parent field names, for example, results for +percentile take the form: + +```json +{ + "aggregations" : { + "size_percentiles" : { + "values" : { + "1.0" : 21.984375, + "5.0" : 27.984375, + "25.0" : 44.96875, + "50.0" : 64.22061688311689, + "75.0" : 93.0, + "95.0" : 156.0, + "99.0" : 222.0 + } + } + } +} +``` + +Thus, our results would take the form `size_percentiles_values_1.0`. This +structure applies to `percentiles` and `extended_stats` functions. + +Note: `extended_stats` is currently limited to 2 standard deviations only. + +## Example Output + +```toml +[[inputs.opensearch_query.aggregation]] + measurement_name = "bytes_stats" + index = "opensearch_dashboards_sample_data_logs" + date_field = "timestamp" + query_period = "10m" + filter_query = "*" + metric_fields = ["bytes"] + metric_function = "stats" + tags = ["response.keyword"] +``` + +```text +bytes_stats,host=localhost,response_keyword=200 bytes_stats_sum=22231,doc_count=4i,bytes_stats_count=4,bytes_stats_min=941,bytes_stats_max=9544,bytes_stats_avg=5557.75 1672327840000000000 +bytes_stats,host=localhost,response_keyword=404 bytes_stats_min=5330,bytes_stats_max=5330,bytes_stats_avg=5330,doc_count=1i,bytes_stats_sum=5330,bytes_stats_count=1 1672327840000000000 +``` diff --git a/plugins/inputs/opensearch_query/aggregation.bucket.go b/plugins/inputs/opensearch_query/aggregation.bucket.go new file mode 100644 index 000000000..358c015ea --- /dev/null +++ b/plugins/inputs/opensearch_query/aggregation.bucket.go @@ -0,0 +1,42 @@ +package opensearch_query + +import "fmt" + +type BucketAggregationRequest map[string]*aggregationFunction + +func (b BucketAggregationRequest) AddAggregation(name, aggType, field string) error { + switch aggType { + case "terms": + default: + return fmt.Errorf("aggregation function %q not supported", aggType) + } + + b[name] = &aggregationFunction{ + aggType: aggType, + field: field, + } + + return nil +} + +func (b BucketAggregationRequest) AddNestedAggregation(name string, a AggregationRequest) { + b[name].nested = a +} + +func (b BucketAggregationRequest) BucketSize(name string, size int) error { + if size <= 0 { + return fmt.Errorf("invalid size; must be integer value > 0") + } + + if _, ok := b[name]; !ok { + return fmt.Errorf("aggregation %q not found", name) + } + + b[name].Size(size) + + return nil +} + +func (b BucketAggregationRequest) Missing(name, missing string) { + b[name].Missing(missing) +} diff --git a/plugins/inputs/opensearch_query/aggregation.go b/plugins/inputs/opensearch_query/aggregation.go new file mode 100644 index 000000000..38c7ae76c --- /dev/null +++ b/plugins/inputs/opensearch_query/aggregation.go @@ -0,0 +1,66 @@ +package opensearch_query + +import ( + "encoding/json" + "fmt" +) + +type AggregationRequest interface { + AddAggregation(string, string, string) error +} + +type NestedAggregation interface { + Nested(string, AggregationRequest) + Missing(string) + Size(int) +} + +type aggregationFunction struct { + aggType string + field string + size int + missing string + + nested AggregationRequest +} + +func (a *aggregationFunction) MarshalJSON() ([]byte, error) { + agg := make(map[string]interface{}) + field := map[string]interface{}{"field": a.field} + if t, _ := getAggregationFunctionType(a.aggType); t == "bucket" { + // We'll use the default size of 10 if it hasn't been set; size == 0 is illegal in a bucket aggregation + if a.size == 0 { + a.size = 10 + } + field["size"] = a.size + } + if a.missing != "" { + field["missing"] = a.missing + } + + agg[a.aggType] = field + + if a.nested != nil { + agg["aggregations"] = a.nested + } + return json.Marshal(agg) +} + +func (a *aggregationFunction) Size(size int) { + a.size = size +} + +func (a *aggregationFunction) Missing(missing string) { + a.missing = missing +} + +func getAggregationFunctionType(field string) (string, error) { + switch field { + case "avg", "sum", "min", "max", "value_count", "stats", "extended_stats", "percentiles": + return "metric", nil + case "terms": + return "bucket", nil + default: + return "", fmt.Errorf("invalid aggregation function %s", field) + } +} diff --git a/plugins/inputs/opensearch_query/aggregation.metric.go b/plugins/inputs/opensearch_query/aggregation.metric.go new file mode 100644 index 000000000..a77b23c23 --- /dev/null +++ b/plugins/inputs/opensearch_query/aggregation.metric.go @@ -0,0 +1,18 @@ +package opensearch_query + +import "fmt" + +type MetricAggregationRequest map[string]*aggregationFunction + +func (m MetricAggregationRequest) AddAggregation(name, aggType, field string) error { + if t, _ := getAggregationFunctionType(aggType); t != "metric" { + return fmt.Errorf("aggregation function %q not supported", aggType) + } + + m[name] = &aggregationFunction{ + aggType: aggType, + field: field, + } + + return nil +} diff --git a/plugins/inputs/opensearch_query/aggregation.response.go b/plugins/inputs/opensearch_query/aggregation.response.go new file mode 100644 index 000000000..0b950dfbf --- /dev/null +++ b/plugins/inputs/opensearch_query/aggregation.response.go @@ -0,0 +1,143 @@ +package opensearch_query + +import ( + "encoding/json" + + "github.com/influxdata/telegraf" +) + +type AggregationResponse struct { + Hits *SearchHits `json:"hits"` + Aggregations *Aggregation `json:"aggregations"` +} + +type SearchHits struct { + TotalHits *TotalHits `json:"total,omitempty"` +} + +type TotalHits struct { + Relation string `json:"relation"` + Value int64 `json:"value"` +} + +type MetricAggregation map[string]interface{} + +type AggregateValue struct { + metrics MetricAggregation + buckets []BucketData +} + +type Aggregation map[string]AggregateValue + +type BucketData struct { + DocumentCount int64 `json:"doc_count"` + Key string `json:"key"` + + subaggregation Aggregation +} + +func (a *AggregationResponse) GetMetrics(acc telegraf.Accumulator, measurement string) error { + // Simple case (no aggregations) + if a.Aggregations == nil { + tags := make(map[string]string) + fields := map[string]interface{}{ + "doc_count": a.Hits.TotalHits.Value, + } + acc.AddFields(measurement, fields, tags) + return nil + } + + return a.Aggregations.GetMetrics(acc, measurement, a.Hits.TotalHits.Value, map[string]string{}) +} + +func (a *Aggregation) GetMetrics(acc telegraf.Accumulator, measurement string, docCount int64, tags map[string]string) error { + var err error + fields := make(map[string]interface{}) + for name, agg := range *a { + if agg.IsAggregation() { + for _, bucket := range agg.buckets { + tt := map[string]string{name: bucket.Key} + for k, v := range tags { + tt[k] = v + } + err = bucket.subaggregation.GetMetrics(acc, measurement, bucket.DocumentCount, tt) + if err != nil { + return err + } + } + return nil + } + for metric, value := range agg.metrics { + switch value := value.(type) { + case map[string]interface{}: + for k, v := range value { + fields[name+"_"+metric+"_"+k] = v + } + default: + fields[name+"_"+metric] = value + } + } + } + + fields["doc_count"] = docCount + acc.AddFields(measurement, fields, tags) + + return nil +} + +func (a *AggregateValue) UnmarshalJSON(bytes []byte) error { + var partial map[string]json.RawMessage + err := json.Unmarshal(bytes, &partial) + if err != nil { + return err + } + + // We'll continue to unmarshal if we have buckets + if b, found := partial["buckets"]; found { + return json.Unmarshal(b, &a.buckets) + } + + // Use the remaining bytes as metrics + return json.Unmarshal(bytes, &a.metrics) +} + +func (a *AggregateValue) IsAggregation() bool { + return !(a.buckets == nil) +} + +func (b *BucketData) UnmarshalJSON(bytes []byte) error { + var partial map[string]json.RawMessage + var err error + + err = json.Unmarshal(bytes, &partial) + if err != nil { + return err + } + + err = json.Unmarshal(partial["doc_count"], &b.DocumentCount) + if err != nil { + return err + } + delete(partial, "doc_count") + + err = json.Unmarshal(partial["key"], &b.Key) + if err != nil { + return err + } + delete(partial, "key") + + if b.subaggregation == nil { + b.subaggregation = make(Aggregation) + } + + for name, message := range partial { + var subaggregation AggregateValue + err = json.Unmarshal(message, &subaggregation) + if err != nil { + return err + } + b.subaggregation[name] = subaggregation + } + + return nil +} diff --git a/plugins/inputs/opensearch_query/opensearch_query.go b/plugins/inputs/opensearch_query/opensearch_query.go new file mode 100644 index 000000000..d35895de7 --- /dev/null +++ b/plugins/inputs/opensearch_query/opensearch_query.go @@ -0,0 +1,269 @@ +//go:generate ../../../tools/readme_config_includer/generator +package opensearch_query + +import ( + "context" + "crypto/tls" + _ "embed" + "encoding/json" + "fmt" + "net/http" + "strings" + "sync" + "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/config" + influxtls "github.com/influxdata/telegraf/plugins/common/tls" + "github.com/influxdata/telegraf/plugins/inputs" + "github.com/opensearch-project/opensearch-go/v2" + "github.com/opensearch-project/opensearch-go/v2/opensearchapi" +) + +//go:embed sample.conf +var sampleConfig string + +// OpensearchQuery struct +type OpensearchQuery struct { + URLs []string `toml:"urls"` + Username config.Secret `toml:"username"` + Password config.Secret `toml:"password"` + EnableSniffer bool `toml:"enable_sniffer"` + Timeout config.Duration `toml:"timeout"` + HealthCheckInterval config.Duration `toml:"health_check_interval"` + Aggregations []osAggregation `toml:"aggregation"` + + Log telegraf.Logger `toml:"-"` + + influxtls.ClientConfig + osClient *opensearch.Client +} + +// osAggregation struct +type osAggregation struct { + Index string `toml:"index"` + MeasurementName string `toml:"measurement_name"` + DateField string `toml:"date_field"` + DateFieldFormat string `toml:"date_field_custom_format"` + QueryPeriod config.Duration `toml:"query_period"` + FilterQuery string `toml:"filter_query"` + MetricFields []string `toml:"metric_fields"` + MetricFunction string `toml:"metric_function"` + Tags []string `toml:"tags"` + IncludeMissingTag bool `toml:"include_missing_tag"` + MissingTagValue string `toml:"missing_tag_value"` + mapMetricFields map[string]string + + aggregation AggregationRequest +} + +func (*OpensearchQuery) SampleConfig() string { + return sampleConfig +} + +// Init the plugin. +func (o *OpensearchQuery) Init() error { + if o.URLs == nil { + return fmt.Errorf("no urls defined") + } + + err := o.newClient() + if err != nil { + o.Log.Errorf("error creating OpenSearch client: %w", err) + } + + for i, agg := range o.Aggregations { + if agg.MeasurementName == "" { + return fmt.Errorf("field 'measurement_name' is not set") + } + if agg.DateField == "" { + return fmt.Errorf("field 'date_field' is not set") + } + err = o.initAggregation(agg, i) + if err != nil { + return err + } + } + return nil +} + +func (o *OpensearchQuery) initAggregation(agg osAggregation, i int) (err error) { + for _, metricField := range agg.MetricFields { + if _, ok := agg.mapMetricFields[metricField]; !ok { + return fmt.Errorf("metric field '%s' not found on index '%s'", metricField, agg.Index) + } + } + + err = agg.buildAggregationQuery() + if err != nil { + return fmt.Errorf("error building aggregation: %w", err) + } + + o.Aggregations[i] = agg + return nil +} + +func (o *OpensearchQuery) newClient() error { + username, err := o.Username.Get() + if err != nil { + return fmt.Errorf("getting username failed: %w", err) + } + defer config.ReleaseSecret(username) + password, err := o.Password.Get() + if err != nil { + return fmt.Errorf("getting password failed: %w", err) + } + defer config.ReleaseSecret(password) + + clientConfig := opensearch.Config{ + Addresses: o.URLs, + Username: string(username), + Password: string(password), + } + + if o.InsecureSkipVerify { + clientConfig.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + } + + client, err := opensearch.NewClient(clientConfig) + o.osClient = client + + return err +} + +// Gather writes the results of the queries from OpenSearch to the Accumulator. +func (o *OpensearchQuery) Gather(acc telegraf.Accumulator) error { + var wg sync.WaitGroup + + for _, agg := range o.Aggregations { + wg.Add(1) + go func(agg osAggregation) { + defer wg.Done() + err := o.osAggregationQuery(acc, agg) + if err != nil { + acc.AddError(fmt.Errorf("opensearch query aggregation %s: %s ", agg.MeasurementName, err)) + } + }(agg) + } + + wg.Wait() + return nil +} + +func (o *OpensearchQuery) osAggregationQuery(acc telegraf.Accumulator, aggregation osAggregation) error { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(o.Timeout)) + defer cancel() + + searchResult, err := o.runAggregationQuery(ctx, aggregation) + if err != nil { + return err + } + + return searchResult.GetMetrics(acc, aggregation.MeasurementName) +} + +func init() { + inputs.Add("opensearch_query", func() telegraf.Input { + return &OpensearchQuery{ + Timeout: config.Duration(time.Second * 5), + HealthCheckInterval: config.Duration(time.Second * 10), + } + }) +} + +func (o *OpensearchQuery) runAggregationQuery(ctx context.Context, aggregation osAggregation) (*AggregationResponse, error) { + now := time.Now().UTC() + from := now.Add(time.Duration(-aggregation.QueryPeriod)) + filterQuery := aggregation.FilterQuery + if filterQuery == "" { + filterQuery = "*" + } + + aq := &Query{ + Size: 0, + Aggregations: aggregation.aggregation, + Query: nil, + } + + boolQuery := &BoolQuery{ + FilterQueryString: filterQuery, + TimestampField: aggregation.DateField, + TimeRangeFrom: from, + TimeRangeTo: now, + DateFieldFormat: aggregation.DateFieldFormat, + } + + aq.Query = boolQuery + req, err := json.Marshal(aq) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + + searchRequest := &opensearchapi.SearchRequest{ + Body: strings.NewReader(string(req)), + Index: []string{aggregation.Index}, + Timeout: time.Duration(o.Timeout), + } + + resp, err := searchRequest.Do(ctx, o.osClient) + if err != nil { + return nil, err + } + if resp.IsError() { + return nil, fmt.Errorf("Opensearch SearchRequest failure: [%d] %s", resp.StatusCode, resp.Status()) + } + defer resp.Body.Close() + + var searchResult AggregationResponse + + decoder := json.NewDecoder(resp.Body) + err = decoder.Decode(&searchResult) + if err != nil { + return nil, err + } + + return &searchResult, nil +} + +func (aggregation *osAggregation) buildAggregationQuery() error { + var agg AggregationRequest + agg = &MetricAggregationRequest{} + + // create one aggregation per metric field found & function defined for numeric fields + for k, v := range aggregation.mapMetricFields { + switch v { + case "long", "float", "integer", "short", "double", "scaled_float": + default: + continue + } + + err := agg.AddAggregation(strings.ReplaceAll(k, ".", "_")+"_"+aggregation.MetricFunction, aggregation.MetricFunction, k) + if err != nil { + return err + } + } + + // create a terms aggregation per tag + for _, term := range aggregation.Tags { + bucket := &BucketAggregationRequest{} + name := strings.ReplaceAll(term, ".", "_") + err := bucket.AddAggregation(name, "terms", term) + if err != nil { + return err + } + _ = bucket.BucketSize(name, 1000) + if aggregation.IncludeMissingTag && aggregation.MissingTagValue != "" { + bucket.Missing(name, aggregation.MissingTagValue) + } + + bucket.AddNestedAggregation(name, agg) + + agg = bucket + } + + aggregation.aggregation = agg + + return nil +} diff --git a/plugins/inputs/opensearch_query/opensearch_query_test.go b/plugins/inputs/opensearch_query/opensearch_query_test.go new file mode 100644 index 000000000..8f8db4fb0 --- /dev/null +++ b/plugins/inputs/opensearch_query/opensearch_query_test.go @@ -0,0 +1,695 @@ +package opensearch_query + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "os" + "strconv" + "strings" + "testing" + "time" + + "github.com/docker/go-connections/nat" + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/config" + "github.com/influxdata/telegraf/plugins/common/tls" + "github.com/influxdata/telegraf/testutil" + "github.com/opensearch-project/opensearch-go/v2/opensearchutil" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go/wait" +) + +const ( + servicePort = "9200" + testindex = "test-opensearch" +) + +type osAggregationQueryTest struct { + queryName string + testAggregationQueryInput osAggregation + expectedMetrics []telegraf.Metric + wantQueryResErr bool + wantInitErr bool +} + +var queryPeriod = config.Duration(time.Second * 600) + +func testData() []osAggregationQueryTest { + return []osAggregationQueryTest{ + { + queryName: "query 1 (avg)", + testAggregationQueryInput: osAggregation{ + Index: testindex, + MeasurementName: "measurement1", + MetricFields: []string{"size"}, + FilterQuery: "product_1", + MetricFunction: "avg", + DateField: "@timestamp", + QueryPeriod: queryPeriod, + Tags: []string{"URI.keyword"}, + mapMetricFields: map[string]string{"size": "long"}, + }, + expectedMetrics: []telegraf.Metric{ + testutil.MustMetric( + "measurement1", + map[string]string{"URI_keyword": "/downloads/product_1"}, + map[string]interface{}{"size_avg_value": float64(202.30038022813687), "doc_count": int64(263)}, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + }, + }, + { + queryName: "query 2 (avg)", + testAggregationQueryInput: osAggregation{ + Index: testindex, + MeasurementName: "measurement2", + MetricFields: []string{"size"}, + FilterQuery: "downloads", + MetricFunction: "max", + DateField: "@timestamp", + QueryPeriod: queryPeriod, + Tags: []string{"URI.keyword"}, + mapMetricFields: map[string]string{"size": "long"}, + }, + expectedMetrics: []telegraf.Metric{ + testutil.MustMetric( + "measurement2", + map[string]string{"URI_keyword": "/downloads/product_1"}, + map[string]interface{}{"size_max_value": float64(3301), "doc_count": int64(263)}, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + testutil.MustMetric( + "measurement2", + map[string]string{"URI_keyword": "/downloads/product_2"}, + map[string]interface{}{"size_max_value": float64(3318), "doc_count": int64(237)}, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + }, + }, + { + queryName: "query 3 (sum)", + testAggregationQueryInput: osAggregation{ + Index: testindex, + MeasurementName: "measurement3", + MetricFields: []string{"size"}, + FilterQuery: "downloads", + MetricFunction: "sum", + DateField: "@timestamp", + QueryPeriod: queryPeriod, + Tags: []string{"response.keyword"}, + mapMetricFields: map[string]string{"size": "long"}, + }, + expectedMetrics: []telegraf.Metric{ + testutil.MustMetric( + "measurement3", + map[string]string{"response_keyword": "200"}, + map[string]interface{}{"size_sum_value": float64(22790), "doc_count": int64(22)}, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + testutil.MustMetric( + "measurement3", + map[string]string{"response_keyword": "304"}, + map[string]interface{}{"size_sum_value": float64(0), "doc_count": int64(219)}, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + testutil.MustMetric( + "measurement3", + map[string]string{"response_keyword": "404"}, + map[string]interface{}{"size_sum_value": float64(86932), "doc_count": int64(259)}, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + }, + }, + { + queryName: "query 4 (min, 2 fields, 3 tags)", + testAggregationQueryInput: osAggregation{ + Index: testindex, + MeasurementName: "measurement4", + MetricFields: []string{"size", "response_time"}, + FilterQuery: "downloads", + MetricFunction: "min", + DateField: "@timestamp", + QueryPeriod: queryPeriod, + IncludeMissingTag: true, + MissingTagValue: "missing", + Tags: []string{"response.keyword", "URI.keyword", "method.keyword"}, + mapMetricFields: map[string]string{"size": "long", "response_time": "long"}, + }, + expectedMetrics: []telegraf.Metric{ + testutil.MustMetric( + "measurement4", + map[string]string{"response_keyword": "404", "URI_keyword": "/downloads/product_1", "method_keyword": "GET"}, + map[string]interface{}{"size_min_value": float64(318), "response_time_min_value": float64(126), "doc_count": int64(146)}, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + testutil.MustMetric( + "measurement4", + map[string]string{"response_keyword": "304", "URI_keyword": "/downloads/product_1", "method_keyword": "GET"}, + map[string]interface{}{"size_min_value": float64(0), "response_time_min_value": float64(71), "doc_count": int64(113)}, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + testutil.MustMetric( + "measurement4", + map[string]string{"response_keyword": "200", "URI_keyword": "/downloads/product_1", "method_keyword": "GET"}, + map[string]interface{}{"size_min_value": float64(490), "response_time_min_value": float64(1514), "doc_count": int64(3)}, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + testutil.MustMetric( + "measurement4", + map[string]string{"response_keyword": "404", "URI_keyword": "/downloads/product_2", "method_keyword": "GET"}, + map[string]interface{}{"size_min_value": float64(318), "response_time_min_value": float64(237), "doc_count": int64(113)}, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + testutil.MustMetric( + "measurement4", + map[string]string{"response_keyword": "304", "URI_keyword": "/downloads/product_2", "method_keyword": "GET"}, + map[string]interface{}{"size_min_value": float64(0), "response_time_min_value": float64(134), "doc_count": int64(106)}, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + testutil.MustMetric( + "measurement4", + map[string]string{"response_keyword": "200", "URI_keyword": "/downloads/product_2", "method_keyword": "GET"}, + map[string]interface{}{"size_min_value": float64(490), "response_time_min_value": float64(2), "doc_count": int64(13)}, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + testutil.MustMetric( + "measurement4", + map[string]string{"response_keyword": "200", "URI_keyword": "/downloads/product_1", "method_keyword": "HEAD"}, + map[string]interface{}{"size_min_value": float64(0), "response_time_min_value": float64(8479), "doc_count": int64(1)}, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + testutil.MustMetric( + "measurement4", + map[string]string{"response_keyword": "200", "URI_keyword": "/downloads/product_2", "method_keyword": "HEAD"}, + map[string]interface{}{"size_min_value": float64(0), "response_time_min_value": float64(1059), "doc_count": int64(5)}, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + }, + }, + { + queryName: "query 5 (no fields)", + testAggregationQueryInput: osAggregation{ + Index: testindex, + MeasurementName: "measurement5", + FilterQuery: "product_2", + DateField: "@timestamp", + QueryPeriod: queryPeriod, + Tags: []string{"URI.keyword"}, + mapMetricFields: map[string]string{}, + }, + expectedMetrics: []telegraf.Metric{ + testutil.MustMetric( + "measurement5", + map[string]string{"URI_keyword": "/downloads/product_2"}, + map[string]interface{}{"doc_count": int64(237)}, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + }, + }, + { + queryName: "query 6 (no fields, to tags)", + testAggregationQueryInput: osAggregation{ + Index: testindex, + MeasurementName: "measurement6", + FilterQuery: "response: 200", + DateField: "@timestamp", + QueryPeriod: queryPeriod, + Tags: []string{"URI.keyword", "response.keyword"}, + mapMetricFields: map[string]string{}, + }, + expectedMetrics: []telegraf.Metric{ + testutil.MustMetric( + "measurement6", + map[string]string{"response_keyword": "200", "URI_keyword": "/downloads/product_1"}, + map[string]interface{}{"doc_count": int64(4)}, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + testutil.MustMetric( + "measurement6", + map[string]string{"response_keyword": "200", "URI_keyword": "/downloads/product_2"}, + map[string]interface{}{"doc_count": int64(18)}, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + }, + }, + { + queryName: "query 7 (simple query)", + testAggregationQueryInput: osAggregation{ + Index: testindex, + MeasurementName: "measurement7", + FilterQuery: "response: 200", + DateField: "@timestamp", + QueryPeriod: queryPeriod, + Tags: []string{}, + mapMetricFields: map[string]string{}, + }, + expectedMetrics: []telegraf.Metric{ + testutil.MustMetric( + "measurement7", + map[string]string{}, + map[string]interface{}{"doc_count": int64(22)}, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + }, + }, + { + queryName: "query 8 (max, no tags)", + testAggregationQueryInput: osAggregation{ + Index: testindex, + MeasurementName: "measurement8", + MetricFields: []string{"size"}, + FilterQuery: "downloads", + MetricFunction: "max", + DateField: "@timestamp", + QueryPeriod: queryPeriod, + Tags: []string{}, + mapMetricFields: map[string]string{"size": "long"}, + }, + expectedMetrics: []telegraf.Metric{ + testutil.MustMetric( + "measurement8", + map[string]string{}, + map[string]interface{}{"size_max_value": float64(3318), "doc_count": int64(500)}, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + }, + }, + { + queryName: "query 9 (invalid function)", + testAggregationQueryInput: osAggregation{ + Index: testindex, + MeasurementName: "measurement9", + MetricFields: []string{"size"}, + FilterQuery: "downloads", + MetricFunction: "average", + DateField: "@timestamp", + QueryPeriod: queryPeriod, + Tags: []string{}, + mapMetricFields: map[string]string{"size": "long"}, + }, + wantInitErr: true, + }, + { + queryName: "query 10 (non-existing metric field)", + testAggregationQueryInput: osAggregation{ + Index: testindex, + MeasurementName: "measurement10", + MetricFields: []string{"none"}, + DateField: "@timestamp", + QueryPeriod: queryPeriod, + Tags: []string{}, + mapMetricFields: map[string]string{}, + }, + wantQueryResErr: true, + wantInitErr: true, + }, + { + queryName: "query 11 (non-existing index field)", + testAggregationQueryInput: osAggregation{ + Index: "notanindex", + MeasurementName: "measurement11", + DateField: "@timestamp", + QueryPeriod: queryPeriod, + Tags: []string{}, + mapMetricFields: map[string]string{}, + }, + wantQueryResErr: true, + }, + { + queryName: "query 12 (non-existing timestamp field)", + testAggregationQueryInput: osAggregation{ + Index: testindex, + MeasurementName: "measurement12", + MetricFields: []string{"size"}, + MetricFunction: "avg", + DateField: "@notatimestamp", + QueryPeriod: queryPeriod, + Tags: []string{}, + mapMetricFields: map[string]string{"size": "long"}, + }, + expectedMetrics: []telegraf.Metric{ + testutil.MustMetric( + "measurement12", + map[string]string{}, + map[string]interface{}{"doc_count": int64(0)}, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + }, + }, + { + queryName: "query 13 (non-existing tag field)", + testAggregationQueryInput: osAggregation{ + Index: testindex, + MeasurementName: "measurement13", + MetricFields: []string{"size"}, + MetricFunction: "avg", + DateField: "@timestamp", + QueryPeriod: queryPeriod, + IncludeMissingTag: false, + Tags: []string{"nothere"}, + mapMetricFields: map[string]string{"size": "long"}, + }, + }, + { + queryName: "query 14 (non-existing custom date/time format)", + testAggregationQueryInput: osAggregation{ + Index: testindex, + MeasurementName: "measurement14", + DateField: "@timestamp", + DateFieldFormat: "yyyy", + QueryPeriod: queryPeriod, + Tags: []string{}, + mapMetricFields: map[string]string{}, + }, + wantQueryResErr: true, + }, + { + queryName: "query 15 (stats)", + testAggregationQueryInput: osAggregation{ + Index: testindex, + MeasurementName: "measurement15", + MetricFields: []string{"size"}, + FilterQuery: "downloads", + MetricFunction: "stats", + DateField: "@timestamp", + QueryPeriod: queryPeriod, + Tags: []string{"URI.keyword"}, + mapMetricFields: map[string]string{"size": "long"}, + }, + expectedMetrics: []telegraf.Metric{ + testutil.MustMetric( + "measurement15", + map[string]string{"URI_keyword": "/downloads/product_1"}, + map[string]interface{}{ + "size_stats_sum": float64(53205), + "size_stats_min": float64(0), + "size_stats_max": float64(3301), + "size_stats_avg": float64(202.30038022813687), + "size_stats_count": float64(263), + "doc_count": int64(263)}, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + testutil.MustMetric( + "measurement15", + map[string]string{"URI_keyword": "/downloads/product_2"}, + map[string]interface{}{ + "size_stats_sum": float64(56517), + "size_stats_min": float64(0), + "size_stats_max": float64(3318), + "size_stats_avg": float64(238.46835443037975), + "size_stats_count": float64(237), + "doc_count": int64(237)}, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + }, + }, + { + queryName: "query 16 (extended_stats)", + testAggregationQueryInput: osAggregation{ + Index: testindex, + MeasurementName: "measurement16", + MetricFields: []string{"size"}, + FilterQuery: "downloads", + MetricFunction: "extended_stats", + DateField: "@timestamp", + QueryPeriod: queryPeriod, + Tags: []string{"URI.keyword"}, + mapMetricFields: map[string]string{"size": "long"}, + }, + expectedMetrics: []telegraf.Metric{ + testutil.MustMetric( + "measurement16", + map[string]string{"URI_keyword": "/downloads/product_1"}, + map[string]interface{}{ + "size_extended_stats_avg": float64(202.30038022813687), + "size_extended_stats_count": float64(263), + "size_extended_stats_max": float64(3301), + "size_extended_stats_min": float64(0), + "size_extended_stats_sum": float64(53205), + "size_extended_stats_std_deviation": float64(254.33673728231705), + "size_extended_stats_std_deviation_population": float64(254.33673728231705), + "size_extended_stats_std_deviation_sampling": float64(254.8216504723906), + "size_extended_stats_std_deviation_bounds_upper": float64(710.9738547927709), + "size_extended_stats_std_deviation_bounds_lower": float64(-306.3730943364972), + "size_extended_stats_std_deviation_bounds_upper_population": float64(710.9738547927709), + "size_extended_stats_std_deviation_bounds_lower_population": float64(-306.3730943364972), + "size_extended_stats_std_deviation_bounds_upper_sampling": float64(711.9436811729181), + "size_extended_stats_std_deviation_bounds_lower_sampling": float64(-307.3429207166443), + "size_extended_stats_variance": float64(64687.17593141436), + "size_extended_stats_variance_sampling": float64(64934.07354947319), + "size_extended_stats_variance_population": float64(64687.17593141436), + "size_extended_stats_sum_of_squares": float64(27776119), + "doc_count": int64(263)}, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + testutil.MustMetric( + "measurement16", + map[string]string{"URI_keyword": "/downloads/product_2"}, + map[string]interface{}{ + "size_extended_stats_avg": float64(238.46835443037975), + "size_extended_stats_count": float64(237), + "size_extended_stats_max": float64(3318), + "size_extended_stats_min": float64(0), + "size_extended_stats_sum": float64(56517), + "size_extended_stats_std_deviation": float64(411.39122310768215), + "size_extended_stats_std_deviation_population": float64(411.39122310768215), + "size_extended_stats_std_deviation_sampling": float64(412.2618933368743), + "size_extended_stats_std_deviation_bounds_upper": float64(1061.250800645744), + "size_extended_stats_std_deviation_bounds_lower": float64(-584.3140917849846), + "size_extended_stats_std_deviation_bounds_upper_population": float64(1061.250800645744), + "size_extended_stats_std_deviation_bounds_lower_population": float64(-584.3140917849846), + "size_extended_stats_std_deviation_bounds_upper_sampling": float64(1062.9921411041285), + "size_extended_stats_std_deviation_bounds_lower_sampling": float64(-586.0554322433688), + "size_extended_stats_variance": float64(169242.7384500347), + "size_extended_stats_variance_sampling": float64(169959.86869770434), + "size_extended_stats_variance_population": float64(169242.7384500347), + "size_extended_stats_sum_of_squares": float64(53588045), + "doc_count": int64(237)}, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + }, + }, + { + queryName: "query 17 (percentiles)", + testAggregationQueryInput: osAggregation{ + Index: testindex, + MeasurementName: "measurement16", + MetricFields: []string{"size"}, + FilterQuery: "downloads", + MetricFunction: "percentiles", + DateField: "@timestamp", + QueryPeriod: queryPeriod, + Tags: []string{"URI.keyword"}, + mapMetricFields: map[string]string{"size": "long"}, + }, + expectedMetrics: []telegraf.Metric{ + testutil.MustMetric( + "measurement16", + map[string]string{"URI_keyword": "/downloads/product_1"}, + map[string]interface{}{ + "size_percentiles_values_1.0": float64(0), + "size_percentiles_values_5.0": float64(0), + "size_percentiles_values_25.0": float64(0), + "size_percentiles_values_50.0": float64(324), + "size_percentiles_values_75.0": float64(337), + "size_percentiles_values_95.0": float64(341), + "size_percentiles_values_99.0": float64(471.28000000000065), + "doc_count": int64(263)}, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + testutil.MustMetric( + "measurement16", + map[string]string{"URI_keyword": "/downloads/product_2"}, + map[string]interface{}{ + "size_percentiles_values_1.0": float64(0), + "size_percentiles_values_5.0": float64(0), + "size_percentiles_values_25.0": float64(0), + "size_percentiles_values_50.0": float64(324), + "size_percentiles_values_75.0": float64(339), + "size_percentiles_values_95.0": float64(490), + "size_percentiles_values_99.0": float64(2677.419999999997), + "doc_count": int64(237)}, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + }, + }, + } +} + +func opensearchTestImages() []string { + return []string{"opensearchproject/opensearch:2.5.0", "opensearchproject/opensearch:1.3.7"} +} + +func newOpensearchQuery(url string) *OpensearchQuery { + return &OpensearchQuery{ + URLs: []string{url}, + Timeout: config.Duration(time.Second * 30), + Log: testutil.Logger{}, + Username: config.NewSecret([]byte("admin")), + Password: config.NewSecret([]byte("admin")), + ClientConfig: tls.ClientConfig{InsecureSkipVerify: true}, + } +} + +func setupIntegrationTest(t *testing.T, image string) (*testutil.Container, *OpensearchQuery, error) { + var err error + + type nginxlog struct { + IPaddress string `json:"IP"` + Timestamp time.Time `json:"@timestamp"` + Method string `json:"method"` + URI string `json:"URI"` + Httpversion string `json:"http_version"` + Response string `json:"response"` + Size float64 `json:"size"` + ResponseTime float64 `json:"response_time"` + } + + container := testutil.Container{ + Image: image, + ExposedPorts: []string{servicePort}, + Env: map[string]string{ + "discovery.type": "single-node", + }, + WaitingFor: wait.ForAll( + wait.ForLog(".opendistro_security is used as internal security index."), + wait.ForListeningPort(nat.Port(servicePort)), + ), + } + err = container.Start() + require.NoError(t, err, "failed to start container") + + url := fmt.Sprintf("https://%s:%s", container.Address, container.Ports[servicePort]) + + o := newOpensearchQuery(url) + + err = o.newClient() + if err != nil { + return &container, o, err + } + + // parse and build query + file, err := os.Open("testdata/nginx_logs") + if err != nil { + return &container, o, err + } + defer func(file *os.File) { + _ = file.Close() + }(file) + + indexer, err := opensearchutil.NewBulkIndexer(opensearchutil.BulkIndexerConfig{ + Client: o.osClient, + Index: testindex, + Refresh: "true", + }) + if err != nil { + return &container, o, err + } + + scanner := bufio.NewScanner(file) + + for scanner.Scan() { + parts := strings.Split(scanner.Text(), " ") + size, _ := strconv.Atoi(parts[9]) + responseTime, _ := strconv.Atoi(parts[len(parts)-1]) + + logline := nginxlog{ + IPaddress: parts[0], + Timestamp: time.Now().UTC(), + Method: strings.ReplaceAll(parts[5], `"`, ""), + URI: parts[6], + Httpversion: strings.ReplaceAll(parts[7], `"`, ""), + Response: parts[8], + Size: float64(size), + ResponseTime: float64(responseTime), + } + + body, e := json.Marshal(logline) + if e != nil { + return &container, o, e + } + + e = indexer.Add( + context.Background(), + opensearchutil.BulkIndexerItem{ + Index: testindex, + Action: "index", + Body: strings.NewReader(string(body)), + }) + if e != nil { + return &container, o, e + } + } + + if scanner.Err() != nil { + return &container, o, err + } + + if err = indexer.Close(context.Background()); err != nil { + return &container, o, err + } + + return &container, o, nil +} + +func TestOpensearchQueryIntegration(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + for _, image := range opensearchTestImages() { + func() { + container, o, err := setupIntegrationTest(t, image) + require.NoError(t, err) + defer container.Terminate() + + for _, tt := range testData() { + t.Run(tt.queryName, func(t *testing.T) { + var err error + var acc testutil.Accumulator + + o.Aggregations = []osAggregation{tt.testAggregationQueryInput} + err = o.Init() + if (err != nil) != tt.wantInitErr { + t.Errorf("OpensearchQuery.Init() error = %v, wantInitErr %v", err, tt.wantInitErr) + return + } else if err != nil { + // Init() failures mean we're done + return + } + + err = o.Gather(&acc) + if (len(acc.Errors) > 0) != tt.wantQueryResErr { + for _, err = range acc.Errors { + t.Errorf("OpensearchQuery.Gather() error: %v, wantQueryResErr %v", err, tt.wantQueryResErr) + } + return + } + + require.NoError(t, err) + + testutil.RequireMetricsEqual(t, tt.expectedMetrics, acc.GetTelegrafMetrics(), testutil.SortMetrics(), testutil.IgnoreTime()) + }) + } + }() + } +} + +func TestMetricAggregationMarshal(t *testing.T) { + agg := &MetricAggregationRequest{} + err := agg.AddAggregation("sum_taxful_total_price", "sum", "taxful_total_price") + require.NoError(t, err) + + _, err = json.Marshal(agg) + require.NoError(t, err) + + bucket := &BucketAggregationRequest{} + err = bucket.AddAggregation("terms_by_currency", "terms", "currency") + require.NoError(t, err) + + bucket.AddNestedAggregation("terms_by_currency", agg) + _, err = json.Marshal(bucket) + require.NoError(t, err) +} diff --git a/plugins/inputs/opensearch_query/query.go b/plugins/inputs/opensearch_query/query.go new file mode 100644 index 000000000..1a12aa5e3 --- /dev/null +++ b/plugins/inputs/opensearch_query/query.go @@ -0,0 +1,44 @@ +package opensearch_query + +import ( + "encoding/json" + "time" +) + +type Query struct { + Size int `json:"size"` + Aggregations AggregationRequest `json:"aggregations"` + Query interface{} `json:"query,omitempty"` +} + +type BoolQuery struct { + FilterQueryString string + TimestampField string + TimeRangeFrom time.Time + TimeRangeTo time.Time + DateFieldFormat string +} + +func (b *BoolQuery) MarshalJSON() ([]byte, error) { + // Construct range + dateTimeRange := map[string]interface{}{ + "from": b.TimeRangeFrom, + "to": b.TimeRangeTo, + "include_lower": true, + "include_upper": true, + } + if b.DateFieldFormat != "" { + dateTimeRange["format"] = b.DateFieldFormat + } + rangeFilter := map[string]map[string]interface{}{"range": {b.TimestampField: dateTimeRange}} + + // Construct Filter + queryFilter := map[string]map[string]interface{}{ + "query_string": {"query": b.FilterQueryString}, + } + + // Construct boolean query + bq := map[string]map[string]interface{}{"bool": {"filter": []interface{}{rangeFilter, queryFilter}}} + + return json.Marshal(&bq) +} diff --git a/plugins/inputs/opensearch_query/sample.conf b/plugins/inputs/opensearch_query/sample.conf new file mode 100644 index 000000000..30b672e23 --- /dev/null +++ b/plugins/inputs/opensearch_query/sample.conf @@ -0,0 +1,59 @@ +# Derive metrics from aggregating OpenSearch query results +[[inputs.opensearch_query]] + ## OpenSearch cluster endpoint(s). Multiple urls can be specified as part + ## of the same cluster. Only one succesful call will be made per interval. + urls = [ "https://node1.os.example.com:9200" ] # required. + + ## OpenSearch client timeout, defaults to "5s". + # timeout = "5s" + + ## HTTP basic authentication details + # username = "admin" + # password = "admin" + + ## Skip TLS validation. Useful for local testing and self-signed certs. + # insecure_skip_verify = false + + [[inputs.opensearch_query.aggregation]] + ## measurement name for the results of the aggregation query + measurement_name = "measurement" + + ## OpenSearch index or index pattern to search + index = "index-*" + + ## The date/time field in the OpenSearch index (mandatory). + date_field = "@timestamp" + + ## If the field used for the date/time field in OpenSearch is also using + ## a custom date/time format it may be required to provide the format to + ## correctly parse the field. + ## + ## If using one of the built in OpenSearch formats this is not required. + ## https://opensearch.org/docs/2.4/opensearch/supported-field-types/date/#built-in-formats + # date_field_custom_format = "" + + ## Time window to query (eg. "1m" to query documents from last minute). + ## Normally should be set to same as collection interval + query_period = "1m" + + ## Lucene query to filter results + # filter_query = "*" + + ## Fields to aggregate values (must be numeric fields) + # metric_fields = ["metric"] + + ## Aggregation function to use on the metric fields + ## Must be set if 'metric_fields' is set + ## Valid values are: avg, sum, min, max, sum + # metric_function = "avg" + + ## Fields to be used as tags. Must be text, non-analyzed fields. Metric + ## aggregations are performed per tag + # tags = ["field.keyword", "field2.keyword"] + + ## Set to true to not ignore documents when the tag(s) above are missing + # include_missing_tag = false + + ## String value of the tag when the tag does not exist + ## Required when include_missing_tag is true + # missing_tag_value = "null" diff --git a/plugins/inputs/opensearch_query/testdata/nginx_logs b/plugins/inputs/opensearch_query/testdata/nginx_logs new file mode 100644 index 000000000..f6e9c8a11 --- /dev/null +++ b/plugins/inputs/opensearch_query/testdata/nginx_logs @@ -0,0 +1,500 @@ +93.180.71.3 - - [17/May/2015:08:05:32 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" 12060 +93.180.71.3 - - [17/May/2015:08:05:23 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" 12355 +80.91.33.133 - - [17/May/2015:08:05:24 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 26272 +217.168.17.5 - - [17/May/2015:08:05:34 +0000] "GET /downloads/product_1 HTTP/1.1" 200 490 "-" "Debian APT-HTTP/1.3 (0.8.10.3)" 1514 +217.168.17.5 - - [17/May/2015:08:05:09 +0000] "GET /downloads/product_2 HTTP/1.1" 200 490 "-" "Debian APT-HTTP/1.3 (0.8.10.3)" 2204 +93.180.71.3 - - [17/May/2015:08:05:57 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" 6012 +217.168.17.5 - - [17/May/2015:08:05:02 +0000] "GET /downloads/product_2 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.8.10.3)" 11220 +217.168.17.5 - - [17/May/2015:08:05:42 +0000] "GET /downloads/product_1 HTTP/1.1" 404 332 "-" "Debian APT-HTTP/1.3 (0.8.10.3)" 17843 +80.91.33.133 - - [17/May/2015:08:05:01 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 22599 +93.180.71.3 - - [17/May/2015:08:05:27 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" 24828 +217.168.17.5 - - [17/May/2015:08:05:12 +0000] "GET /downloads/product_2 HTTP/1.1" 200 3316 "-" "-" 6947 +188.138.60.101 - - [17/May/2015:08:05:49 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 28288 +80.91.33.133 - - [17/May/2015:08:05:14 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 23182 +46.4.66.76 - - [17/May/2015:08:05:45 +0000] "GET /downloads/product_1 HTTP/1.1" 404 318 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 16302 +93.180.71.3 - - [17/May/2015:08:05:26 +0000] "GET /downloads/product_1 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" 16102 +91.234.194.89 - - [17/May/2015:08:05:22 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 20268 +80.91.33.133 - - [17/May/2015:08:05:07 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 2794 +37.26.93.214 - - [17/May/2015:08:05:38 +0000] "GET /downloads/product_2 HTTP/1.1" 404 319 "-" "Go 1.1 package http" 22809 +188.138.60.101 - - [17/May/2015:08:05:25 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 8807 +93.180.71.3 - - [17/May/2015:08:05:11 +0000] "GET /downloads/product_1 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" 30172 +46.4.66.76 - - [17/May/2015:08:05:02 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 1973 +62.75.198.179 - - [17/May/2015:08:05:06 +0000] "GET /downloads/product_2 HTTP/1.1" 200 490 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 10182 +80.91.33.133 - - [17/May/2015:08:05:55 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 14307 +173.203.139.108 - - [17/May/2015:08:05:53 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 10828 +210.245.80.75 - - [17/May/2015:08:05:32 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 21956 +46.4.83.163 - - [17/May/2015:08:05:52 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 5726 +91.234.194.89 - - [17/May/2015:08:05:18 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 10841 +31.22.86.126 - - [17/May/2015:08:05:24 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 18132 +217.168.17.5 - - [17/May/2015:08:05:25 +0000] "GET /downloads/product_1 HTTP/1.1" 200 3301 "-" "-" 10094 +80.91.33.133 - - [17/May/2015:08:05:50 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 12355 +173.203.139.108 - - [17/May/2015:08:05:03 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 27325 +80.91.33.133 - - [17/May/2015:08:05:35 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 14101 +5.83.131.103 - - [17/May/2015:08:05:51 +0000] "GET /downloads/product_1 HTTP/1.1" 200 490 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 20175 +80.91.33.133 - - [17/May/2015:08:05:59 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 21384 +200.6.73.40 - - [17/May/2015:08:05:42 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 6570 +80.91.33.133 - - [17/May/2015:08:05:48 +0000] "GET /downloads/product_1 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 26145 +93.180.71.3 - - [17/May/2015:08:05:58 +0000] "GET /downloads/product_1 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" 32705 +62.75.198.179 - - [17/May/2015:08:05:39 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 18865 +50.57.209.92 - - [17/May/2015:08:05:41 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 21639 +188.138.60.101 - - [17/May/2015:08:05:48 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 31242 +46.4.66.76 - - [17/May/2015:08:05:02 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 5910 +50.57.209.92 - - [17/May/2015:08:05:25 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 22900 +91.239.186.133 - - [17/May/2015:08:05:04 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 23919 +173.203.139.108 - - [17/May/2015:08:05:08 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 25169 +80.91.33.133 - - [17/May/2015:08:05:04 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 24395 +93.190.71.150 - - [17/May/2015:08:05:33 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 25750 +91.234.194.89 - - [17/May/2015:08:05:57 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 26673 +46.4.83.163 - - [17/May/2015:08:05:20 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 32509 +173.203.139.108 - - [17/May/2015:08:05:39 +0000] "GET /downloads/product_1 HTTP/1.1" 404 335 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 32714 +54.187.216.43 - - [17/May/2015:08:05:07 +0000] "GET /downloads/product_2 HTTP/1.1" 200 951 "-" "urlgrabber/3.9.1 yum/3.4.3" 5016 +50.57.209.92 - - [17/May/2015:08:05:59 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 14449 +80.91.33.133 - - [17/May/2015:08:05:02 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 13183 +173.203.139.108 - - [17/May/2015:08:05:07 +0000] "GET /downloads/product_1 HTTP/1.1" 404 332 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 7791 +5.83.131.103 - - [17/May/2015:08:05:31 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 586 +173.203.139.108 - - [17/May/2015:08:05:14 +0000] "GET /downloads/product_1 HTTP/1.1" 404 334 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 5036 +80.91.33.133 - - [17/May/2015:08:05:46 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 20358 +50.57.209.92 - - [17/May/2015:08:05:01 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 2106 +80.91.33.133 - - [17/May/2015:08:05:41 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 9757 +37.26.93.214 - - [17/May/2015:08:05:52 +0000] "GET /downloads/product_2 HTTP/1.1" 200 3318 "-" "Go 1.1 package http" 6222 +23.23.226.37 - - [17/May/2015:08:05:19 +0000] "GET /downloads/product_2 HTTP/1.1" 200 2578 "-" "urlgrabber/3.9.1 yum/3.4.3" 9523 +93.180.71.3 - - [17/May/2015:08:05:20 +0000] "GET /downloads/product_1 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" 7228 +173.203.139.108 - - [17/May/2015:08:05:56 +0000] "GET /downloads/product_1 HTTP/1.1" 404 331 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 31464 +62.75.198.179 - - [17/May/2015:08:05:13 +0000] "GET /downloads/product_2 HTTP/1.1" 404 346 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 22462 +31.22.86.126 - - [17/May/2015:08:05:10 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 29906 +50.57.209.92 - - [17/May/2015:08:05:58 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 16217 +91.239.186.133 - - [17/May/2015:08:05:11 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 18335 +46.4.66.76 - - [17/May/2015:08:05:00 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 27375 +200.6.73.40 - - [17/May/2015:08:05:23 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 32073 +173.203.139.108 - - [17/May/2015:08:05:13 +0000] "GET /downloads/product_1 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 31071 +93.190.71.150 - - [17/May/2015:08:05:35 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 1200 +91.234.194.89 - - [17/May/2015:08:05:26 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 13143 +173.203.139.108 - - [17/May/2015:08:05:18 +0000] "GET /downloads/product_1 HTTP/1.1" 404 333 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 16138 +80.91.33.133 - - [17/May/2015:08:05:23 +0000] "GET /downloads/product_1 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 21432 +217.168.17.5 - - [17/May/2015:08:05:27 +0000] "GET /downloads/product_2 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.8.10.3)" 1419 +46.4.83.163 - - [17/May/2015:08:05:54 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 28449 +80.91.33.133 - - [17/May/2015:08:05:25 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 25906 +50.57.209.92 - - [17/May/2015:08:05:56 +0000] "GET /downloads/product_2 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 27099 +173.203.139.108 - - [17/May/2015:08:05:52 +0000] "GET /downloads/product_1 HTTP/1.1" 404 334 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 32238 +188.138.60.101 - - [17/May/2015:08:05:04 +0000] "GET /downloads/product_2 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 237 +80.91.33.133 - - [17/May/2015:08:05:11 +0000] "GET /downloads/product_1 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 7103 +134.119.20.172 - - [17/May/2015:08:05:26 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 5423 +173.203.139.108 - - [17/May/2015:08:05:29 +0000] "GET /downloads/product_1 HTTP/1.1" 404 331 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 6373 +80.91.33.133 - - [17/May/2015:08:05:44 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 22230 +91.121.161.213 - - [17/May/2015:08:05:14 +0000] "GET /downloads/product_2 HTTP/1.1" 200 490 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 14196 +80.91.33.133 - - [17/May/2015:08:05:17 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 17820 +80.91.33.133 - - [17/May/2015:08:05:27 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 9097 +37.26.93.214 - - [17/May/2015:08:05:03 +0000] "GET /downloads/product_2 HTTP/1.1" 200 490 "-" "Go 1.1 package http" 27632 +5.83.131.103 - - [17/May/2015:08:05:57 +0000] "GET /downloads/product_1 HTTP/1.1" 404 346 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 14609 +50.57.209.92 - - [17/May/2015:08:05:39 +0000] "GET /downloads/product_2 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 21926 +173.203.139.108 - - [17/May/2015:08:05:52 +0000] "GET /downloads/product_1 HTTP/1.1" 404 331 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 4915 +54.64.16.235 - - [17/May/2015:08:05:13 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.20.1)" 12816 +93.180.71.3 - - [17/May/2015:08:05:28 +0000] "GET /downloads/product_1 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" 30742 +202.143.95.26 - - [17/May/2015:08:05:55 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 24544 +202.143.95.26 - - [17/May/2015:08:05:58 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 25819 +202.143.95.26 - - [17/May/2015:08:05:01 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 26831 +80.91.33.133 - - [17/May/2015:08:05:14 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 1344 +91.239.186.133 - - [17/May/2015:08:05:03 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 4987 +173.203.139.108 - - [17/May/2015:08:05:35 +0000] "GET /downloads/product_1 HTTP/1.1" 404 328 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 13419 +80.91.33.133 - - [17/May/2015:08:05:39 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 12879 +87.233.156.242 - - [17/May/2015:08:05:37 +0000] "GET /downloads/product_2 HTTP/1.1" 404 318 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 20611 +62.75.198.179 - - [17/May/2015:08:05:33 +0000] "GET /downloads/product_2 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 1387 +50.57.209.92 - - [17/May/2015:08:05:16 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 31286 +80.91.33.133 - - [17/May/2015:08:05:53 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 15247 +93.190.71.150 - - [17/May/2015:08:05:34 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 134 +46.4.66.76 - - [17/May/2015:08:05:38 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 23909 +80.91.33.133 - - [17/May/2015:08:05:09 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 15771 +91.234.194.89 - - [17/May/2015:08:05:58 +0000] "GET /downloads/product_2 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 4641 +217.168.17.5 - - [17/May/2015:08:05:07 +0000] "GET /downloads/product_1 HTTP/1.1" 404 341 "-" "Debian APT-HTTP/1.3 (0.8.10.3)" 6382 +46.4.83.163 - - [17/May/2015:08:05:23 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 14599 +50.57.209.92 - - [17/May/2015:08:05:19 +0000] "GET /downloads/product_1 HTTP/1.1" 404 335 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 8263 +200.6.73.40 - - [17/May/2015:08:05:46 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 23514 +91.121.161.213 - - [17/May/2015:08:05:28 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 29473 +80.91.33.133 - - [17/May/2015:08:05:52 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 26659 +188.138.60.101 - - [17/May/2015:08:05:22 +0000] "GET /downloads/product_2 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 5147 +144.76.151.58 - - [17/May/2015:08:05:54 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 21698 +134.119.20.172 - - [17/May/2015:09:05:28 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 21077 +80.91.33.133 - - [17/May/2015:09:05:26 +0000] "GET /downloads/product_1 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 7173 +80.91.33.133 - - [17/May/2015:09:05:55 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 1878 +5.83.131.103 - - [17/May/2015:09:05:08 +0000] "GET /downloads/product_1 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 24451 +93.180.71.3 - - [17/May/2015:09:05:18 +0000] "GET /downloads/product_1 HTTP/1.1" 404 341 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" 30170 +80.91.33.133 - - [17/May/2015:09:05:05 +0000] "GET /downloads/product_1 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 13156 +50.57.209.92 - - [17/May/2015:09:05:25 +0000] "GET /downloads/product_1 HTTP/1.1" 404 332 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 306 +5.83.131.103 - - [17/May/2015:09:05:18 +0000] "GET /downloads/product_1 HTTP/1.1" 404 345 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 24862 +62.75.167.106 - - [17/May/2015:09:05:56 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 10227 +37.26.93.214 - - [17/May/2015:09:05:42 +0000] "GET /downloads/product_2 HTTP/1.1" 404 339 "-" "Go 1.1 package http" 28504 +93.64.134.186 - - [17/May/2015:09:05:29 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 27681 +87.233.156.242 - - [17/May/2015:09:05:36 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 1502 +80.91.33.133 - - [17/May/2015:09:05:26 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 18177 +80.91.33.133 - - [17/May/2015:09:05:15 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 7934 +54.193.30.212 - - [17/May/2015:09:05:23 +0000] "GET /downloads/product_2 HTTP/1.1" 200 951 "-" "urlgrabber/3.9.1 yum/3.4.3" 2 +62.75.198.179 - - [17/May/2015:09:05:09 +0000] "GET /downloads/product_2 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 23920 +91.239.186.133 - - [17/May/2015:09:05:46 +0000] "GET /downloads/product_2 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 9333 +83.161.14.106 - - [17/May/2015:09:05:09 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 19640 +80.91.33.133 - - [17/May/2015:09:05:54 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 11061 +80.91.33.133 - - [17/May/2015:09:05:46 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 24501 +93.190.71.150 - - [17/May/2015:09:05:38 +0000] "GET /downloads/product_2 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 15895 +50.57.209.92 - - [17/May/2015:09:05:40 +0000] "GET /downloads/product_2 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 20558 +80.91.33.133 - - [17/May/2015:09:05:49 +0000] "GET /downloads/product_1 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 2338 +80.91.33.133 - - [17/May/2015:09:05:25 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 12192 +217.168.17.5 - - [17/May/2015:09:05:09 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.10.3)" 9824 +80.91.33.133 - - [17/May/2015:09:05:59 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 2246 +54.191.136.177 - - [17/May/2015:09:05:08 +0000] "GET /downloads/product_2 HTTP/1.1" 200 951 "-" "urlgrabber/3.9.1 yum/3.4.3" 7239 +80.91.33.133 - - [17/May/2015:09:05:27 +0000] "GET /downloads/product_1 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 21154 +91.234.194.89 - - [17/May/2015:09:05:57 +0000] "GET /downloads/product_2 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 2966 +80.91.33.133 - - [17/May/2015:09:05:05 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 10715 +80.91.33.133 - - [17/May/2015:09:05:22 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 14856 +46.4.83.163 - - [17/May/2015:09:05:12 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 17717 +91.121.161.213 - - [17/May/2015:09:05:58 +0000] "GET /downloads/product_2 HTTP/1.1" 404 346 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 9951 +188.138.60.101 - - [17/May/2015:09:05:57 +0000] "GET /downloads/product_2 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 25787 +144.76.151.58 - - [17/May/2015:09:05:33 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 4930 +195.154.77.170 - - [17/May/2015:09:05:00 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 21921 +50.57.209.92 - - [17/May/2015:09:05:19 +0000] "GET /downloads/product_2 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 29773 +31.22.86.126 - - [17/May/2015:09:05:41 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 7593 +54.64.16.235 - - [17/May/2015:09:05:51 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.20.1)" 26867 +202.143.95.26 - - [17/May/2015:09:05:20 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 31361 +202.143.95.26 - - [17/May/2015:09:05:28 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 13167 +87.233.156.242 - - [17/May/2015:09:05:47 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 22554 +62.75.167.106 - - [17/May/2015:09:05:37 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 29795 +152.90.220.17 - - [17/May/2015:09:05:01 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 18753 +80.91.33.133 - - [17/May/2015:09:05:02 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 27083 +93.180.71.3 - - [17/May/2015:09:05:38 +0000] "GET /downloads/product_1 HTTP/1.1" 404 338 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" 28187 +80.91.33.133 - - [17/May/2015:09:05:03 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 25595 +5.83.131.103 - - [17/May/2015:09:05:15 +0000] "GET /downloads/product_1 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 26070 +5.83.131.103 - - [17/May/2015:09:05:56 +0000] "GET /downloads/product_1 HTTP/1.1" 404 338 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 27724 +200.6.73.40 - - [17/May/2015:09:05:33 +0000] "GET /downloads/product_1 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 8086 +46.4.88.134 - - [17/May/2015:09:05:49 +0000] "GET /downloads/product_1 HTTP/1.1" 404 318 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 4853 +50.57.209.92 - - [17/May/2015:09:05:34 +0000] "GET /downloads/product_1 HTTP/1.1" 404 334 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 9464 +93.64.134.186 - - [17/May/2015:09:05:28 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 12194 +80.91.33.133 - - [17/May/2015:09:05:50 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 26621 +62.75.198.180 - - [17/May/2015:09:05:55 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 29857 +80.91.33.133 - - [17/May/2015:09:05:07 +0000] "GET /downloads/product_1 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 20514 +80.91.33.133 - - [17/May/2015:09:05:36 +0000] "GET /downloads/product_1 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 5526 +62.75.198.179 - - [17/May/2015:09:05:46 +0000] "GET /downloads/product_2 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 14143 +80.91.33.133 - - [17/May/2015:09:05:17 +0000] "GET /downloads/product_1 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 20873 +91.239.186.133 - - [17/May/2015:09:05:16 +0000] "GET /downloads/product_2 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 23230 +80.91.33.133 - - [17/May/2015:09:05:25 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 25246 +83.161.14.106 - - [17/May/2015:09:05:45 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 19052 +80.91.33.133 - - [17/May/2015:09:05:31 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 12362 +195.154.77.170 - - [17/May/2015:09:05:35 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 10153 +93.190.71.150 - - [17/May/2015:09:05:56 +0000] "GET /downloads/product_2 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 22418 +80.91.33.133 - - [17/May/2015:09:05:43 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 6565 +80.91.33.133 - - [17/May/2015:09:05:44 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 9883 +144.76.160.62 - - [17/May/2015:09:05:38 +0000] "GET /downloads/product_2 HTTP/1.1" 404 318 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 2564 +91.121.161.213 - - [17/May/2015:09:05:34 +0000] "GET /downloads/product_2 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 17140 +46.4.83.163 - - [17/May/2015:09:05:10 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 22794 +91.234.194.89 - - [17/May/2015:09:05:42 +0000] "GET /downloads/product_2 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 17718 +50.57.209.92 - - [17/May/2015:09:05:40 +0000] "GET /downloads/product_1 HTTP/1.1" 404 331 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 5434 +188.138.60.101 - - [17/May/2015:09:05:41 +0000] "GET /downloads/product_2 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 573 +210.245.80.75 - - [17/May/2015:09:05:07 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 28482 +144.76.151.58 - - [17/May/2015:09:05:28 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 31161 +80.91.33.133 - - [17/May/2015:09:05:11 +0000] "GET /downloads/product_1 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 24151 +144.76.117.56 - - [17/May/2015:09:05:59 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 6185 +80.91.33.133 - - [17/May/2015:09:05:07 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 6276 +31.22.86.126 - - [17/May/2015:09:05:19 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 27127 +80.91.33.133 - - [17/May/2015:09:05:17 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 9549 +62.75.167.106 - - [17/May/2015:09:05:03 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 21397 +87.233.156.242 - - [17/May/2015:09:05:17 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 10781 +152.90.220.18 - - [17/May/2015:09:05:11 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 19773 +93.180.71.3 - - [17/May/2015:09:05:01 +0000] "GET /downloads/product_1 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" 11889 +80.91.33.133 - - [17/May/2015:09:05:54 +0000] "GET /downloads/product_1 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 14111 +31.22.86.126 - - [17/May/2015:09:05:07 +0000] "GET /downloads/product_1 HTTP/1.1" 404 319 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 17787 +50.57.209.92 - - [17/May/2015:09:05:42 +0000] "GET /downloads/product_2 HTTP/1.1" 404 341 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 18330 +5.83.131.103 - - [17/May/2015:09:05:49 +0000] "GET /downloads/product_1 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 8993 +46.4.88.134 - - [17/May/2015:09:05:51 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 17460 +80.91.33.133 - - [17/May/2015:09:05:06 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 32412 +80.91.33.133 - - [17/May/2015:09:05:19 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 12639 +62.75.198.180 - - [17/May/2015:09:05:43 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 32511 +80.91.33.133 - - [17/May/2015:09:05:22 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 29012 +80.91.33.133 - - [17/May/2015:09:05:23 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 9767 +5.83.131.103 - - [17/May/2015:09:05:07 +0000] "GET /downloads/product_1 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 12212 +5.83.131.103 - - [17/May/2015:09:05:22 +0000] "GET /downloads/product_1 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 2440 +5.83.131.103 - - [17/May/2015:09:05:27 +0000] "GET /downloads/product_1 HTTP/1.1" 404 338 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 8157 +195.154.77.170 - - [17/May/2015:09:05:23 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 16242 +202.143.95.26 - - [17/May/2015:09:05:08 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 22261 +93.64.134.186 - - [17/May/2015:09:05:19 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 15048 +85.214.47.178 - - [17/May/2015:09:05:39 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 27105 +83.161.14.106 - - [17/May/2015:09:05:15 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 32234 +80.70.214.71 - - [17/May/2015:09:05:20 +0000] "HEAD /downloads/product_1 HTTP/1.1" 200 0 "-" "Wget/1.13.4 (linux-gnu)" 8479 +87.233.156.242 - - [17/May/2015:09:05:08 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 20831 +54.64.16.235 - - [17/May/2015:09:05:55 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.20.1)" 18289 +50.57.209.92 - - [17/May/2015:09:05:29 +0000] "GET /downloads/product_2 HTTP/1.1" 404 338 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 9858 +91.239.186.133 - - [17/May/2015:09:05:00 +0000] "GET /downloads/product_2 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 20442 +91.121.161.213 - - [17/May/2015:09:05:09 +0000] "GET /downloads/product_2 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 9004 +200.6.73.40 - - [17/May/2015:09:05:30 +0000] "GET /downloads/product_1 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 13221 +62.75.198.179 - - [17/May/2015:09:05:49 +0000] "GET /downloads/product_2 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 954 +93.190.71.150 - - [17/May/2015:09:05:13 +0000] "GET /downloads/product_2 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 26398 +80.91.33.133 - - [17/May/2015:09:05:33 +0000] "GET /downloads/product_1 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 22775 +80.91.33.133 - - [17/May/2015:09:05:32 +0000] "GET /downloads/product_1 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 13886 +80.91.33.133 - - [17/May/2015:09:05:49 +0000] "GET /downloads/product_1 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 19340 +144.76.160.62 - - [17/May/2015:09:05:11 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 17157 +80.91.33.133 - - [17/May/2015:09:05:59 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 9971 +217.168.17.5 - - [17/May/2015:09:05:12 +0000] "GET /downloads/product_1 HTTP/1.1" 404 334 "-" "Debian APT-HTTP/1.3 (0.8.10.3)" 26268 +80.91.33.133 - - [17/May/2015:09:05:47 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 5983 +80.91.33.133 - - [17/May/2015:09:05:09 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 15296 +144.76.117.56 - - [17/May/2015:09:05:52 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 13922 +144.76.151.58 - - [17/May/2015:09:05:42 +0000] "GET /downloads/product_2 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 10692 +80.91.33.133 - - [17/May/2015:10:05:40 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 22550 +62.75.167.106 - - [17/May/2015:10:05:47 +0000] "GET /downloads/product_2 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 20757 +80.91.33.133 - - [17/May/2015:10:05:51 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 25956 +37.187.238.39 - - [17/May/2015:10:05:22 +0000] "GET /downloads/product_2 HTTP/1.1" 404 318 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 16674 +80.70.214.71 - - [17/May/2015:10:05:13 +0000] "GET /downloads/product_2 HTTP/1.1" 404 327 "-" "Wget/1.13.4 (linux-gnu)" 15327 +91.234.194.89 - - [17/May/2015:10:05:48 +0000] "GET /downloads/product_2 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 21807 +80.91.33.133 - - [17/May/2015:10:05:10 +0000] "GET /downloads/product_1 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 20469 +188.138.60.101 - - [17/May/2015:10:05:58 +0000] "GET /downloads/product_2 HTTP/1.1" 404 341 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 10122 +80.91.33.133 - - [17/May/2015:10:05:01 +0000] "GET /downloads/product_1 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 1971 +80.91.33.133 - - [17/May/2015:10:05:32 +0000] "GET /downloads/product_1 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 7263 +93.180.71.3 - - [17/May/2015:10:05:28 +0000] "GET /downloads/product_1 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" 953 +46.4.88.134 - - [17/May/2015:10:05:54 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 23703 +80.91.33.133 - - [17/May/2015:10:05:53 +0000] "GET /downloads/product_1 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 126 +62.210.138.59 - - [17/May/2015:10:05:22 +0000] "GET /downloads/product_2 HTTP/1.1" 404 318 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 19171 +31.22.86.126 - - [17/May/2015:10:05:38 +0000] "GET /downloads/product_1 HTTP/1.1" 404 335 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 31107 +80.91.33.133 - - [17/May/2015:10:05:16 +0000] "GET /downloads/product_1 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 8252 +54.86.157.236 - - [17/May/2015:10:05:24 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.20.1)" 25651 +195.154.233.202 - - [17/May/2015:10:05:39 +0000] "GET /downloads/product_2 HTTP/1.1" 404 318 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 3446 +54.86.157.236 - - [17/May/2015:10:05:43 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.20.1)" 20770 +80.91.33.133 - - [17/May/2015:10:05:14 +0000] "GET /downloads/product_1 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 27979 +94.23.21.169 - - [17/May/2015:10:05:09 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 28723 +54.86.157.236 - - [17/May/2015:10:05:18 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.20.1)" 13439 +195.154.77.170 - - [17/May/2015:10:05:17 +0000] "GET /downloads/product_2 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 22432 +54.86.157.236 - - [17/May/2015:10:05:36 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.20.1)" 1572 +85.214.47.178 - - [17/May/2015:10:05:57 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 27196 +5.83.131.103 - - [17/May/2015:10:05:55 +0000] "GET /downloads/product_1 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 9637 +5.83.131.103 - - [17/May/2015:10:05:03 +0000] "GET /downloads/product_1 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 18830 +5.83.131.103 - - [17/May/2015:10:05:05 +0000] "GET /downloads/product_1 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 844 +5.83.131.103 - - [17/May/2015:10:05:08 +0000] "GET /downloads/product_1 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 20882 +80.91.33.133 - - [17/May/2015:10:05:40 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 1325 +80.91.33.133 - - [17/May/2015:10:05:39 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 11125 +84.53.65.28 - - [17/May/2015:10:05:25 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 10771 +80.91.33.133 - - [17/May/2015:10:05:33 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 24891 +54.86.157.236 - - [17/May/2015:10:05:28 +0000] "GET /downloads/product_1 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.20.1)" 23541 +217.168.17.5 - - [17/May/2015:10:05:02 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.10.3)" 22323 +91.121.161.213 - - [17/May/2015:10:05:18 +0000] "GET /downloads/product_2 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 29114 +80.70.214.71 - - [17/May/2015:10:05:33 +0000] "GET /downloads/product_1 HTTP/1.1" 404 329 "-" "Wget/1.13.4 (linux-gnu)" 13629 +144.76.160.62 - - [17/May/2015:10:05:10 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 32440 +54.86.157.236 - - [17/May/2015:10:05:52 +0000] "GET /downloads/product_1 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.20.1)" 20402 +93.64.134.186 - - [17/May/2015:10:05:54 +0000] "GET /downloads/product_2 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 5113 +93.190.71.150 - - [17/May/2015:10:05:41 +0000] "GET /downloads/product_2 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 31729 +87.233.156.242 - - [17/May/2015:10:05:02 +0000] "GET /downloads/product_2 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 28958 +80.91.33.133 - - [17/May/2015:10:05:22 +0000] "GET /downloads/product_1 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 15630 +91.239.186.133 - - [17/May/2015:10:05:50 +0000] "GET /downloads/product_2 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 7488 +62.75.198.179 - - [17/May/2015:10:05:28 +0000] "GET /downloads/product_2 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 9316 +144.76.117.56 - - [17/May/2015:10:05:46 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 9965 +178.32.54.253 - - [17/May/2015:10:05:33 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 2881 +37.187.238.39 - - [17/May/2015:10:05:05 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 17544 +83.161.14.106 - - [17/May/2015:10:05:47 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 11419 +54.86.157.236 - - [17/May/2015:10:05:48 +0000] "GET /downloads/product_1 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.20.1)" 16406 +91.194.188.90 - - [17/May/2015:10:05:51 +0000] "HEAD /downloads/product_2 HTTP/1.1" 200 0 "-" "Wget/1.13.4 (linux-gnu)" 28324 +83.161.14.106 - - [17/May/2015:10:05:13 +0000] "GET /downloads/product_2 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 1893 +80.91.33.133 - - [17/May/2015:10:05:18 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 14697 +93.180.71.3 - - [17/May/2015:10:05:34 +0000] "GET /downloads/product_1 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" 16168 +62.210.138.59 - - [17/May/2015:10:05:40 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 663 +46.4.88.134 - - [17/May/2015:10:05:16 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 27962 +202.143.95.26 - - [17/May/2015:10:05:50 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 18539 +202.143.95.26 - - [17/May/2015:10:05:02 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 13495 +202.143.95.26 - - [17/May/2015:10:05:10 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 3192 +62.75.198.180 - - [17/May/2015:10:05:36 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 4349 +144.76.137.134 - - [17/May/2015:10:05:03 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 1395 +80.91.33.133 - - [17/May/2015:10:05:23 +0000] "GET /downloads/product_1 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 12898 +54.86.157.236 - - [17/May/2015:10:05:11 +0000] "GET /downloads/product_1 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.20.1)" 26930 +80.70.214.71 - - [17/May/2015:10:05:22 +0000] "GET /downloads/product_2 HTTP/1.1" 404 326 "-" "Wget/1.13.4 (linux-gnu)" 16662 +91.234.194.89 - - [17/May/2015:10:05:06 +0000] "GET /downloads/product_2 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 9445 +188.138.60.101 - - [17/May/2015:10:05:38 +0000] "GET /downloads/product_2 HTTP/1.1" 404 338 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 18804 +80.91.33.133 - - [17/May/2015:10:05:33 +0000] "GET /downloads/product_1 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 22429 +195.154.233.202 - - [17/May/2015:10:05:47 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 8456 +94.23.21.169 - - [17/May/2015:10:05:58 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 32187 +144.76.151.58 - - [17/May/2015:10:05:10 +0000] "GET /downloads/product_2 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 29276 +80.91.33.133 - - [17/May/2015:10:05:42 +0000] "GET /downloads/product_1 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 9700 +62.75.167.106 - - [17/May/2015:10:05:31 +0000] "GET /downloads/product_2 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 10078 +80.91.33.133 - - [17/May/2015:10:05:41 +0000] "GET /downloads/product_1 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 7600 +50.57.209.92 - - [17/May/2015:10:05:16 +0000] "GET /downloads/product_1 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 8540 +202.143.95.26 - - [17/May/2015:10:05:43 +0000] "GET /downloads/product_2 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 24400 +200.6.73.40 - - [17/May/2015:10:05:38 +0000] "GET /downloads/product_1 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 29363 +195.154.77.170 - - [17/May/2015:10:05:33 +0000] "GET /downloads/product_2 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 17025 +54.187.216.43 - - [17/May/2015:10:05:55 +0000] "GET /downloads/product_2 HTTP/1.1" 200 951 "-" "urlgrabber/3.9.1 yum/3.4.3" 27997 +80.91.33.133 - - [17/May/2015:10:05:04 +0000] "GET /downloads/product_1 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 1806 +80.91.33.133 - - [17/May/2015:10:05:09 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 28234 +54.86.157.236 - - [17/May/2015:10:05:06 +0000] "GET /downloads/product_1 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.20.1)" 19286 +202.143.95.26 - - [17/May/2015:10:05:05 +0000] "GET /downloads/product_2 HTTP/1.1" 404 325 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 19522 +202.143.95.26 - - [17/May/2015:10:05:40 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 23841 +54.86.157.236 - - [17/May/2015:10:05:02 +0000] "GET /downloads/product_1 HTTP/1.1" 404 341 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.20.1)" 31135 +80.91.33.133 - - [17/May/2015:10:05:50 +0000] "GET /downloads/product_1 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 21510 +80.91.33.133 - - [17/May/2015:10:05:51 +0000] "GET /downloads/product_1 HTTP/1.1" 404 341 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 26977 +80.91.33.133 - - [17/May/2015:10:05:55 +0000] "GET /downloads/product_1 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 1078 +80.91.33.133 - - [17/May/2015:10:05:47 +0000] "GET /downloads/product_1 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 7473 +84.53.65.28 - - [17/May/2015:10:05:30 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 28347 +92.50.100.22 - - [17/May/2015:10:05:15 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 8699 +85.214.47.178 - - [17/May/2015:10:05:30 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 2078 +80.91.33.133 - - [17/May/2015:10:05:08 +0000] "GET /downloads/product_1 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 7013 +54.86.157.236 - - [17/May/2015:10:05:36 +0000] "GET /downloads/product_1 HTTP/1.1" 404 338 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.20.1)" 29440 +5.83.131.103 - - [17/May/2015:10:05:05 +0000] "GET /downloads/product_1 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 24206 +37.187.238.39 - - [17/May/2015:10:05:33 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 5674 +80.91.33.133 - - [17/May/2015:10:05:04 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 15781 +195.210.47.239 - - [17/May/2015:10:05:49 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" 1462 +80.91.33.133 - - [17/May/2015:10:05:11 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 9446 +54.64.16.235 - - [17/May/2015:10:05:12 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.20.1)" 23687 +178.32.54.253 - - [17/May/2015:10:05:54 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 17314 +144.92.16.161 - - [17/May/2015:10:05:39 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" 4021 +54.86.157.236 - - [17/May/2015:10:05:51 +0000] "GET /downloads/product_1 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.20.1)" 13168 +87.233.156.242 - - [17/May/2015:10:05:49 +0000] "GET /downloads/product_2 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 8142 +31.22.86.126 - - [17/May/2015:10:05:18 +0000] "GET /downloads/product_1 HTTP/1.1" 404 332 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 28923 +80.91.33.133 - - [17/May/2015:10:05:49 +0000] "GET /downloads/product_1 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 17021 +91.121.161.213 - - [17/May/2015:10:05:48 +0000] "GET /downloads/product_2 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 711 +80.91.33.133 - - [17/May/2015:10:05:06 +0000] "GET /downloads/product_1 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 15815 +50.57.209.92 - - [17/May/2015:10:05:19 +0000] "GET /downloads/product_1 HTTP/1.1" 404 333 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 12290 +91.239.186.133 - - [17/May/2015:10:05:15 +0000] "GET /downloads/product_2 HTTP/1.1" 404 341 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 9172 +144.76.117.56 - - [17/May/2015:10:05:31 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 27106 +144.76.160.62 - - [17/May/2015:10:05:47 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 2607 +62.210.138.59 - - [17/May/2015:10:05:45 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 26922 +54.86.157.236 - - [17/May/2015:10:05:07 +0000] "GET /downloads/product_1 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.20.1)" 2045 +62.75.198.179 - - [17/May/2015:10:05:14 +0000] "GET /downloads/product_2 HTTP/1.1" 404 338 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 14090 +93.190.71.150 - - [17/May/2015:10:05:07 +0000] "GET /downloads/product_2 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 2233 +144.76.117.56 - - [17/May/2015:10:05:18 +0000] "GET /downloads/product_1 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 14988 +94.23.21.169 - - [17/May/2015:10:05:23 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 11645 +91.194.188.90 - - [17/May/2015:10:05:05 +0000] "HEAD /downloads/product_2 HTTP/1.1" 200 0 "-" "Wget/1.13.4 (linux-gnu)" 28064 +93.64.134.186 - - [17/May/2015:10:05:51 +0000] "GET /downloads/product_2 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 16583 +54.86.157.236 - - [17/May/2015:10:05:48 +0000] "GET /downloads/product_1 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.20.1)" 23208 +80.70.214.71 - - [17/May/2015:10:05:23 +0000] "HEAD /downloads/product_2 HTTP/1.1" 200 0 "-" "Wget/1.13.4 (linux-gnu)" 1059 +93.180.71.3 - - [17/May/2015:10:05:22 +0000] "GET /downloads/product_1 HTTP/1.1" 404 333 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" 16367 +195.154.233.202 - - [17/May/2015:10:05:43 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 26788 +193.192.58.163 - - [17/May/2015:11:05:31 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 6753 +144.76.137.134 - - [17/May/2015:11:05:00 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 18307 +54.86.157.236 - - [17/May/2015:11:05:22 +0000] "GET /downloads/product_1 HTTP/1.1" 404 333 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.20.1)" 10520 +83.161.14.106 - - [17/May/2015:11:05:09 +0000] "GET /downloads/product_2 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 5640 +144.76.151.58 - - [17/May/2015:11:05:16 +0000] "GET /downloads/product_2 HTTP/1.1" 404 338 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 9992 +144.92.16.161 - - [17/May/2015:11:05:06 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" 3262 +195.154.77.170 - - [17/May/2015:11:05:20 +0000] "GET /downloads/product_2 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 17687 +62.75.198.180 - - [17/May/2015:11:05:05 +0000] "GET /downloads/product_2 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 18911 +91.234.194.89 - - [17/May/2015:11:05:29 +0000] "GET /downloads/product_2 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 22038 +80.91.33.133 - - [17/May/2015:11:05:28 +0000] "GET /downloads/product_1 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 2238 +188.138.60.101 - - [17/May/2015:11:05:38 +0000] "GET /downloads/product_2 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 10581 +62.75.167.106 - - [17/May/2015:11:05:58 +0000] "GET /downloads/product_2 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 14869 +46.4.88.134 - - [17/May/2015:11:05:51 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 6669 +80.91.33.133 - - [17/May/2015:11:05:35 +0000] "GET /downloads/product_1 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 12780 +80.91.33.133 - - [17/May/2015:11:05:05 +0000] "GET /downloads/product_1 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 24133 +84.53.65.28 - - [17/May/2015:11:05:25 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 14350 +152.90.220.17 - - [17/May/2015:11:05:08 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 23513 +80.91.33.133 - - [17/May/2015:11:05:07 +0000] "GET /downloads/product_1 HTTP/1.1" 404 341 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 31695 +80.91.33.133 - - [17/May/2015:11:05:21 +0000] "GET /downloads/product_1 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 12243 +178.32.54.253 - - [17/May/2015:11:05:44 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 2641 +54.72.39.202 - - [17/May/2015:11:05:19 +0000] "GET /downloads/product_2 HTTP/1.1" 200 951 "-" "urlgrabber/3.9.1 yum/3.4.3" 27639 +91.120.61.154 - - [17/May/2015:11:05:03 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 21180 +37.187.238.39 - - [17/May/2015:11:05:25 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 30661 +85.214.47.178 - - [17/May/2015:11:05:12 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 20380 +80.91.33.133 - - [17/May/2015:11:05:47 +0000] "GET /downloads/product_1 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 11957 +5.83.131.103 - - [17/May/2015:11:05:10 +0000] "GET /downloads/product_1 HTTP/1.1" 404 338 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 19230 +200.6.73.40 - - [17/May/2015:11:05:19 +0000] "GET /downloads/product_1 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 4087 +5.83.131.103 - - [17/May/2015:11:05:45 +0000] "GET /downloads/product_1 HTTP/1.1" 404 341 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 16383 +91.121.161.213 - - [17/May/2015:11:05:08 +0000] "GET /downloads/product_2 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 11487 +91.239.186.133 - - [17/May/2015:11:05:40 +0000] "GET /downloads/product_2 HTTP/1.1" 404 338 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 11774 +50.57.209.92 - - [17/May/2015:11:05:39 +0000] "GET /downloads/product_2 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 28472 +80.91.33.133 - - [17/May/2015:11:05:18 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 24011 +144.92.16.161 - - [17/May/2015:11:05:44 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" 26633 +87.233.156.242 - - [17/May/2015:11:05:33 +0000] "GET /downloads/product_2 HTTP/1.1" 404 338 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 16170 +94.23.21.169 - - [17/May/2015:11:05:56 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 15992 +5.83.131.103 - - [17/May/2015:11:05:31 +0000] "GET /downloads/product_1 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 20999 +80.91.33.133 - - [17/May/2015:11:05:40 +0000] "GET /downloads/product_1 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 23097 +202.143.95.26 - - [17/May/2015:11:05:30 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 3282 +202.143.95.26 - - [17/May/2015:11:05:44 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 4869 +80.91.33.133 - - [17/May/2015:11:05:28 +0000] "GET /downloads/product_1 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 9310 +80.91.33.133 - - [17/May/2015:11:05:51 +0000] "GET /downloads/product_1 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 23547 +80.91.33.133 - - [17/May/2015:11:05:11 +0000] "GET /downloads/product_1 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 5516 +80.91.33.133 - - [17/May/2015:11:05:13 +0000] "GET /downloads/product_1 HTTP/1.1" 404 338 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 26601 +62.210.138.59 - - [17/May/2015:11:05:23 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 26830 +144.76.160.62 - - [17/May/2015:11:05:06 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 15405 +93.190.71.150 - - [17/May/2015:11:05:29 +0000] "GET /downloads/product_2 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 16982 +80.91.33.133 - - [17/May/2015:11:05:00 +0000] "GET /downloads/product_1 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 6019 +202.143.95.26 - - [17/May/2015:11:05:29 +0000] "GET /downloads/product_2 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 3822 +193.192.58.163 - - [17/May/2015:11:05:54 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 13461 +195.154.233.202 - - [17/May/2015:11:05:46 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 32439 +80.70.214.71 - - [17/May/2015:11:05:59 +0000] "HEAD /downloads/product_2 HTTP/1.1" 200 0 "-" "Wget/1.13.4 (linux-gnu)" 31402 +62.75.198.179 - - [17/May/2015:11:05:17 +0000] "GET /downloads/product_2 HTTP/1.1" 404 341 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 452 +80.91.33.133 - - [17/May/2015:11:05:51 +0000] "GET /downloads/product_1 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 25508 +144.92.16.161 - - [17/May/2015:11:05:39 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" 29252 +195.154.77.170 - - [17/May/2015:11:05:28 +0000] "GET /downloads/product_2 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 19649 +50.57.209.92 - - [17/May/2015:11:05:56 +0000] "GET /downloads/product_2 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 24457 +144.76.117.56 - - [17/May/2015:11:05:49 +0000] "GET /downloads/product_1 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 10519 +80.91.33.133 - - [17/May/2015:11:05:36 +0000] "GET /downloads/product_1 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 6815 +144.76.137.134 - - [17/May/2015:11:05:07 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 798 +188.138.60.101 - - [17/May/2015:11:05:00 +0000] "GET /downloads/product_2 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 19441 +54.172.198.124 - - [17/May/2015:11:05:43 +0000] "GET /downloads/product_2 HTTP/1.1" 200 2582 "-" "urlgrabber/3.9.1 yum/3.4.3" 17903 +37.187.238.39 - - [17/May/2015:11:05:27 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 3443 +178.32.54.253 - - [17/May/2015:11:05:03 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 9634 +62.75.198.180 - - [17/May/2015:11:05:16 +0000] "GET /downloads/product_2 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 5417 +62.75.167.106 - - [17/May/2015:11:05:26 +0000] "GET /downloads/product_2 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 1055 +195.210.47.239 - - [17/May/2015:11:05:36 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" 4218 +91.234.194.89 - - [17/May/2015:11:05:48 +0000] "GET /downloads/product_2 HTTP/1.1" 404 341 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 23355 +31.22.86.126 - - [17/May/2015:11:05:19 +0000] "GET /downloads/product_1 HTTP/1.1" 404 334 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 29547 +91.194.188.90 - - [17/May/2015:11:05:42 +0000] "GET /downloads/product_2 HTTP/1.1" 404 340 "-" "Wget/1.13.4 (linux-gnu)" 26988 +92.50.100.22 - - [17/May/2015:11:05:35 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 13600 +144.76.151.58 - - [17/May/2015:11:05:45 +0000] "GET /downloads/product_2 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 18988 +93.64.134.186 - - [17/May/2015:11:05:48 +0000] "GET /downloads/product_2 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 2281 +85.214.47.178 - - [17/May/2015:11:05:19 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 16054 +94.23.21.169 - - [17/May/2015:11:05:11 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 21647 +80.91.33.133 - - [17/May/2015:11:05:31 +0000] "GET /downloads/product_1 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 31277 +80.91.33.133 - - [17/May/2015:11:05:20 +0000] "GET /downloads/product_1 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 19500 +91.121.161.213 - - [17/May/2015:11:05:03 +0000] "GET /downloads/product_2 HTTP/1.1" 404 338 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 29579 +83.161.14.106 - - [17/May/2015:11:05:52 +0000] "GET /downloads/product_2 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 1080 +54.64.16.235 - - [17/May/2015:11:05:43 +0000] "GET /downloads/product_2 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.20.1)" 15057 +84.53.65.28 - - [17/May/2015:11:05:31 +0000] "GET /downloads/product_2 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 5805 +80.91.33.133 - - [17/May/2015:11:05:09 +0000] "GET /downloads/product_1 HTTP/1.1" 404 338 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 32764 +50.57.209.92 - - [17/May/2015:11:05:15 +0000] "GET /downloads/product_1 HTTP/1.1" 404 334 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 28248 +91.239.186.133 - - [17/May/2015:11:05:17 +0000] "GET /downloads/product_2 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 32046 +144.92.16.161 - - [17/May/2015:11:05:30 +0000] "GET /downloads/product_1 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" 31342 +62.210.138.59 - - [17/May/2015:11:05:29 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 22861 +210.245.80.75 - - [17/May/2015:11:05:05 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 32649 +80.91.33.133 - - [17/May/2015:11:05:12 +0000] "GET /downloads/product_1 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 11268 +83.161.14.106 - - [17/May/2015:11:05:55 +0000] "GET /downloads/product_2 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 8233 +87.233.156.242 - - [17/May/2015:11:05:02 +0000] "GET /downloads/product_2 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 10052 +5.83.131.103 - - [17/May/2015:11:05:49 +0000] "GET /downloads/product_1 HTTP/1.1" 404 338 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 20084 +80.91.33.133 - - [17/May/2015:11:05:05 +0000] "GET /downloads/product_1 HTTP/1.1" 404 341 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 9007 +91.120.61.154 - - [17/May/2015:11:05:48 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 8410 +195.154.233.202 - - [17/May/2015:11:05:55 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 20582 +80.91.33.133 - - [17/May/2015:11:05:56 +0000] "GET /downloads/product_1 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 8327 +193.192.58.163 - - [17/May/2015:11:05:58 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 4041 +93.190.71.150 - - [17/May/2015:11:05:11 +0000] "GET /downloads/product_2 HTTP/1.1" 404 338 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 26973 +144.76.160.62 - - [17/May/2015:11:05:20 +0000] "GET /downloads/product_2 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 24342 +50.57.209.92 - - [17/May/2015:11:05:56 +0000] "GET /downloads/product_1 HTTP/1.1" 404 331 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 27744 +62.75.198.179 - - [17/May/2015:11:05:19 +0000] "GET /downloads/product_2 HTTP/1.1" 404 338 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 2455 +193.192.59.41 - - [17/May/2015:11:05:55 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 19596 +195.154.77.170 - - [17/May/2015:11:05:35 +0000] "GET /downloads/product_2 HTTP/1.1" 404 341 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 23424 +80.91.33.133 - - [17/May/2015:11:05:17 +0000] "GET /downloads/product_1 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 4171 +200.6.73.40 - - [17/May/2015:11:05:26 +0000] "GET /downloads/product_1 HTTP/1.1" 404 341 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 8274 +188.138.60.101 - - [17/May/2015:11:05:56 +0000] "GET /downloads/product_2 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 2949 +80.91.33.133 - - [17/May/2015:11:05:53 +0000] "GET /downloads/product_1 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 5641 +80.91.33.133 - - [17/May/2015:11:05:42 +0000] "GET /downloads/product_1 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 28746 +80.91.33.133 - - [17/May/2015:11:05:17 +0000] "GET /downloads/product_1 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 18396 +80.91.33.133 - - [17/May/2015:11:05:32 +0000] "GET /downloads/product_1 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 17638 +80.91.33.133 - - [17/May/2015:11:05:23 +0000] "GET /downloads/product_1 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.17)" 7865 +144.76.137.134 - - [17/May/2015:11:05:57 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 4280 +80.70.214.71 - - [17/May/2015:11:05:16 +0000] "GET /downloads/product_2 HTTP/1.1" 404 339 "-" "Wget/1.13.4 (linux-gnu)" 32436 +144.76.117.56 - - [17/May/2015:11:05:28 +0000] "GET /downloads/product_1 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 30048 +94.23.21.169 - - [17/May/2015:11:05:21 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 6186 +198.61.216.151 - - [17/May/2015:11:05:16 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 21567 +80.91.33.133 - - [17/May/2015:11:05:11 +0000] "GET /downloads/product_1 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 674 +91.194.188.90 - - [17/May/2015:11:05:32 +0000] "HEAD /downloads/product_2 HTTP/1.1" 200 0 "-" "Wget/1.13.4 (linux-gnu)" 5354 +62.75.198.180 - - [17/May/2015:11:05:39 +0000] "GET /downloads/product_2 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 5345 +80.91.33.133 - - [17/May/2015:11:05:52 +0000] "GET /downloads/product_1 HTTP/1.1" 404 341 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 2326 +31.22.86.126 - - [17/May/2015:12:05:15 +0000] "GET /downloads/product_1 HTTP/1.1" 404 331 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 3114 +84.53.65.28 - - [17/May/2015:12:05:38 +0000] "GET /downloads/product_2 HTTP/1.1" 404 337 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 9036 +144.92.16.161 - - [17/May/2015:12:05:32 +0000] "GET /downloads/product_1 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" 9410 +50.57.209.92 - - [17/May/2015:12:05:38 +0000] "GET /downloads/product_2 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 2039 +5.83.131.103 - - [17/May/2015:12:05:26 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 14852 +5.83.131.103 - - [17/May/2015:12:05:27 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 71 +62.75.167.106 - - [17/May/2015:12:05:01 +0000] "GET /downloads/product_2 HTTP/1.1" 404 341 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 6439 +178.32.54.253 - - [17/May/2015:12:05:26 +0000] "GET /downloads/product_1 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 8721 +91.121.161.213 - - [17/May/2015:12:05:00 +0000] "GET /downloads/product_2 HTTP/1.1" 404 341 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 1795 +91.234.194.89 - - [17/May/2015:12:05:11 +0000] "GET /downloads/product_2 HTTP/1.1" 404 338 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 8556 +37.187.238.39 - - [17/May/2015:12:05:29 +0000] "GET /downloads/product_2 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 17627 +91.239.186.133 - - [17/May/2015:12:05:38 +0000] "GET /downloads/product_2 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 10970 +87.233.156.242 - - [17/May/2015:12:05:34 +0000] "GET /downloads/product_2 HTTP/1.1" 404 333 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 409 +202.143.95.26 - - [17/May/2015:12:05:22 +0000] "GET /downloads/product_2 HTTP/1.1" 404 338 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 10283 +144.76.151.58 - - [17/May/2015:12:05:25 +0000] "GET /downloads/product_2 HTTP/1.1" 404 333 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 22461 +62.210.138.59 - - [17/May/2015:12:05:12 +0000] "GET /downloads/product_2 HTTP/1.1" 404 340 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" 22736 +80.91.33.133 - - [17/May/2015:12:05:05 +0000] "GET /downloads/product_1 HTTP/1.1" 404 336 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 21014 +83.161.14.106 - - [17/May/2015:12:05:48 +0000] "GET /downloads/product_2 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 18047 +80.91.33.133 - - [17/May/2015:12:05:31 +0000] "GET /downloads/product_1 HTTP/1.1" 404 341 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 25206 +5.83.131.103 - - [17/May/2015:12:05:21 +0000] "GET /downloads/product_1 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 15330 +80.91.33.133 - - [17/May/2015:12:05:54 +0000] "GET /downloads/product_1 HTTP/1.1" 404 339 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" 8763 +198.61.216.151 - - [17/May/2015:12:05:59 +0000] "GET /downloads/product_2 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.22)" 11132 +195.154.77.170 - - [17/May/2015:12:05:05 +0000] "GET /downloads/product_2 HTTP/1.1" 404 338 "-" "Debian APT-HTTP/1.3 (0.9.7.9)" 23768