diff --git a/.gitignore b/.gitignore index 8fdaa6b007..70966b8fa0 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,16 @@ guava-rpm-maker/\.project src-main src-test plugin_test.jar +bin/ + +#Docker +tools/docker/libs +tools/docker/*.jar +tools/docker/logback.xml +tools/docker/opentsdb.conf + +# FatJar +fat-jar-pom.xml +src-resources/ +test-resources/ +third_party/*/*.jar diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 1126638b1e..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,8 +0,0 @@ -language: java -before_script: ./build.sh pom.xml -script: export MAVEN_OPTS="-Xmx1024m" && mvn test --quiet -jdk: - - oraclejdk7 - - openjdk6 -notifications: - email: false diff --git a/AUTHORS b/AUTHORS index f52737fcdf..828a00bfd9 100644 --- a/AUTHORS +++ b/AUTHORS @@ -24,5 +24,6 @@ Chris Larsen David Bainbridge Geoffrey Anderson Ion Savin +Jonathan Creasy Nicholas Whitehead Will Moss diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000..0043b493eb --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,51 @@ +# Contribution Guidelines + +There are a number of talented developers creating tools for OpenTSDB or contributing code directly to the project. +If you are interested in helping, by adding new features, fixing bugs, adding tools or simply updating documentation, please read the guidelines below. Then sign the contributors agreement and send us a pull request! + +## Guidelines + + +- Please file [issues on GitHub](https://github.com/OpenTSDB/opentsdb/issues) after checking to see if anyone has posted a bug already. Make sure your bug reports contain enough details so they can be easily understood by others and quickly fixed. +- Read the Development page for tips +- The best way to contribute code is to fork the main repo and [send a pull request](https://help.github.com/articles/using-pull-requests) on GitHub. + - Bug fixes should be done in the `master` branch + - New features or major changes should be done in the `next` branch +- Alternatively, you can send a plain-text patch to the [mailing list](https://groups.google.com/forum/#!forum/opentsdb). +- Before your code changes can be included, please file the [Contribution License Agreement](https://docs.google.com/spreadsheet/embeddedform?formkey=dFNiOFROLXJBbFBmMkQtb1hNMWhUUnc6MQ). +- Unlike, say, the Apache Software Foundation, we do not require every single code change to be attached to an issue. Feel free to send as many small fixes as you want. +- Please break down your changes into as many small commits as possible. +- Please respect the coding style of the code you're changing. + - Indent code with 2 spaces, no tabs + - Keep code to 80 columns + - Curly brace on the same line as if, for, while, etc + - Variables need descriptive names `like_this` (instead of the typical Java style of `likeThis`) + - Methods named `likeThis()` starting with lower case letters + - Classes named `LikeThis`, starting with upper case letters + - Use the `final` keyword as much as you can, particularly in method parameters and returns statements + - Avoid checked exceptions as much as possible + - Always provide the most restrictive visibility to classes and members + - Javadoc all of your classes and methods. Some folks make use the Java API directly and we'll build docs for the site, so the more the merrier + - Don't add dependencies to the core OpenTSDB library unless absolutely necessary + - Add unit tests for any classes/methods you create and verify that your change doesn't break existing unit tests. We know UTs aren't fun, but they are useful + +## Git Repository + +OpenTSDB is maintained in [GitHub](https://github.com/OpenTSDB/opentsdb/). There are a limited number of branches and the purpose of each branch is as follows: + +- `maintenance` - This was the previously released version of OpenTSDB, usually the last minor version. E.g. if 2.3.0 or 2.3.1 is the current release, `maintenance` will have the last 2.2.x version. This branch +should rarely have PRs pointed at it. Rather patches against `master` that would apply to previous releases can be cherry-picked. +- `master` - The current release version of OpenTSDB. Only pull requests with bug fixes should be given against `master`. When enough PRs have been merged, we'll cut another PATCH version, e.g. 2.2.0 to 2.2.1. Patches with new features or behavior modifications should point to `next`. Patches against `master` should be cherry-picked to the downstream branches. +- `next` - This is the next minor release version of OpenTSDB and contains code in development or, when the version is marked as RC, then release candidate code. If the version is marked as a SNAPSHOT then new features can be applied to `next`. Once it moves to RC, then new features should be issued against the `put` branch. Otherwise only bug fixes should be given for RC code. +- `put` - When the next branch is in an RC state, new features should be applied against `put` which will be the next minor version. +- `X.0` - The next major version of OpenTSDB that may include breaking API changes. When the code is in a fairly stable state, it will be promoted up to `next` as an RC, then `master` for an official `release`. + +Any other branches are likely to be pruned at some point as they may contain stale code or temporary hacks. + +## Details + +- [General Development](http://opentsdb.net/docs/build/html/development/development.html) +- [Plugins](http://opentsdb.net/docs/build/html/development/plugins.html) +- [HTTP API](http://opentsdb.net/docs/build/html/development/http_api.html) + +Thank you for your contributions! diff --git a/Makefile.am b/Makefile.am index 9cbde8a8bb..f046853a18 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2012 The OpenTSDB Authors. +# Copyright (C) 2011-2021 The OpenTSDB Authors. # # This library is free software: you can redistribute it and/or modify it # under the terms of the GNU Lesser General Public License as published @@ -18,21 +18,28 @@ ACLOCAL_AMFLAGS = -I build-aux all-am: jar staticroot package = net.opentsdb +builddata_subpackage = tools spec_title = OpenTSDB spec_vendor = The OpenTSDB Authors jar := tsdb-$(PACKAGE_VERSION).jar plugin_test_jar := plugin_test.jar -builddata_SRC := src/BuildData.java +builddata_SRC := src/tools/BuildData.java BUILT_SOURCES = $(builddata_SRC) nodist_bin_SCRIPTS = tsdb dist_noinst_SCRIPTS = src/create_table.sh src/upgrade_1to2.sh src/mygnuplot.sh \ src/mygnuplot.bat src/opentsdb.conf tools/opentsdb_restart.py src/logback.xml dist_noinst_DATA = pom.xml.in build-aux/rpm/opentsdb.conf \ - build-aux/rpm/logback.xml build-aux/rpm/init.d/opentsdb + build-aux/rpm/logback.xml build-aux/rpm/init.d/opentsdb \ + build-aux/rpm/systemd/opentsdb.service \ + build-aux/rpm/systemd/opentsdb@.service tsdb_SRC := \ + src/core/AbstractSpanGroup.java \ + src/core/AbstractQuery.java \ src/core/AggregationIterator.java \ src/core/Aggregator.java \ src/core/Aggregators.java \ + src/core/AppendDataPoints.java \ + src/core/BatchedDataPoints.java \ src/core/ByteBufferList.java \ src/core/ColumnDatapointIterator.java \ src/core/CompactionQueue.java \ @@ -41,35 +48,132 @@ tsdb_SRC := \ src/core/DataPoints.java \ src/core/DataPointsIterator.java \ src/core/Downsampler.java \ + src/core/DownsamplingSpecification.java \ + src/core/FillingDownsampler.java \ + src/core/FillPolicy.java \ + src/core/GroupCallback.java \ + src/core/Histogram.java \ + src/core/HistogramAggregation.java \ + src/core/HistogramAggregationIterator.java \ + src/core/HistogramAggregator.java \ + src/core/HistogramBucketDataPointsAdaptor.java \ + src/core/HistogramCodecManager.java \ + src/core/HistogramDataPoint.java \ + src/core/HistogramDataPointCodec.java \ + src/core/HistogramDataPoints.java \ + src/core/HistogramDataPointsToDataPointsAdaptor.java \ + src/core/HistogramDownsampler.java \ + src/core/HistogramPojo.java \ + src/core/HistogramRowSeq.java \ + src/core/HistogramSeekableView.java \ + src/core/HistogramSpan.java \ + src/core/HistogramSpanGroup.java \ + src/core/iHistogramRowSeq.java \ src/core/IncomingDataPoint.java \ src/core/IncomingDataPoints.java \ src/core/IllegalDataException.java \ src/core/Internal.java \ + src/core/MultiGetQuery.java \ src/core/MutableDataPoint.java \ src/core/Query.java \ - src/core/RateOptions.java \ - src/core/RateSpan.java \ + src/core/QueryException.java \ + src/core/RateOptions.java \ + src/core/RateSpan.java \ + src/core/RequestBuilder.java \ src/core/RowKey.java \ src/core/RowSeq.java \ + src/core/RpcResponder.java \ + src/core/iRowSeq.java \ + src/core/SaltScanner.java \ src/core/SeekableView.java \ + src/core/SeekableViewChain.java \ + src/core/SimpleHistogram.java \ + src/core/SimpleHistogramDataPointAdapter.java \ + src/core/SimpleHistogramDecoder.java \ src/core/Span.java \ src/core/SpanGroup.java \ + src/core/SplitRollupQuery.java \ + src/core/SplitRollupSpanGroup.java \ src/core/TSDB.java \ src/core/Tags.java \ src/core/TsdbQuery.java \ src/core/TSQuery.java \ src/core/TSSubQuery.java \ src/core/WritableDataPoints.java \ + src/core/WriteableDataPointFilterPlugin.java \ src/graph/Plot.java \ + src/auth/AllowAllAuthenticatingAuthorizer.java \ + src/auth/AuthenticationChannelHandler.java \ + src/auth/Authentication.java \ + src/auth/Authorization.java \ + src/auth/AuthState.java \ + src/auth/Permissions.java \ + src/auth/Roles.java \ src/meta/Annotation.java \ + src/meta/MetaDataCache.java \ src/meta/TSMeta.java \ src/meta/TSUIDQuery.java \ src/meta/UIDMeta.java \ + src/normalize/NormalizePlugin.java \ + src/query/QueryUtil.java \ + src/query/QueryLimitOverride.java \ + src/query/expression/Absolute.java \ + src/query/expression/Alias.java \ + src/query/expression/DiffSeries.java \ + src/query/expression/DivideSeries.java \ + src/query/expression/EDPtoDPS.java \ + src/query/expression/Expression.java \ + src/query/expression/ExpressionDataPoint.java \ + src/query/expression/ExpressionFactory.java \ + src/query/expression/ExpressionIterator.java \ + src/query/expression/ExpressionReader.java \ + src/query/expression/Expressions.java \ + src/query/expression/ExpressionTree.java \ + src/query/expression/FirstDifference.java \ + src/query/expression/HighestCurrent.java \ + src/query/expression/HighestMax.java \ + src/query/expression/IntersectionIterator.java \ + src/query/expression/ITimeSyncedIterator.java \ + src/query/expression/NumericFillPolicy.java \ + src/query/expression/MovingAverage.java \ + src/query/expression/MultiplySeries.java \ + src/query/expression/PostAggregatedDataPoints.java \ + src/query/expression/Scale.java \ + src/query/expression/SumSeries.java \ + src/query/expression/TimeShift.java \ + src/query/expression/TimeSyncedIterator.java \ + src/query/expression/UnionIterator.java \ + src/query/expression/VariableIterator.java \ + src/query/filter/TagVFilter.java \ + src/query/filter/TagVLiteralOrFilter.java \ + src/query/filter/TagVNotKeyFilter.java \ + src/query/filter/TagVNotLiteralOrFilter.java \ + src/query/filter/TagVRegexFilter.java \ + src/query/filter/TagVWildcardFilter.java \ + src/query/pojo/Downsampler.java \ + src/query/pojo/Expression.java \ + src/query/pojo/Filter.java \ + src/query/pojo/Join.java \ + src/query/pojo/Metric.java \ + src/query/pojo/Output.java \ + src/query/pojo/Query.java \ + src/query/pojo/Timespan.java \ + src/query/pojo/Validatable.java \ + src/rollup/NoSuchRollupForIntervalException.java \ + src/rollup/NoSuchRollupForTableException.java \ + src/rollup/RollupConfig.java \ + src/rollup/RollUpDataPoint.java \ + src/rollup/RollupInterval.java \ + src/rollup/RollupQuery.java \ + src/rollup/RollupSeq.java \ + src/rollup/RollupSpan.java \ + src/rollup/RollupUtils.java \ src/search/SearchPlugin.java \ src/search/SearchQuery.java \ src/search/TimeSeriesLookup.java \ src/stats/Histogram.java \ src/stats/StatsCollector.java \ + src/stats/QueryStats.java \ src/tools/ArgP.java \ src/tools/CliOptions.java \ src/tools/CliQuery.java \ @@ -80,55 +184,78 @@ tsdb_SRC := \ src/tools/MetaPurge.java \ src/tools/MetaSync.java \ src/tools/Search.java \ + src/tools/StartupPlugin.java \ src/tools/TSDMain.java \ src/tools/TextImporter.java \ src/tools/TreeSync.java \ src/tools/UidManager.java \ + src/tools/ArgValueValidator.java \ + src/tools/ConfigArgP.java \ + src/tools/ConfigMetaType.java \ + src/tools/GnuplotInstaller.java \ + src/tools/OpenTSDBMain.java \ src/tree/Branch.java \ src/tree/Leaf.java \ src/tree/Tree.java \ src/tree/TreeBuilder.java \ src/tree/TreeRule.java \ + src/tsd/AbstractHttpQuery.java \ src/tsd/AnnotationRpc.java \ src/tsd/BadRequestException.java \ src/tsd/ConnectionManager.java \ + src/tsd/DropCachesRpc.java \ src/tsd/GnuplotException.java \ src/tsd/GraphHandler.java \ + src/tsd/HistogramDataPointRpc.java \ src/tsd/HttpJsonSerializer.java \ src/tsd/HttpSerializer.java \ src/tsd/HttpQuery.java \ src/tsd/HttpRpc.java \ + src/tsd/HttpRpcPlugin.java \ + src/tsd/HttpRpcPluginQuery.java \ src/tsd/LineBasedFrameDecoder.java \ src/tsd/LogsRpc.java \ src/tsd/PipelineFactory.java \ src/tsd/PutDataPointRpc.java \ + src/tsd/QueryExecutor.java \ src/tsd/QueryRpc.java \ + src/tsd/RollupDataPointRpc.java \ src/tsd/RpcHandler.java \ src/tsd/RpcPlugin.java \ + src/tsd/RpcManager.java \ + src/tsd/RpcUtil.java \ src/tsd/RTPublisher.java \ src/tsd/SearchRpc.java \ src/tsd/StaticFileRpc.java \ src/tsd/StatsRpc.java \ + src/tsd/StorageExceptionHandler.java \ src/tsd/SuggestRpc.java \ src/tsd/TelnetRpc.java \ src/tsd/TreeRpc.java \ src/tsd/UniqueIdRpc.java \ src/tsd/WordSplitter.java \ + src/uid/FailedToAssignUniqueIdException.java \ src/uid/NoSuchUniqueId.java \ src/uid/NoSuchUniqueName.java \ + src/uid/RandomUniqueId.java \ src/uid/UniqueId.java \ + src/uid/UniqueIdFilterPlugin.java \ src/uid/UniqueIdInterface.java \ src/utils/ByteArrayPair.java \ + src/utils/ByteSet.java \ src/utils/Config.java \ src/utils/DateTime.java \ + src/utils/Exceptions.java \ src/utils/FileSystem.java \ src/utils/JSON.java \ src/utils/JSONException.java \ src/utils/Pair.java \ - src/utils/PluginLoader.java + src/utils/PluginLoader.java \ + src/utils/Threads.java tsdb_DEPS = \ - $(ASYNCHBASE) \ + $(ASM) \ + $(COMMONS_LOGGING) \ $(GUAVA) \ $(LOG4J_OVER_SLF4J) \ $(LOGBACK_CLASSIC) \ @@ -136,41 +263,148 @@ tsdb_DEPS = \ $(JACKSON_ANNOTATIONS) \ $(JACKSON_CORE) \ $(JACKSON_DATABIND) \ + $(JAVACC) \ + $(JEXL) \ + $(JGRAPHT) \ + $(KRYO) \ + $(MINLOG) \ $(NETTY) \ - $(PROTOBUF) \ + $(REFLECTASM) \ $(SLF4J_API) \ $(SUASYNC) \ + $(APACHE_MATH) + +if BIGTABLE +tsdb_DEPS += \ + $(ASYNCBIGTABLE) +maven_profile_bigtable := true +maven_profile_hbase := false +maven_profile_cassandra := false +else +if CASSANDRA +tsdb_DEPS += \ + $(ASYNCCASSANDRA) +maven_profile_bigtable := false +maven_profile_hbase := false +maven_profile_cassandra := true +else +tsdb_DEPS += \ + $(ASYNCHBASE) \ + $(PROTOBUF) \ $(ZOOKEEPER) +maven_profile_bigtable := false +maven_profile_hbase := true +maven_profile_cassandra := false +endif +endif test_SRC := \ test/core/SeekableViewsForTest.java \ + test/core/BaseTsdbTest.java \ + test/core/HistogramSeekableViewForTest.java \ + test/core/LongHistogramDataPointForTest.java \ + test/core/LongHistogramDataPointForTestDecoder.java \ test/core/TestAggregationIterator.java \ test/core/TestAggregators.java \ + test/core/TestAppendDataPoints.java \ + test/core/TestBatchedDataPoints.java \ test/core/TestCompactionQueue.java \ test/core/TestDownsampler.java \ + test/core/TestDownsamplingSpecification.java \ + test/core/TestFillingDownsampler.java \ + test/core/TestHistogramAggregationIterator.java \ + test/core/TestHistogramCodecManager.java \ + test/core/TestHistogramDataPointsToDataPointsAdaptor.java \ + test/core/TestHistogramDownsampler.java \ + test/core/TestHistogramPojo.java \ + test/core/TestHistogramRowSeq.java \ + test/core/TestHistogramSpan.java \ + test/core/TestHistogramSpanGroup.java \ + test/core/TestIncomingDataPoints.java \ test/core/TestInternal.java \ test/core/TestMutableDataPoint.java \ test/core/TestRateSpan.java \ + test/core/TestRowKey.java \ test/core/TestRowSeq.java \ + test/core/TestRpcResponsder.java \ + test/core/TestSaltScanner.java \ + test/core/TestSeekableViewChain.java \ test/core/TestSpan.java \ + test/core/TestSpanGroup.java \ + test/core/TestSplitRollupQuery.java \ + test/core/TestSplitRollupSpanGroup.java \ test/core/TestTags.java \ test/core/TestTSDB.java \ + test/core/TestTSDBAddPoint.java \ + test/core/TestTSDBTableAvailability.java \ test/core/TestTsdbQueryDownsample.java \ + test/core/TestTsdbQueryDownsampleSalted.java \ test/core/TestTsdbQuery.java \ + test/core/TestTsdbQueryAggregators.java \ + test/core/TestTsdbQueryAggregatorsSalted.java \ + test/core/TestTsdbQueryAppend.java \ + test/core/TestTsdbQueryQueries.java \ + test/core/TestTsdbQuerySalted.java \ + test/core/TestTsdbQuerySaltedAppend.java \ test/core/TestTSQuery.java \ test/core/TestTSSubQuery.java \ + test/core/TestTsdbTSConfig.java \ test/plugin/DummyPlugin.java \ test/meta/TestAnnotation.java \ test/meta/TestTSMeta.java \ test/meta/TestTSUIDQuery.java \ test/meta/TestUIDMeta.java \ + test/query/expression/BaseTimeSyncedIteratorTest.java \ + test/query/expression/TestAbsolute.java \ + test/query/expression/TestAlias.java \ + test/query/expression/TestDiffSeries.java \ + test/query/expression/TestDivideSeries.java \ + test/query/expression/TestExpressionFactory.java \ + test/query/expression/TestExpressionIterator.java \ + test/query/expression/TestExpressionReader.java \ + test/query/expression/TestExpressions.java \ + test/query/expression/TestExpressionTree.java \ + test/query/expression/TestHighestCurrent.java \ + test/query/expression/TestHighestMax.java \ + test/query/expression/TestIntersectionIterator.java \ + test/query/expression/TestNumericFillPolicy.java \ + test/query/expression/TestMovingAverage.java \ + test/query/expression/TestMultiplySeries.java \ + test/query/expression/TestPostAggregatedDataPoints.java \ + test/query/expression/TestScale.java \ + test/query/expression/TestSumSeries.java \ + test/query/expression/TestTimeSyncedIterator.java \ + test/query/expression/TestUnionIterator.java \ + test/query/filter/TestTagVFilter.java \ + test/query/filter/TestTagVLiteralOrFilter.java \ + test/query/filter/TestTagVNotKeyFilter.java \ + test/query/filter/TestTagVNotLiteralOrFilter.java \ + test/query/filter/TestTagVRegexFilter.java \ + test/query/filter/TestTagVWildcardFilter.java \ + test/query/pojo/TestDownsampler.java \ + test/query/pojo/TestExpression.java \ + test/query/pojo/TestFilter.java \ + test/query/pojo/TestJoin.java \ + test/query/pojo/TestMetric.java \ + test/query/pojo/TestOutput.java \ + test/query/pojo/TestQuery.java \ + test/query/pojo/TestTimeSpan.java \ + test/rollup/TestRollupConfig.java \ + test/rollup/TestRollupInterval.java \ + test/rollup/TestRollupQuery.java \ + test/rollup/TestRollupSeq.java \ + test/rollup/TestRollupUtils.java \ test/search/TestSearchPlugin.java \ test/search/TestSearchQuery.java \ test/search/TestTimeSeriesLookup.java \ test/stats/TestHistogram.java \ + test/stats/TestQueryStats.java \ test/storage/MockBase.java \ + test/storage/MockDataPoints.java \ test/tools/TestDumpSeries.java \ + test/tools/TestCliUtils.java \ test/tools/TestFsck.java \ + test/tools/TestFsckSalted.java \ test/tools/TestTextImporter.java \ test/tools/TestUID.java \ test/tree/TestBranch.java \ @@ -178,25 +412,36 @@ test_SRC := \ test/tree/TestTree.java \ test/tree/TestTreeBuilder.java \ test/tree/TestTreeRule.java \ + test/tsd/BaseTestPutRpc.java \ test/tsd/NettyMocks.java \ test/tsd/TestAnnotationRpc.java \ test/tsd/TestGraphHandler.java \ test/tsd/TestHttpJsonSerializer.java \ test/tsd/TestHttpQuery.java \ + test/tsd/TestHttpRpcPluginQuery.java \ test/tsd/TestPutRpc.java \ + test/tsd/TestQueryExecutor.java \ test/tsd/TestQueryRpc.java \ + test/tsd/TestQueryRpcLastDataPoint.java \ + test/tsd/TestRollupRpc.java \ test/tsd/TestRpcHandler.java \ test/tsd/TestRpcPlugin.java \ + test/tsd/TestRpcManager.java \ test/tsd/TestRTPublisher.java \ test/tsd/TestSearchRpc.java \ + test/tsd/TestStatsRpc.java \ + test/tsd/TestStatusRpc.java \ test/tsd/TestSuggestRpc.java \ test/tsd/TestTreeRpc.java \ test/tsd/TestUniqueIdRpc.java \ test/uid/TestNoSuchUniqueId.java \ + test/uid/TestRandomUniqueId.java \ test/uid/TestUniqueId.java \ test/utils/TestByteArrayPair.java \ + test/utils/TestByteSet.java \ test/utils/TestConfig.java \ test/utils/TestDateTime.java \ + test/utils/TestExceptions.java \ test/utils/TestJSON.java \ test/utils/TestPair.java \ test/utils/TestPluginLoader.java @@ -206,16 +451,20 @@ test_plugin_SRC := \ test/plugin/DummyPluginB.java \ test/search/DummySearchPlugin.java \ test/tsd/DummyHttpSerializer.java \ + test/tsd/DummyHttpRpcPlugin.java \ test/tsd/DummyRpcPlugin.java \ - test/tsd/DummyRTPublisher.java + test/tsd/DummyRTPublisher.java \ + test/tsd/DummySEHPlugin.java # Do NOT include the test dir path, just the META portion test_plugin_SVCS := \ META-INF/services/net.opentsdb.plugin.DummyPlugin \ META-INF/services/net.opentsdb.search.SearchPlugin \ META-INF/services/net.opentsdb.tsd.HttpSerializer \ + META-INF/services/net.opentsdb.tsd.HttpRpcPlugin \ META-INF/services/net.opentsdb.tsd.RpcPlugin \ - META-INF/services/net.opentsdb.tsd.RTPublisher + META-INF/services/net.opentsdb.tsd.RTPublisher \ + META-INF/services/net.opentsdb.tsd.StorageExceptionHandler test_plugin_MF := \ test/META-INF/MANIFEST.MF @@ -224,11 +473,11 @@ test_DEPS = \ $(tsdb_DEPS) \ $(JAVASSIST) \ $(JUNIT) \ - $(HAMCREST) \ + $(HAMCREST) \ $(MOCKITO) \ - $(OBJENESIS) \ + $(OBJENESIS) \ $(POWERMOCK_MOCKITO) \ - $(jar) + $(jar) httpui_SRC := \ src/tsd/client/DateTimeBox.java \ @@ -242,10 +491,18 @@ httpui_SRC := \ httpui_DEPS = src/tsd/QueryUi.gwt.xml +# TODO(CL) - There is likely a MUCH better way to compile and add the expression sources and jars. +expr_grammar = $(srcdir)/src/parser.jj +expr_package = net/opentsdb/query/expression/parser +expr_src_dir = $(builddir)/src/$(expr_package) +get_expr_classes = `classes=''; for f in $(packagedir)$(expr_package)/*.class; do classes="$$classes $$f"; done; echo $$classes;` + #dist_pkgdata_DATA = src/logback.xml -dist_static_DATA = src/tsd/static/favicon.ico +dist_static_DATA = \ + src/tsd/static/favicon.ico \ + src/tsd/static/opentsdb_header.jpg -EXTRA_DIST = tsdb.in $(tsdb_SRC) $(test_SRC) \ +EXTRA_DIST = tsdb.in $(tsdb_SRC) $(test_SRC) $(expr_grammar) \ $(test_plugin_SRC) $(test_plugin_MF) $(test_plugin_SVCS:%=test/%) \ $(THIRD_PARTY) $(THIRD_PARTY:=.md5) \ $(httpui_SRC) $(httpui_DEPS) \ @@ -292,13 +549,13 @@ printdeps: # This is kind of a hack, but I couldn't find a better way to adjust the paths # in the script before it gets installed... install-exec-hook: - script=tsdb; pkgdatadir='$(pkgdatadir)'; configdir='$(pkgdatadir)/etc/opentsdb'; \ + script=tsdb; pkgdatadir='$(pkgdatadir)'; configdir='$(sysconfigdir)/etc/opentsdb'; \ abs_srcdir=''; abs_builddir=''; $(edit_tsdb_script) cat tsdb.tmp >"$(DESTDIR)$(bindir)/tsdb" rm -f tsdb.tmp $(builddata_SRC): .git/HEAD $(tsdb_SRC) $(top_srcdir)/build-aux/gen_build_data.sh - $(srcdir)/build-aux/gen_build_data.sh $(builddata_SRC) $(package) $(PACKAGE_VERSION) + $(srcdir)/build-aux/gen_build_data.sh $(builddata_SRC) $(package).$(builddata_subpackage) $(PACKAGE_VERSION) jar: $(jar) .javac-unittests-stamp .gwtc-stamp @@ -315,6 +572,9 @@ filter_src = \ src="$$src $$i";; \ esac; \ done; \ + for f in $(expr_src_dir)/*.java; do \ + src="$$src $$f"; \ + done; \ test -n "$$src" || exit 0 # Touches all the targets if any of the dependencies are newer. # This is useful to force-recompile all files if one of the @@ -331,25 +591,25 @@ $(tsdb_SRC): $(tsdb_DEPS) find_jar = test -f "$$jar" && echo "$$jar" || echo "$(srcdir)/$$jar" get_dep_classpath = `for jar in $(tsdb_DEPS); do $(find_jar); done | tr '\n' ':'` -.javac-stamp: $(tsdb_SRC) $(builddata_SRC) +.javac-stamp: $(tsdb_SRC) $(builddata_SRC) runjavacc @$(filter_src); cp=$(get_dep_classpath); \ echo "$(JAVA_COMPILE) -cp $$cp $$src"; \ $(JAVA_COMPILE) -cp $$cp $$src @touch "$@" VALIDATION_API_CLASSPATH = `jar=$(VALIDATION_API); $(find_jar)`:`jar=$(VALIDATION_API_SOURCES); $(find_jar)` -GWT_CLASSPATH = $(VALIDATION_API_CLASSPATH):`jar=$(GWT_DEV); $(find_jar)`:`jar=$(GWT_USER); $(find_jar)`:$(srcdir)/src +GWT_CLASSPATH = $(VALIDATION_API_CLASSPATH):`jar=$(GWT_DEV); $(find_jar)`:`jar=$(GWT_USER); $(find_jar)`:`jar=$(GWT_THEME); $(find_jar)`:$(srcdir)/src # The GWT compiler is way too slow, that's not very Googley. So we save the # MD5 of the files we compile in the stamp file and everytime `make' things it # needs to recompile the GWT code, we verify whether the code really changed # or whether it's just a file that was touched (which happens frequently when # using Git while rebasing and whatnot). gwtc: .gwtc-stamp -.gwtc-stamp: $(httpui_SRC) $(httpui_DEPS) $(VALIDATION_API) $(VALIDATION_API_SOURCES) $(GWT_DEV) $(GWT_USER) +.gwtc-stamp: $(httpui_SRC) $(httpui_DEPS) $(VALIDATION_API) $(VALIDATION_API_SOURCES) $(GWT_DEV) $(GWT_USER) $(GWT_THEME) @$(mkdir_p) gwt { cd $(srcdir) && cat $(httpui_SRC); } | $(MD5) >"$@-t" cmp -s "$@" "$@-t" && exit 0; \ - $(JAVA) $(GWTC_JVM_ARGS) -cp $(GWT_CLASSPATH) com.google.gwt.dev.Compiler \ + $(JAVA) -Djava.util.prefs.userRoot=$(HOME) $(GWTC_JVM_ARGS) -cp $(GWT_CLASSPATH) com.google.gwt.dev.Compiler \ $(GWTC_ARGS) -war gwt tsd.QueryUi @mv "$@-t" "$@" @@ -375,7 +635,7 @@ gwttsd: staticroot $(mkdir_p) $(DEV_TSD_STATICROOT) cp $(dist_static_DATA:%=$(srcdir)/%) $(DEV_TSD_STATICROOT) find -L $(DEV_TSD_STATICROOT) -type l -exec rm {} \; - p=`pwd`/gwt/queryui && cd $(DEV_TSD_STATICROOT) \ + p=../gwt/queryui && cd $(DEV_TSD_STATICROOT) \ && for i in $$p/*; do ln -s -f "$$i" || break; done find -L $(DEV_TSD_STATICROOT)/gwt -type f | xargs touch @touch .staticroot-stamp @@ -427,11 +687,13 @@ install-data-tools: $(tsdb_DEPS) $(jar) destdatatoolsdir="$(DESTDIR)$(pkgdatadir)/tools" ; \ echo " $(mkdir_p) $$destdatatoolsdir"; \ $(mkdir_p) "$$destdatatoolsdir" || exit 1; \ - tools="$$tools $(top_srcdir)/tools/*" ; \ tools="$$tools $(top_srcdir)/src/create_table.sh" ; \ tools="$$tools $(top_srcdir)/src/upgrade_1to2.sh" ; \ echo " $(INSTALL_SCRIPT)" $$tools "$$destdatatoolsdir" ; \ - $(INSTALL_SCRIPT) $$tools "$$destdatatoolsdir" || exit 1; + $(INSTALL_SCRIPT) $$tools "$$destdatatoolsdir" || exit 1; \ + tools="-r $(top_srcdir)/tools/*" ; \ + echo " cp" $$tools "$$destdatatoolsdir" ; \ + cp $$tools "$$destdatatoolsdir" || exit 1; uninstall-data-tools: @$(NORMAL_UNINSTALL) @@ -465,17 +727,24 @@ install-data-etc: destdataetcdir="$(DESTDIR)$(pkgdatadir)/etc" ; \ destdataconfdir="$$destdataetcdir/opentsdb" ; \ destdatainitdir="$$destdataetcdir/init.d" ; \ + destdatasystemddir="$$destdataetcdir/systemd/system" ; \ echo " $(mkdir_p) $$destdataconfdir"; \ $(mkdir_p) "$$destdataconfdir" || exit 1; \ echo " $(mkdir_p) $$destdatainitdir"; \ $(mkdir_p) "$$destdatainitdir" || exit 1; \ + echo " $(mkdir_p) $$destdatasystemddir"; \ + $(mkdir_p) "$$destdatasystemddir" || exit 1; \ conf_files="$$conf_files $(top_srcdir)/build-aux/rpm/opentsdb.conf" ; \ conf_files="$$conf_files $(top_srcdir)/build-aux/rpm/logback.xml" ; \ echo " $(INSTALL_SCRIPT)" $$conf_files "$$destdataconfdir" ; \ $(INSTALL_DATA) $$conf_files "$$destdataconfdir" || exit 1; \ init_file="$(top_srcdir)/build-aux/rpm/init.d/opentsdb" ; \ echo " $(INSTALL_SCRIPT)" $$init_file "$$destdatainitdir" ; \ - $(INSTALL_SCRIPT) $$init_file "$$destdatainitdir" || exit 1; + $(INSTALL_SCRIPT) $$init_file "$$destdatainitdir" || exit 1; \ + systemd_file="$(top_srcdir)/build-aux/rpm/systemd/opentsdb.service" ; \ + systemd_file="$(top_srcdir)/build-aux/rpm/systemd/opentsdb@.service" ; \ + echo " $(INSTALL_SCRIPT)" $$systemd_file "$$destdatasystemddir" ; \ + $(INSTALL_SCRIPT) $$systemd_file "$$destdatasystemddir" || exit 1; uninstall-data-etc: @$(NORMAL_UNINSTALL) @@ -547,8 +816,8 @@ manifest: .javac-stamp .git/HEAD echo "Implementation-Version: $(git_version)"; \ echo "Implementation-Vendor: $(spec_vendor)"; } >"$@" -$(jar): manifest .javac-stamp $(classes) - $(JAR) cfm `basename $(jar)` manifest $(classes_with_nested_classes) \ +$(jar): manifest .javac-stamp + $(JAR) cfm `basename $(jar)` manifest $(classes_with_nested_classes) $(get_expr_classes) \ || { rv=$$? && rm -f `basename $(jar)` && exit $$rv; } # ^^^^^^^^^^^^^^^^^^^^^^^ # I've seen cases where `jar' exits with an error but leaves a partially built .jar file! @@ -572,6 +841,9 @@ $(JAVADOC_DIR)/index.html: $(tsdb_SRC) -link $(JDK_JAVADOC) -link $(NETTY_JAVADOC) -link $(SUASYNC_JAVADOC) \ $? $(builddata_SRC) +runjavacc: + $(JAVA) -cp $(JAVACC) javacc -OUTPUT_DIRECTORY:$(expr_src_dir) $(expr_grammar); echo PWD: `pwd`; + dist-hook: $(mkdir_p) $(distdir)/.git echo $(git_version) >$(distdir)/.git/HEAD @@ -585,7 +857,7 @@ mostlyclean-local: && find $(package_dir) -depth -type d -exec rmdir {} ';' \ && dir=$(package_dir) && dir=$${dir%/*} \ && while test x"$$dir" != x"$${dir%/*}"; do \ - rmdir "$$dir" && dir=$${dir%/*} || break; \ + rm -rf "$$dir" && dir=$${dir%/*} || break; \ done \ && rmdir "$$dir" @@ -602,12 +874,56 @@ pom.xml: pom.xml.in Makefile echo ''; \ sed <$< \ -e 's/@ASYNCHBASE_VERSION@/$(ASYNCHBASE_VERSION)/' \ + -e 's/@ASYNCBIGTABLE_VERSION@/$(ASYNCBIGTABLE_VERSION)/' \ + -e 's/@ASYNCCASSANDRA_VERSION@/$(ASYNCCASSANDRA_VERSION)/' \ + -e 's/@GUAVA_VERSION@/$(GUAVA_VERSION)/' \ + -e 's/@GWT_VERSION@/$(GWT_VERSION)/' \ + -e 's/@GWT_THEME_VERSION@/$(GWT_THEME_VERSION)/' \ + -e 's/@HAMCREST_VERSION@/$(HAMCREST_VERSION)/' \ + -e 's/@JACKSON_VERSION@/$(JACKSON_VERSION)/' \ + -e 's/@JAVASSIST_VERSION@/$(JAVASSIST_VERSION)/' \ + -e 's/@JUNIT_VERSION@/$(JUNIT_VERSION)/' \ + -e 's/@LOG4J_OVER_SLF4J_VERSION@/$(LOG4J_OVER_SLF4J_VERSION)/' \ + -e 's/@LOGBACK_CLASSIC_VERSION@/$(LOGBACK_CLASSIC_VERSION)/' \ + -e 's/@LOGBACK_CORE_VERSION@/$(LOGBACK_CORE_VERSION)/' \ + -e 's/@MOCKITO_VERSION@/$(MOCKITO_VERSION)/' \ + -e 's/@NETTY_VERSION@/$(NETTY_VERSION)/' \ + -e 's/@OBJENESIS_VERSION@/$(OBJENESIS_VERSION)/' \ + -e 's/@POWERMOCK_MOCKITO_VERSION@/$(POWERMOCK_MOCKITO_VERSION)/' \ + -e 's/@SLF4J_API_VERSION@/$(SLF4J_API_VERSION)/' \ + -e 's/@SUASYNC_VERSION@/$(SUASYNC_VERSION)/' \ + -e 's/@ZOOKEEPER_VERSION@/$(ZOOKEEPER_VERSION)/' \ + -e 's/@APACHE_MATH_VERSION@/$(APACHE_MATH_VERSION)/' \ + -e 's/@JEXL_VERSION@/$(JEXL_VERSION)/' \ + -e 's/@JGRAPHT_VERSION@/$(JGRAPHT_VERSION)/' \ + -e 's/@spec_title@/$(spec_title)/' \ + -e 's/@spec_vendor@/$(spec_vendor)/' \ + -e 's/@spec_version@/$(PACKAGE_VERSION)/' \ + -e 's/@maven_profile_hbase@/$(maven_profile_hbase)/' \ + -e 's/@maven_profile_bigtable@/$(maven_profile_bigtable)/' \ + -e 's/@maven_profile_cassandrae@/$(maven_profile_cassandra)/' \ + ; \ + } >$@-t + mv $@-t ../$@ + +# Generates a maven pom called fat-jar-pom.xml that builds a fat jar +# containing all the dependencies required to run opentsdb +fat-jar-pom.xml: ./fat-jar/fat-jar-pom.xml.in Makefile + (cd $(top_srcdir) ; ./fat-jar/create-src-dir-overlay.sh) + { \ + echo ''; \ + sed <$< \ + -e 's/@ASYNCHBASE_VERSION@/$(ASYNCHBASE_VERSION)/' \ + -e 's/@ASYNCBIGTABLE_VERSION@/$(ASYNCBIGTABLE_VERSION)/' \ + -e 's/@ASYNCCASSANDRA_VERSION@/$(ASYNCCASSANDRA_VERSION)/' \ -e 's/@GUAVA_VERSION@/$(GUAVA_VERSION)/' \ -e 's/@GWT_VERSION@/$(GWT_VERSION)/' \ + -e 's/@GWT_THEME_VERSION@/$(GWT_THEME_VERSION)/' \ -e 's/@HAMCREST_VERSION@/$(HAMCREST_VERSION)/' \ -e 's/@JACKSON_VERSION@/$(JACKSON_VERSION)/' \ -e 's/@JAVASSIST_VERSION@/$(JAVASSIST_VERSION)/' \ -e 's/@JUNIT_VERSION@/$(JUNIT_VERSION)/' \ + -e 's/@KRYO_VERSION@/$(KRYO_VERSION)/' \ -e 's/@LOG4J_OVER_SLF4J_VERSION@/$(LOG4J_OVER_SLF4J_VERSION)/' \ -e 's/@LOGBACK_CLASSIC_VERSION@/$(LOGBACK_CLASSIC_VERSION)/' \ -e 's/@LOGBACK_CORE_VERSION@/$(LOGBACK_CORE_VERSION)/' \ @@ -618,6 +934,9 @@ pom.xml: pom.xml.in Makefile -e 's/@SLF4J_API_VERSION@/$(SLF4J_API_VERSION)/' \ -e 's/@SUASYNC_VERSION@/$(SUASYNC_VERSION)/' \ -e 's/@ZOOKEEPER_VERSION@/$(ZOOKEEPER_VERSION)/' \ + -e 's/@APACHE_MATH_VERSION@/$(APACHE_MATH_VERSION)/' \ + -e 's/@JEXL_VERSION@/$(JEXL_VERSION)/' \ + -e 's/@JGRAPHT_VERSION@/$(JGRAPHT_VERSION)/' \ -e 's/@spec_title@/$(spec_title)/' \ -e 's/@spec_vendor@/$(spec_vendor)/' \ -e 's/@spec_version@/$(PACKAGE_VERSION)/' \ @@ -628,8 +947,8 @@ pom.xml: pom.xml.in Makefile TIMESTAMP := $(shell date +"%Y%m%d%H%M%S") RPM_REVISION := 1 RPM_TARGET := noarch -RPM := opentsdb-$(PACKAGE_VERSION)-$(RPM_REVISION).$(RPM_TARGET).rpm -RPM_SNAPSHOT := opentsdb-$(PACKAGE_VERSION)-$(RPM_REVISION)-$(TIMESTAMP)-"`whoami`".$(RPM_TARGET).rpm +RPM := opentsdb-$(subst -,_,$(PACKAGE_VERSION))-$(RPM_REVISION).$(RPM_TARGET).rpm +RPM_SNAPSHOT := opentsdb-$(subst -,_,$(PACKAGE_VERSION))-$(RPM_REVISION)-$(TIMESTAMP)-"`whoami`".$(RPM_TARGET).rpm SOURCE_TARBALL := opentsdb-$(PACKAGE_VERSION).tar.gz rpm: $(RPM) @@ -671,10 +990,11 @@ debian: dist staticroot cp $(top_srcdir)/build-aux/deb/init.d/opentsdb $(distdir)/debian/etc/init.d cp $(jar) $(distdir)/debian/usr/share/opentsdb/lib cp -r staticroot/favicon.ico $(distdir)/debian/usr/share/opentsdb/static + cp -r staticroot/opentsdb_header.jpg $(distdir)/debian/usr/share/opentsdb/static cp -r gwt/queryui/* $(distdir)/debian/usr/share/opentsdb/static `for dep_jar in $(tsdb_DEPS); do cp $$dep_jar \ $(distdir)/debian/usr/share/opentsdb/lib; done;` - cp $(top_srcdir)/tools/* $(distdir)/debian/usr/share/opentsdb/tools + cp -r $(top_srcdir)/tools/* $(distdir)/debian/usr/share/opentsdb/tools dpkg -b $(distdir)/debian $(distdir)/opentsdb-$(PACKAGE_VERSION)_all.deb .PHONY: jar doc check gwtc gwtdev printdeps staticroot gwttsd rpm diff --git a/NEWS b/NEWS index b45f3dccef..c75ffd3ebf 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,400 @@ -OpenTSDB - User visible changes. +OpenTSDB - Changelog +* Version 2.5.0RC1 (2024-12-12) + +Noteworthy Changes: + - Bump Jackson to 2.14.1 WARNING: The minimum JDK is now version 8 due to Jackson. (#2263) + - Enhance range tests and regex and add better default for output validator (#2217) + - Converting to using SystemD systemctl (#1515) + - Status API, Start of a new HTTP API (#1742) + - Allow end_time = start_time to be able to query and delete a single datapoint. (#2036) + - Add support for splitting rollup queries (#1853) + - ExplicitTags filtering with FuzzyFilters (#1896) + - Add "check_tsd_v2" script (#1567) + - Do PutDataPointRpc.GroupCB asynchronously (#1472) + - Tag Normalization Plugin Support (#1525) + +Bug Fixes: + - Fix race condition in UID lookup (#2176) + - Tighten up the regexes for Gnuplot URI params per multiple security reports. (#2272) + - Trigger callback chain when reached limit. (#2204, #839) + - Use ConcurrentHashMap when serializing tags. (#2282, #1055) + - Test for null, avoid NPE (#2281, #2280) + - Ensure we always add the {} for group_by filters, otherwise the non group_by filters act as group_by filters (#1697) + - Added tracking of metrics which are null due to auto_metric being disabled Fixes (#786,#2042) + - Fixed function description Fixes (#841,#2040) + - Fix concurrent result reporting from scanners (#1753) + - Fix SaltScanner race condition on spans maps (#1651) + - Fix an edge case in TsdbQuery.getScanEndTimeSeconds() (#1581, #1582) + - Fix a compilation error about missing FirstDifference (#1471) + +Security Fixes: + - Bump Logback version to patch CVE-2021-42550 (#2208, #2218) + - Make sure the inputs are only plain ASCII printables first. Fixes CVE-2018-12972, CVE-2020-35476 (#2275) + +* Version 2.4.1 (2021-09-02) + +Noteworthy Changes: + - Add a config flag to enable splitting queries across the rollup and + raw data tables. Another config determines when to split the queries. + - Fix for CVE-2020-35476 that now validates and limits the inputs for + Gnuplot query parameters to prevent remote code execution. + - Default log config will log CLI tools at INFO to standard out. + - New check_tsd_v2 script that evaluates individual metric groups given + a filter. + - Collect stats from meta cache plugins. + - Add a python script to list and pretty-print queries running on a TSD. + - Add a single, standalone TSD systemd startup script and default to that + instead of the multi-port TSD script. + +Bug Fixes: + - Fix the "--sync" flag for FSCK to wait for repairs to execute against + storage before proceeding. + - Fix expression queries to allow metric only and filterless queries. + - Fix an NPE in /api/query/last when appends are enabled. + - Fix races in the salt scanner and multigets on the storage maps. + - Fix rollup queries that need sum and count for downsampling instead of + group bys. + - Fix fuzzy row filters for later versions of HBase by using filter pairs. + And allow it to be combined with a regex filter. + - Fix stats from the individual salt scanners and overall query stats + concurrency issues. + - Rename the stat "maxScannerUidtoStringTime" to + "maxScannerUidToStringTime" + - Fix the min initial value for double values in the AggregationIterator + - Fix rollup queries producing unexpected results. + - Fix various UTs + - Support rollups in FSCK + +* Version 2.4.0 (2018-12-16) + +Noteworthy Changes: + - Set default data block encoding to `DIFF` in the create table script. + - Add callbacks to log errors in the FSCK tool when a call was made to + fix something. + - Add a sum of squares aggregator "squareSum". + - Add the diff aggregator that computes the difference between the first + and last values. + - Add a SystemD template to the RPM package. + - Allow tags to be added via HTTP header. + - Add example implementations for the Authorization and Authentication + plugins. + - Change `tsd.storage.use_otsdb_timestamp` to default to false. + - Literal or filter now allows single character values. + - Rollup query code now only uses the downsampler value to pick an interval. + - Add jdk 8 in the debian script. + - Setup fill policies in the Nagios check + +Bug Fixes: + - Fix rollup scanner filter for single aggregate queries. + - Fix FSCK HBase timestamps when deduping. Sometimes they were negative. + - Fix exception handling when writing data over HTTP with the sync flag enabled. + - Fix missing source files in the Makefile. + - Change UID cache to longs from ints and add hit and miss counters. + - Fix HighestCurrent returning the wrong results. + - Fix running query stats queryStart timestamp to millis. + - Fix TimeShift millisecond bug. + - Fix post remove step in the debian package. + +* Version 2.3.2 (2018-12-16) + +Noteworthy Changes: + - A new Python wrapper script to make FSCK repair runs easier. + - Track performance in the Nagios/Icinga script + - Add a Contributions file. + - Add a config, 'tsd.core.bulk.allow_out_of_order_timestamps' to allow out of + order timestamps for bulk ingest. + - NOTE: This version also includes a JDK 8 compiled version of Jackson due to + security patches. If you need to run with an older JDK please replace the + Jackson JARs with older versions. + +Bug Fixes: + - Unwrap NoSuchUniqueIds when writing data points to make it easier to understand + exceptions. + - Fix an NPE in the PutDataPointRpc class if a data point in the list is null. + - Fix a Makefile error in the clean portion. + - Fix an NPOE in the UIDManager print result. + - Fix a bug in the UI where Y formats may contain a percent sign. + - Allow specifying the data block encoding and TTL in the HBase table creation + script. + - Change the make and TSDB scripts to use relative paths. + - Fix parsing of `use_meta` from the URI for the search endpoint. + - Fix the clean cache script to be a bit more OS agnostic. + + +* Version 2.4.0 RC2 (2017-10-08) + +Noteworthy Changes: + - Modify the RPC handler plugin system so that it parses only the first part of + the URI instead of the entire path. Now plugins can implement sub-paths. + - Return the HTML 5 doctype for built-in UI pages + - Add an optional byte and/or data point limit to the amount of data fetched + from storage. This allows admins to prevent OOMing TSDs due to massive queries. + - Allow a start time via config when enabling the date tiered compaction in HBase + - Provide the option of using an LRU for caching UIDs to avoid OOMing writers and + readers with too many strings + - Optionally avoid writing to the forward or reverse UID maps when a specific TSD + operational mode is enabled to avoid wasting memory on maps that will never be + used. + +* Version 2.3.1 (2018-04-21) + +Noteworthy Changes: + - When setting up aggregators, advance to the first data point equal to or greater + than the query start timestamp. This helps with calendar downsampling intervals. + - Add support to the Nagios check script for downsampling fill policies. + +Bug Fixes: + - Fix expression calculation by avoiding double execution and checking both + output types for boolean values. + - Fixing missing tools scripts in builds. + - Default HBase 1.2.5 in the OSX install script + - Upgrade AsyncBigtable to 0.3.1 + - Log query stats when a channel is closed unexpectedly. + - Add the Java 8 path in the debian init script and remove Java 6. + - Pass the column family name to the get requests in the compaction scheduler. + - Fix a comparison issue in the UI on group by tags. + - Filter annotation queries by the starting timestamp, excluding those in a row that + began before the query start time. + - Tiny stap at purging backticks from Gnuplot scripts. + - Remove the `final` annotation from the meta classes so they can be extended. + - Fix the javacc maven plugin version. + - Fix the literal or filter to allow single character filters. + - Fix query start stats logging to use the ms instead of nano time. + - Move Jackson and Netty to newer versions for security reasons. + - Upgrade to AsyncHBase 1.8.2 for compatibility with HBase 1.3 and 2.0 + - Fix the Highest Current calculation to handle empty time series. + - Change the cache hits counters to longs. + +* Version 2.3.0 (2016-12-31) + +Noteworthy Changes: + - Release of 2.3.0. + - Add example classes for using the Java API. + +Bug Fixes: + - Same fixes as in 2.2.2 + - Fix a null UID check on decoding metric names from row keys. + - Fix unit tests for JDK 8 and later. + +* Version 2.2.2 (2016-12-29) + +Bug Fixes: + - Fix an issue with writing metadata where using custom tags could cause the compare- + and-set to fail due to variable ordering in Java's heap. Now tags are sorted so the + custom tag ordering will be consistent. + - Fix millisecond queries that would miss data the top of the final hour if the end + time was set to 1 second or less than the top of that final hour. + - Fix a concurrent modification issue where salt scanners were not synchronized on the + annotation map and could cause spinning threads. + +* Version 2.3.0 RC2 (2016-10-08) + +Noteworthy Changes: + - Added a docker file and tool to build TSD docker containers (#871). + - Log X-Forwarded-For addresses when handling HTTP requests. + - Expand aggregator options in the Nagios check script. + - Allow enabling or disabling the HTTP API or UI. + - TSD will now exit when an unrecognized CLI param is passed. + +Bug Fixes: + - Improved ALPN version detection when using Google Bigtable. + - Fix the DumpSeries class to support appended data point types. + - Fix queries where groupby is set to false on all filters. + - Fix a missing attribute in the Nagios check script (#728). + - Fix a major security bug where requesting a PNG with certain URI params could execute code + on the host (#793, #781). + - Return a proper error code when dropping caches with the DELETE HTTP verb (#830). + - Fix backwards compatibility with HBase 0.94 when using explicit tags by removing the + fuzzy filter (#837). + - Fix an RPM build issue when creating the GWT directory.s + +* Version 2.2.1 (2016-10-08) + +Noteworthy Changes + - Generate an incrementing TSMeta request only if both enable_tsuid_incrementing and + tsd.core.meta.enable_realtime_ts are enabled. Previously, increments would run + regardless of whether or not the real time ts setting was enabled. If tsuid + incrementing is disabled then a get and optional put is executed each time without + modifying the meta counter field. + - Improve metadata storage performance by removing an extra getFromStorage() call. + - Add global Annotations to the gnuplot graphs (#773) + - Allow creation of a TSMeta object without a TSUID (#778) + - Move to AsyncHBase 1.7.2 + +Bug Fixes: + - Fix Python scripts to use the environment directory. + - Fix config name for "tsd.network.keep_alive" in included config files. + - Fix an issue with the filter metric and tag resolution chain during queries. + - Fix an issue with malformed, double dotted timestamps (#724). + - Fix an issue with tag filters where we need a copy before modifying the list. + - Fix comments in the config file around TCP no delay settings. + - Fix some query stats calculations around averaging and estimating the number + of data points (#784). + - Clean out old .SWO files (#821) + - Fix a live-lock situation when performing regular expression or wildcard queries (#823). + - Change the static file path for the HTTP API to be relative (#857). + - Fix an issue where the GUI could flicker when two or more tag filters were set (#708). + +* Version 2.3.0 RC1 (2016-05-02) + +Noteworthy Changes: + - Introduced option --max-connection/tsd.core.connections.limit to set the maximum number + of connection a TSD will accept (#638) + - 'tsdb import' can now read from stdin (#580) + - Added datapoints counter (#369) + - Improved metadata storage performance (#699) + - added checkbox for showing global annotations in UI (#736) + - Added startup plugins, can be used for Service Discovery or other integration (#719) + - Added MetaDataCache plugin api + - Added timeshift() function (#175) + - Now align downsampling to Gregorian Calendar (#548, #657) + - Added None aggregator to fetch raw data along with first and last aggregators to + fetch only the first or last data points when downsampling. + - Added script to build OpenTSDB/HBase on OSX (#674) + - Add cross-series expressions with mathematical operators using Jexl + - Added query epxressions (alias(), scale(), absolute(), movingAverage(), highestCurrent(), + highestMax(), timeShift(), divide(), sum(), difference(), multiply()) (#625) + - Add a Unique ID assignment filter API for enforcing UID assignment naming conventions. + - Add a whitelist regular expression based UID assignment filter + - Add a time series storage filter plugin API that allows processing time series data + and determining if it should be stored or not. + - Allow using OpenTSDB with Google's Bigtable cloud platform or with Apache Cassandra + +Bug Fixes: + - Some improperly formatted timestamps were allowed (#724) + - Removed stdout logging from packaged logback.xml files (#715) + - Restore the ability to create TSMeta objects via URI + - Restore raw data points (along with post-filtered data points) in query stats + - Built in UI will now properly display global annotations when the query string is passed + +* Version 2.2.0 (2016-02-14) + +Noteworthy Changes + - Rework the QueryStats output to be a bit more useful and add timings from the + various scanners and query components. + - Modify the UI to allow for group by or aggregate per tag (use the new query feature) + - Rework the UI skin with the new TSDB logo and color scheme. + - Add the QueryLog config to logback.xml so users can optionally enable logging of + all queries along with their stats. + +Buf Fixes: + - Properly handle append data points in the FSCK utility. + - Fix FSCK to handle salting properly. + - Fix the IncomingDataPoints class for the CLI import tool to properly account for + salting. + - Fix the QueryStats maps by making sure the hash accounts for an unmodified list of + filters (return copies to callers so sorting won't break the hash code). + - Fix the case-insensitive wildcard filter to properly ignore the case. + - Fix the CLI dumper/scan utility to properly handle salted data. + - Fix a case where the compaction queue could grow unbounded when salting was enabled. + - Allow duplicate queries by default (as we did in the past) and users must now block + them explicitly. + - Fix the /api/stats endpoint to allow for returning a value if the max UID width is + set to 8 for any type. Previously it would throw an exception. + - Add a try catch to FSCK to debug issues where printing a problematic row would cause + a hangup or no output. + +* Version 2.2.0 RC3 (2015-11-11) + +Bug Fixes: + - Fix build issues where the static files were not copied into the proper location. + +* Version 2.2.0 RC2 (2015-11-09) + +Noteworthy Changes: + - Allow overriding the metric and tag UID widths via config file instead of + having to modify the source code. + +Bug Fixes: + - OOM handling script now handles multiple TSDs installed on the same host. + - Fix a bug where queries never return if an exception is thrown from the + storage layer. + - Fix random metric UID assignment in the CLI tool. + - Fix for meta data sync when salting is enabled. + - + +* Version 2.2.0 RC1 (2015-09-12) + +Noteworthy Changes: + - Add the option to randomly assign UIDs to metrics to improve distribution across + HBase region servers. + - Introduce salting of data to improve distribution of high cardinality regions + across region servers. + - Introduce query stats for tracking various timings related to TSD queries. + - Add more stats endpoints including /threads, /jvm and /region_clients + - Allow for deleting UID mappings via CLI or the API + - Name the various threads for easier debugging, particularly for distinguishing + between TSD and AsyncHBase threads. + - Allow for pre-fetching all of the meta information for the tables to improve + performance. + - Update to the latest AsyncHBase with support for secure HBase clusters and RPC + timeouts. + - Allow for overriding metric and tag widths via the config file. (Be careful!) + - URLs from the API are now relative instead of absolute, allowing for easier reverse + proxy use. + - Allow for percent deviation in the Nagios check + - Let queries skip over unknown tag values that may not exist yet (via config) + - Add various query filters such as case (in)sensitive pipes, wildcards and pipes + over tag values. Filters do not work over metrics at this time. + - Add support for writing data points using Appends in HBase as a way of writing + compacted data without having to read and re-write at the top of each hour. + - Introduce an option to emit NaNs or Nulls in the JSON output when downsampling and + a bucket is missing values. + - Introduce query time flags to show the original query along with some timing stats + in the response. + - Introduce a storage exception handler plugin that will allow users to spool or + requeue data points that fail writes to HBase due to various issues. + - Rework the HTTP pipeline to support plugins with RPC implementations. + - Allow for some style options in the Gnuplot graphs. + - Allow for timing out long running HTTP queries. + - Text importer will now log and continue bad rows instead of failing. + - New percentile and count aggregators. + - Add the /api/annotations endpoint to fetch multiple annotations in one call. + - Add a class to support improved bulk imports by batching requests in memory for a + full hour before writing. + +Bug Fixes: + - Modify the .rpm build to allow dashes in the name. + - Allow the Nagios check script to handle 0 values properly in checks. + - Fix FSCK where floating point values were not processed correctly (#430) + - Fix missing information from the /appi/uid/tsmeta calls (#498) + - Fix more issues with the FSCK around deleting columns that were in the list (#436) + - Avoid OOM issues over Telnet when the sending client isn't reading errors off it's + socket fast enough by blocking writes. + +* Version 2.1.4 (2016-02-14) + +Bug Fixes: + - Fix the meta table where the UID/TSMeta APIs were not sorting tags properly + prior to creating the row key, thus allowing for duplicates if the caller changed + the order of tags. + - Fix a situation where meta sync could hang forever if a routine threw an exception. + - Fix an NPE thrown when accessing the /logs endpoint if the Cyclic appender is not + enabled in the logback config. + - Remove an overly chatty log line in TSMeta on new time series creation. + +* Version 2.1.3 (2015-11-11) + +Bug Fixes: + - Fix build issues where the static files were not copied into the proper location. + +* Version 2.1.2 (2015-11-09) + +Bug Fixes: + - Fix the built-in UI to handle query parameter parsing properly (found when Firefox + changed their URI behavior) + - Fix comments about the Zookeeper quorum setting in various config files. + - Fix quoting in the Makefile when installing. + - Make sure builds write files in the proper location on FreeBSD. + +* Version 2.1.1 (2015-09-12) + +Bug Fixes: + - Relax the pgrep regex to correctly find and kill the java process in the RPM init.d + script. + - Improve query performance slightly when aggregating multiple series. + - Fix the /api/search/lookup API call to properly handle the limit parameter. + - Fix the /api/query/last endpoint to properly handle missing tsdb-meta tables. * Version 2.1.0 (2015-05-06) diff --git a/README b/README index 639c9d8b92..d4fddaf69a 100644 --- a/README +++ b/README @@ -12,7 +12,7 @@ systems, applications) at a large scale, and make this data easily accessible and graphable. Thanks to HBase's scalability, OpenTSDB allows you to collect thousands of -metrics from tens of thousands of hosts and applications, at a high rate +metrics from tens of thousands of hosts and applications, at a high rate (every few seconds). OpenTSDB will never delete or downsample data and can easily store hundreds of billions of data points. diff --git a/THANKS b/THANKS index b8005c0884..dd4c950fdd 100644 --- a/THANKS +++ b/THANKS @@ -7,51 +7,136 @@ complete and free from errors. Also see the AUTHORS file for the list of people and organizations with contributions significant enough to warrant copyright assignment. - +Adrian Goll Adrian Muraru +Adrian Goll Adrien Mogenet Alex Ioffe +Andre Pech Andrey Stepachev +Andy Flury +Anna Claiborne Aravind Gottipati Arvind Jayaprakash Berk D. Demir +BHourlier +Bikrant Neupane +Bizhu Qiu +Björn Marschollek +Björn Zettergren +Bryan Hernandez Bryan Zubrod +Camden Narzt +Can Zhang +Carlos Devoto +Chaotian Chris McClymont +Cristian Sechel +Christos Soulios Christophe Furmaniak Dave Barr +Davide D Amico +Designershao +Dfsklar +Eric Price +Ethan Wang Filippo Giunchedi +Gabriel Nicolas Avellaneda +GreatSnoopy Guenther Schmuelling +Haiyang Jiang +Hari Krishna Dara +Hari Sekhon +Hong Dai Thanh +Hugo M Fernandes Hugo Trippaers +Ioanszilgyi +Isaiah Choe +Ioan Szilagyi +Ivan Babrou Jacek Masiulaniec Jari Takkala +James Royalty Jan Mangs +Jason Harvey +Jim Scott +Jeffery Lim Jesse Chang +Jim Westfall Johan Zeeck +Johannes Meixner +John Ewing +John Seekins Jonathan Works Josh Thomas +Jsbali +Karan Mehta +Kevin Bowling +Kevin Landreth Kieren Hynd Kimoon Kim +Kousha Hamidi Kris Beevers +Kyle Brandt +Lex Herbert +Li Zhe Liangliang He +Liu Yubao +Loïs Burg +Lou Yunlong +Marcin Januszkiewicz Matt Jibson +Matt Schallert +Marc Tamsky +Marcin Januszkiewicz Mark Smith Martin Jansen +Max Meng +Michal Kimle Mike Bryant Mike Kobyakov +Misha Brukman +Nathan Owens Nicole Nagele +Neil Fordyce Nikhil Benesch +Nitin Aggarwal +NoHarm +Opsun +Øyvind Matheson Wergeland +Qu Dong Fang Paula Keezer +pengmengqing +Peter Edwards Peter Gotz +Peter Edwards +Ping Yong Pradeep Chhetri +Rajesh G +Rohan Nog +Ronan Harmegnies Ryan Berdeen +Sean Miller +Selim Chergui Siddartha Guthikonda Simon Matic Langford Slawek Ligus +Suman Newton Sy Le Tay Ray Chuan +Thomas Krajca Thomas Sanchez Tibor Vass -Tristan Colgate-McFarlane +Tony Di Nucci Tony Landells +Tristan Colgate-McFarlane +Utkarsh Bhatnagar Vasiliy Kiryanov -Zachary Kurey \ No newline at end of file +Vitaliy Fuks +William Kronmiller +White Lilis +Xiayang +Yulai Fu +Zachary Kurey +Zephyr Guo +Zong Chaoqiang \ No newline at end of file diff --git a/build-aux/deb/control/postrm b/build-aux/deb/control/postrm index 8cd9f45e92..aa12a2d040 100644 --- a/build-aux/deb/control/postrm +++ b/build-aux/deb/control/postrm @@ -7,7 +7,7 @@ case "$1" in rm -rf /var/log/opentsdb # remove **only** empty data dir - rmdir -p --ignore-fail-on-non-empty /tmp/opentsdb + rmdir --ignore-fail-on-non-empty /tmp/opentsdb ;; purge) diff --git a/build-aux/deb/init.d/opentsdb b/build-aux/deb/init.d/opentsdb index 4eb8ee3847..f0e69b2d25 100644 --- a/build-aux/deb/init.d/opentsdb +++ b/build-aux/deb/init.d/opentsdb @@ -29,11 +29,14 @@ MAX_OPEN_FILES=65535 # The first existing directory is used for JAVA_HOME # (if JAVA_HOME is not defined in $DEFAULT) -JDK_DIRS="/usr/lib/jvm/java-7-oracle /usr/lib/jvm/java-7-openjdk \ - /usr/lib/jvm/java-7-openjdk-amd64/ /usr/lib/jvm/java-7-openjdk-i386/ \ - /usr/lib/jvm/java-6-sun /usr/lib/jvm/java-6-openjdk \ - /usr/lib/jvm/java-6-openjdk-amd64 /usr/lib/jvm/java-6-openjdk-i386 \ - /usr/lib/jvm/default-java" +JDK_DIRS="\ + /usr/lib/jvm/java-8-oracle /usr/lib/jvm/java-8-openjdk \ + /usr/lib/jvm/java-8-openjdk-amd64/ /usr/lib/jvm/java-8-openjdk-i386/ \ + \ + /usr/lib/jvm/java-7-oracle /usr/lib/jvm/java-7-openjdk \ + /usr/lib/jvm/java-7-openjdk-amd64/ /usr/lib/jvm/java-7-openjdk-i386/ \ + \ + /usr/lib/jvm/default-java" # Look for the right JVM to use for jdir in $JDK_DIRS; do diff --git a/build-aux/deb/logback.xml b/build-aux/deb/logback.xml index 7f0fb57694..e3e04945a4 100644 --- a/build-aux/deb/logback.xml +++ b/build-aux/deb/logback.xml @@ -9,10 +9,15 @@ + 1024 + /var/log/opentsdb/opentsdb.log true @@ -27,18 +32,45 @@ 128MB - %d{HH:mm:ss.SSS} %-5level [%logger{0}.%M] - %msg%n + + + + /var/log/opentsdb/queries.log + true + + + /var/log/opentsdb/queries.log.%i + 1 + 4 + + + + 128MB + + + %date{ISO8601} [%logger.%M] %msg%n + + + + + + - + + + + + + diff --git a/build-aux/deb/opentsdb.conf b/build-aux/deb/opentsdb.conf index d95b65efe2..052936b962 100644 --- a/build-aux/deb/opentsdb.conf +++ b/build-aux/deb/opentsdb.conf @@ -6,17 +6,16 @@ tsd.network.port = 4242 # The IPv4 network address to bind to, defaults to all addresses # tsd.network.bind = 0.0.0.0 -# Enables Nagel's algorithm to reduce the number of packets sent over the -# network, default is True -#tsd.network.tcpnodelay = true +# Disable Nagel's algorithm, default is True +#tsd.network.tcp_no_delay = true -# Determines whether or not to send keepalive packets to peers, default +# Determines whether or not to send keepalive packets to peers, default # is True -#tsd.network.keepalive = true +#tsd.network.keep_alive = true -# Determines if the same socket should be used for new connections, default +# Determines if the same socket should be used for new connections, default # is True -#tsd.network.reuseaddress = true +#tsd.network.reuse_address = true # Number of worker threads dedicated to Netty, defaults to # of CPUs * 2 #tsd.network.worker_threads = 8 @@ -45,7 +44,7 @@ tsd.core.plugin_path = /usr/share/opentsdb/plugins # Whether or not to enable data compaction in HBase, default is True #tsd.storage.enable_compaction = true -# How often, in milliseconds, to flush the data point queue to storage, +# How often, in milliseconds, to flush the data point queue to storage, # default is 1,000 # tsd.storage.flush_interval = 1000 @@ -58,6 +57,6 @@ tsd.core.plugin_path = /usr/share/opentsdb/plugins # Path under which the znode for the -ROOT- region is located, default is "/hbase" #tsd.storage.hbase.zk_basedir = /hbase -# A space separated list of Zookeeper hosts to connect to, with or without +# A comma separated list of Zookeeper hosts to connect to, with or without # port specifiers, default is "localhost" #tsd.storage.hbase.zk_quorum = localhost diff --git a/build-aux/gen_build_data.sh b/build-aux/gen_build_data.sh index f402ae6354..4f211efbbc 100755 --- a/build-aux/gen_build_data.sh +++ b/build-aux/gen_build_data.sh @@ -39,6 +39,7 @@ eval "$sh" # Sets the timestamp and date variables. user=`whoami` host=`hostname` repo=`pwd` +branch=`git branch | grep -h '\*.*' | awk '{print $2}'` sh=`git rev-list --pretty=format:%h HEAD --max-count=1 \ | sed '1s/commit /full_rev=/;2s/^/short_rev=/'` @@ -92,6 +93,8 @@ public final class $CLASS { public static final String host = "$host"; /** Path to the repository in which this package was built. */ public static final String repo = "$repo"; + /** Git branch */ + public static final String branch = "$branch"; /** Human readable string describing the revision of this package. */ public static final String revisionString() { @@ -143,5 +146,10 @@ public final class $CLASS { // Can't instantiate. private $CLASS() {} + + public static void main(String[] args) { + System.out.println(revisionString()); + System.out.println(buildString()); + } } EOF diff --git a/build-aux/rpm/init.d/opentsdb b/build-aux/rpm/init.d/opentsdb index d721d483ba..8d589f9b0a 100644 --- a/build-aux/rpm/init.d/opentsdb +++ b/build-aux/rpm/init.d/opentsdb @@ -75,7 +75,7 @@ start() { # Set a default value for JVMARGS : ${JVMXMX:=-Xmx6000m} - : ${JVMARGS:=-DLOG_FILE_PREFIX=${LOG_FILE} -enableassertions -enablesystemassertions $JVMXMX -XX:OnOutOfMemoryError=/usr/share/opentsdb/tools/opentsdb_restart.py} + : ${JVMARGS:=-DLOG_FILE=${LOG_FILE}opentsdb.log -DQUERY_LOG=${LOG_FILE}queries.log -enableassertions -enablesystemassertions $JVMXMX -XX:OnOutOfMemoryError=/usr/share/opentsdb/tools/opentsdb_restart.py} export JVMARGS if [ "`id -u -n`" == root ] ; then @@ -83,6 +83,7 @@ start() { # daemons to create and rename log files. chown $USER: $LOG_DIR > /dev/null 2>&1 chown $USER: ${LOG_FILE}*opentsdb.log > /dev/null 2>&1 + chown $USER: ${LOG_FILE}*queries.log > /dev/null 2>&1 chown $USER: ${LOG_FILE}opentsdb.out > /dev/null 2>&1 chown $USER: ${LOG_FILE}opentsdb.err > /dev/null 2>&1 @@ -139,7 +140,7 @@ rh_status_q() { } findproc() { - pgrep -f "^java .* net.opentsdb.tools.TSDMain .*${NAME}" + pgrep -f "java .* net.opentsdb.tools.TSDMain .*${NAME}" } case "$1" in diff --git a/build-aux/rpm/logback.xml b/build-aux/rpm/logback.xml index 7f0fb57694..c1bb905908 100644 --- a/build-aux/rpm/logback.xml +++ b/build-aux/rpm/logback.xml @@ -9,16 +9,21 @@ + 1024 + - /var/log/opentsdb/opentsdb.log + ${LOG_FILE} true - /var/log/opentsdb/opentsdb.log.%i + ${LOG_FILE}.%i 1 3 @@ -27,18 +32,45 @@ 128MB - %d{HH:mm:ss.SSS} %-5level [%logger{0}.%M] - %msg%n + + + + ${QUERY_LOG} + true + + + ${QUERY_LOG}.%i + 1 + 4 + + + + 128MB + + + %date{ISO8601} [%logger.%M] %msg%n + + + + + + - + + + + + + diff --git a/build-aux/rpm/opentsdb.conf b/build-aux/rpm/opentsdb.conf index 11f66ca6cf..6c90f316c1 100644 --- a/build-aux/rpm/opentsdb.conf +++ b/build-aux/rpm/opentsdb.conf @@ -6,17 +6,16 @@ tsd.network.port = 4242 # The IPv4 network address to bind to, defaults to all addresses # tsd.network.bind = 0.0.0.0 -# Enables Nagel's algorithm to reduce the number of packets sent over the -# network, default is True -#tsd.network.tcpnodelay = true +# Disable Nagel's algorithm, default is True +#tsd.network.tcp_no_delay = true # Determines whether or not to send keepalive packets to peers, default # is True -#tsd.network.keepalive = true +#tsd.network.keep_alive = true # Determines if the same socket should be used for new connections, default # is True -#tsd.network.reuseaddress = true +#tsd.network.reuse_address = true # Number of worker threads dedicated to Netty, defaults to # of CPUs * 2 #tsd.network.worker_threads = 8 @@ -31,7 +30,7 @@ tsd.http.staticroot = /usr/share/opentsdb/static/ # Where TSD should write it's cache files to # *** REQUIRED *** -tsd.http.cachedir = /tmp/opentsdb +tsd.http.cachedir = /var/tmp/opentsdb # --------- CORE ---------- # Whether or not to automatically create UIDs for new metric types, default @@ -58,6 +57,6 @@ tsd.core.plugin_path = /usr/share/opentsdb/plugins # Path under which the znode for the -ROOT- region is located, default is "/hbase" #tsd.storage.hbase.zk_basedir = /hbase -# A space separated list of Zookeeper hosts to connect to, with or without +# A comma separated list of Zookeeper hosts to connect to, with or without # port specifiers, default is "localhost" #tsd.storage.hbase.zk_quorum = localhost diff --git a/build-aux/rpm/systemd/opentsdb.service b/build-aux/rpm/systemd/opentsdb.service new file mode 100644 index 0000000000..55a56e74d7 --- /dev/null +++ b/build-aux/rpm/systemd/opentsdb.service @@ -0,0 +1,14 @@ +[Unit] +Description=OpenTSDB +After=network-online.target +Before=shutdown.target + +[Service] +Type=simple +User=opentsdb +Group=opentsdb +LimitNOFILE=65535 +Environment='JVMARGS=-DLOG_FILE=/var/log/opentsdb/opentsdb.log -DQUERY_LOG=/var/log/opentsdb/queries.log -XX:+ExitOnOutOfMemoryError -enableassertions -enablesystemassertions' +ExecStart=/usr/bin/tsdb tsd --config /usr/share/opentsdb/etc/opentsdb/opentsdb.conf +Restart=always +StandardOutput=journal diff --git a/build-aux/rpm/systemd/opentsdb@.service b/build-aux/rpm/systemd/opentsdb@.service new file mode 100644 index 0000000000..847667a032 --- /dev/null +++ b/build-aux/rpm/systemd/opentsdb@.service @@ -0,0 +1,15 @@ +[Unit] +Description=OpenTSDB on port %i +After=network-online.target +Before=shutdown.target + +[Service] +Type=simple +User=opentsdb +Group=opentsdb +LimitNOFILE=65535 +Environment=JAVA_HOME=/usr/lib/jvm/jre-openjdk +Environment='JVMARGS=-Xmx6000m -DLOG_FILE=/var/log/opentsdb/%p_%i.log -DQUERY_LOG=/var/log/opentsdb/%p_%i_queries.log -XX:+ExitOnOutOfMemoryError -enableassertions -enablesystemassertions' +ExecStart=/usr/bin/tsdb tsd --config /etc/opentsdb/opentsdb.conf --port %i +Restart=always +StandardOutput=journal diff --git a/build-bigtable.sh b/build-bigtable.sh new file mode 100644 index 0000000000..1076da2f8e --- /dev/null +++ b/build-bigtable.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -xe +test -f configure || ./bootstrap +test -d build || mkdir build +cd build +test -f Makefile || ../configure --with-bigtable "$@" +MAKE=make +[ `uname -s` = "FreeBSD" ] && MAKE=gmake +exec ${MAKE} "$@" \ No newline at end of file diff --git a/build-cassandra.sh b/build-cassandra.sh new file mode 100644 index 0000000000..a29a383e34 --- /dev/null +++ b/build-cassandra.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -xe +test -f configure || ./bootstrap +test -d build || mkdir build +cd build +test -f Makefile || ../configure --with-cassandra "$@" +MAKE=make +[ `uname -s` = "FreeBSD" ] && MAKE=gmake +exec ${MAKE} "$@" \ No newline at end of file diff --git a/configure.ac b/configure.ac index 66ab5753b8..37e8e7c768 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2012 The OpenTSDB Authors. +# Copyright (C) 2011-2024 The OpenTSDB Authors. # # This library is free software: you can redistribute it and/or modify it # under the terms of the GNU Lesser General Public License as published @@ -14,7 +14,7 @@ # along with this library. If not, see . # Semantic Versioning (see http://semver.org/). -AC_INIT([opentsdb], [2.1.0], [opentsdb@googlegroups.com]) +AC_INIT([opentsdb], [2.5.0-RC1], [opentsdb@googlegroups.com]) AC_CONFIG_AUX_DIR([build-aux]) AM_INIT_AUTOMAKE([foreign]) @@ -24,6 +24,26 @@ AC_CONFIG_FILES([ AC_CONFIG_FILES([opentsdb.spec]) AC_CONFIG_FILES([build-aux/fetchdep.sh], [chmod +x build-aux/fetchdep.sh]) +AC_ARG_WITH([bigtable], + [AS_HELP_STRING([--with-bigtable], [Enable Google's Bigtable backend])], + [with_bigtable=yes], + [with_bigtable=no]) + +AS_IF([test "x$with_bigtable" = "xyes"], + [AM_CONDITIONAL(BIGTABLE, true)], + [AM_CONDITIONAL(BIGTABLE, false)] +) + +AC_ARG_WITH([cassandra], + [AS_HELP_STRING([--with-cassandra], [Enable Cassandra backend])], + [with_cassandra=yes], + [with_cassandra=no]) + +AS_IF([test "x$with_cassandra" = "xyes"], + [AM_CONDITIONAL(CASSANDRA, true)], + [AM_CONDITIONAL(CASSANDRA, false)] +) + TSDB_FIND_PROG([md5], [md5sum md5 gmd5sum digest]) if test x`basename "$MD5"` = x'digest'; then MD5='digest -a md5' diff --git a/fat-jar/create-src-dir-overlay.sh b/fat-jar/create-src-dir-overlay.sh new file mode 100755 index 0000000000..34c0053ee7 --- /dev/null +++ b/fat-jar/create-src-dir-overlay.sh @@ -0,0 +1,29 @@ +# Creates directory structure overlay on top of original source directories so +# that the overlay matches Java package hierarchy. +#!/usr/bin/env bash + +if [ ! -d src-main ]; then + mkdir src-main + mkdir src-main/net + mkdir src-main/tsd + (cd src-main/net && ln -s ../../src opentsdb) + (cd src-main/tsd && ln -s ../../src/tsd/QueryUi.gwt.xml QueryUi.gwt.xml) + (cd src-main/tsd && ln -s ../../src/tsd/client client) +fi +if [ ! -d src-test ]; then + mkdir src-test + mkdir src-test/net + (cd src-test/net && ln -s ../../test opentsdb) +fi +if [ ! -d src-resources ]; then + mkdir src-resources + (cd src-resources && ln -s ../fat-jar/logback.xml) + (cd src-resources && ln -s ../fat-jar/file-logback.xml) + (cd src-resources && ln -s ../fat-jar/opentsdb.conf.json) +fi +if [ ! -d test-resources ]; then + mkdir test-resources + (cd test-resources && ln -s ../fat-jar/test-logback.xml) + (cd test-resources && ln -s ../fat-jar/opentsdb.conf.json) +fi + diff --git a/fat-jar/fat-jar-pom.xml.in b/fat-jar/fat-jar-pom.xml.in new file mode 100644 index 0000000000..314a12e224 --- /dev/null +++ b/fat-jar/fat-jar-pom.xml.in @@ -0,0 +1,759 @@ + + + 4.0.0 + + net.opentsdb + opentsdb-fatjar + @spec_version@ + @spec_title@ + + @spec_vendor@ + http://opentsdb.net + + + OpenTSDB is a distributed, scalable Time Series Database (TSDB) + written on top of HBase. OpenTSDB was written to address a common need: + store, index and serve metrics collected from computer systems (network + gear, operating systems, applications) at a large scale, and make this + data easily accessible and graphable. + + http://opentsdb.net + + + LGPLv2.1+ + http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html + repo + + + + + scm:git:git@github.com:OpenTSDB/opentsdb.git + https://github.com/OpenTSDB/opentsdb + + + GitHub + https://github.com/OpenTSDB/opentsdb/issues + + + + User List + opentsdb@googlegroups.com + opentsdb+subscribe@googlegroups.com + opentsdb+unsubscribe@googlegroups.com + https://groups.google.com/group/opentsdb + + + + + tsuna + Benoit "tsuna" Sigoure + tsunanet@gmail.com + + developer + + -8 + + + 2010 + + jar + + + + UTF-8 + 1.6 + 1.6 + + @GUAVA_VERSION@ + @JACKSON_VERSION@ + @NETTY_VERSION@ + @SUASYNC_VERSION@ + @ZOOKEEPER_VERSION@ + @SLF4J_API_VERSION@ + @ASYNCHBASE_VERSION@ + @KRYO_VERSION@ + @LOG4J_OVER_SLF4J_VERSION@ + @LOGBACK_CORE_VERSION@ + @LOGBACK_CLASSIC_VERSION@ + @HAMCREST_VERSION@ + @JAVASSIST_VERSION@ + @JUNIT_VERSION@ + @MOCKITO_VERSION@ + @OBJENESIS_VERSION@ + @POWERMOCK_MOCKITO_VERSION@ + @APACHE_MATH_VERSION@ + @JEXL_VERSION@ + @JGRAPHT_VERSION@ + + @GWT_VERSION@ + @GWT_THEME_VERSION@ + 2.1.2 + 2.5.1 + 2.1 + 1.2.1 + 1.7 + 1.7 + 2.9 + 2.16 + 2.4 + 1.4 + 2.8.1 + + + + + src-main + src-test + + + src-resources + + + + + test-resources + + + + + + + + + org.apache.maven.plugins + maven-eclipse-plugin + ${eclipse-plugin.version} + + true + true + + net/opentsdb/tsd/client/** + + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${compiler-plugin.version} + + ${project.source} + ${project.source.target} + -Xlint + + **/client/*.java + + + **/TestGraphHandler.java + + + + + + org.codehaus.mojo + exec-maven-plugin + ${exec-maven.version} + + + generate-build-data + + build-aux/gen_build_data.sh + + + target/generated-sources/net/opentsdb/tools/BuildData.java + net.opentsdb.tools + BuildData + + + generate-sources + + exec + + + + create-plugin-test-jar + + + jar + + cvfm + plugin_test.jar + test/META-INF/MANIFEST.MF + -C + target/test-classes + net/opentsdb/plugin/DummyPluginA.class + -C + target/test-classes + net/opentsdb/plugin/DummyPluginB.class + -C + target/test-classes + net/opentsdb/search/DummySearchPlugin.class + -C + target/test-classes + net/opentsdb/tsd/DummyHttpSerializer.class + -C + target/test-classes + net/opentsdb/tsd/DummyRpcPlugin.class + -C + target/test-classes + net/opentsdb/tsd/DummyRTPublisher.class + -C + test + META-INF/services/net.opentsdb.plugin.DummyPlugin + -C + test + META-INF/services/net.opentsdb.search.SearchPlugin + -C + test + META-INF/services/net.opentsdb.tsd.HttpSerializer + -C + test + META-INF/services/net.opentsdb.tsd.RpcPlugin + -C + test + META-INF/services/net.opentsdb.tsd.RTPublisher + + + test-compile + + exec + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + ${build-helper.version} + + + add-source + generate-sources + + add-source + + + + target/generated-sources + + + + + + + + maven-antrun-plugin + ${antrun-plugin.version} + + + process-resources + + + + + + + + + + run + + + + include-query-ui + package + + + + + + + + + + + run + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${surefire-plugin.version} + + -Xmx2048m -XX:MaxPermSize=256m + true + classes + 2 + false + + ${basedir}/test-resources/test-logback.xml + + + + + + org.apache.maven.plugins + maven-jar-plugin + ${jar-plugin.version} + + + + true + true + + + net.opentsdb.tools.OpenTSDBMain + + + + WEB-INF/deploy/** + queryui/** + + + + + + org.codehaus.mojo + gwt-maven-plugin + ${gwt.version} + + + + true + tsd.QueryUi + + compile + + compile + + + + + + + org.apache.maven.plugins + maven-source-plugin + ${source-plugin.version} + + + attach-sources + + jar + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + ${javadoc-plugin.version} + + + attach-javadocs + + jar + + + + + + true + true + + Copyright © {inceptionYear}-{currentYear}, + ${project.organization.name} + + + + + + org.apache.maven.plugins + maven-gpg-plugin + ${gpg-plugin.version} + + + sign-artifacts + verify + + sign + + + + + tsunanet@gmail.com + + + + + org.apache.maven.plugins + maven-shade-plugin + ${shade.version} + + + package + + shade + + + target/${project.build.finalName}-fat.jar + true + + + *:* + + + /*.proto + + + + + net.opentsdb.tools.OpenTSDBMain + + + + + + + + + + + + + + + com.google.guava + guava + ${guava.version} + + + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + + + + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + + + io.netty + netty + ${netty.version} + + + + + com.stumbleupon + async + ${suasync.version} + + + + + org.apache.zookeeper + zookeeper + ${zookeeper.version} + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + jline + jline + + + junit + junit + + + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + + org.hbase + asynchbase + ${asynchbase.version} + + + + org.apache.commons + commons-math3 + ${apache-math.version} + + + + org.apache.commons + commons-jexl + ${jexl.version} + + + + org.jgrapht + jgrapht-core + ${jgrapht.version} + + + + com.esotericsoftware.kryo + kryo + ${kryo.version} + + + + + + + org.slf4j + log4j-over-slf4j + ${log4j.version} + + + + + ch.qos.logback + logback-core + ${logback-core.version} + + + + + ch.qos.logback + logback-classic + ${logback-classic.version} + + + + + + + org.hamcrest + hamcrest-core + ${hamcrest.version} + test + + + + + org.javassist + javassist + ${javassist.version} + test + + + + + junit + junit + ${junit.version} + test + + + + + org.mockito + mockito-core + ${mockito.version} + test + + + + + org.objenesis + objenesis + ${objenesis.version} + test + + + + + org.powermock + powermock-api-mockito + ${powermock.version} + test + + + + + org.powermock + powermock-module-junit4 + ${powermock.version} + test + + + + + + com.google.gwt + gwt-user + ${gwt.version} + + + + net.opentsdb + opentsdb_gwt_theme + ${gwt-theme.version} + + + + + + + + + + + + windows + + + + + + org.codehaus.mojo + exec-maven-plugin + ${exec-maven.version} + + + build-data-codegen + generate-sources + + exec + + + cmd.exe + + /C + "build-aux\gen_build_data.cmd" + + + + + + + com.helger.maven + ph-javacc-maven-plugin + 2.8.0 + + + jjc + generate-sources + + javacc + + + 1.6 + true + net.opentsdb.query.expression.parser + ${basedir}/src/ + ${project.build.directory}/generated-sources/ + + + + + + + + + + Linux + + + !windows + + + + + + + org.codehaus.mojo + exec-maven-plugin + ${exec-maven.version} + + + build-data-codegen + generate-sources + + exec + + + build-aux/gen_build_data.sh + + + target/generated-sources/net/opentsdb/BuildData.java + net.opentsdb + BuildData + + + + + + + com.helger.maven + ph-javacc-maven-plugin + 2.8.0 + + + jjc + generate-sources + + javacc + + + 1.6 + true + net.opentsdb.query.expression.parser + ${basedir}/src/ + ${project.build.directory}/generated-sources/ + + + + + + + + + + + + + org.sonatype.oss + oss-parent + 7 + + + \ No newline at end of file diff --git a/fat-jar/fat-jar-readme.md b/fat-jar/fat-jar-readme.md new file mode 100644 index 0000000000..829cdc760c --- /dev/null +++ b/fat-jar/fat-jar-readme.md @@ -0,0 +1,180 @@ +## OpenTSDB FatJar + +### Introduction + +FatJar is a modified build procedure for OpenTSDB intended to provide a single executable jar that contains all of the java artifacts and resources required to run an OpenTSDB 2.1+ instance. This includes the tsd and all associated command line tools. It does not include the following which remain external dependencies: +* The gnuplot executable which generates data visualizations in the OpenTSDB graphical web console. +* The HBase instance or cluster where time-series data is stored. +* The Java runtime. + +With a FatJar build and the above dependencies, it is possible to start an OpenTSDB as simply as: + +```java -jar opentsdb-2.1.jar``` + +#### Building the FatJar + +These steps assume building from my current fork repo. + +1. Clone: ```git clone https://github.com/nickman/opentsdb.git``` +2. Switch to project directory: ```cd opentsdb``` +3. Switch to **next** branch: ```git checkout next``` +4. Build the **fat-jar-pom.xml**: ```./build.sh fat-jar-pom.xml``` +5. Run maven, specifying the fat-jar pom (and skipping the gpg plugin and tests): ```mvn -f fat-jar-pom.xml -Dgpg.skip -DskipTests clean install``` +6. Fire her up, Scotty: ```java -jar ./target/opentsdb-2.1.0RC1-fat.jar tsd``` + +**NOTE**: FatJar requires *maven 3*. + +**ANOTHER NOTE**: THe OpenTSDB code base has some non-compliant/missing javadoc tags. If you are compiling with Java 8, [the build will fail](http://http://stackoverflow.com/questions/15886209/maven-is-not-working-in-java-8-when-javadoc-tags-are-incomplete), so: +1. Use Java 6 or 7 OR... +2. Edit the fat-jar-pom.xml, find the javadoc plugin and uncomment the line: + +`````` + +###### This is currently on line 356. It is commented because is it not supported for Java 6 & 7. + +(Note that an [unreviewed] aspect of FatJar is the provision of default values for *all* configuration parameters so the three traditionally required parameters, **port**, **staticroot** and **cachedir** are not required.) + +FatJar is accomplished with: +1. Small changes to 3 existing OpenTSDB classes (all other aspects of the code base remain unchanged) +2. The addition of 8 new classes +3. Additions to the OpenTSDB make scripts to generate the FatJar maven pom +3. A maven based build that packages the following into a single jar: + * The OpenTSDB core + * All of the maven defined dependencies (jars) + * An executable jar defining manifest + * A JSON based configuration definition repository + * All of the OpenTSDB UI resources + +### Motivation and Benefits + +The original motivation for a FatJar build was the simplification of deploying OpenTSDB to multiple "locked-down" environments where it was not feasible to build from source. The addition of RPM and DEB packages for Linux have significantly eased this process, but deployment to non-Linux environments remains a largely manual procedure. While it would be conceivable to implement Solaris and Windows install packages (our 2 pain points), it seemed simpler to adopt the FatJar approach and kill the entire flock of birds with one stone. + +The FatJar has been optimized to support simplicity of deployment and configuration, while maintaining complete backward compatability so the modifications are completely additive. + +Among some of the benefits we enjoy from the FatJar build: +* Simplified deployment +* No requirement for a pre-deployed UI static content directory. The FatJar contains all the static resources and unloads this content into the configured/default directory at start time, including the GWT Web App content and the gnuplot wrapper script. If any content already exists and has a newer timestamp than the corresponding resource in the FatJar, the resource is left as is. An additional FatJar provided command line tool allows for the pre-creation of the static content directory, which is useful in development. +* The process PID is written to a default [and configurable] pid file, and deleted on clean shutdown. +* URL based config and include supporting the load of configuration files from http:// or file:// based sources. +* Configuration properties can be tokenized with System Prop and/or Environmental Variable decodes as well as JavaScript snippets to support dynamically computed property values according to the deployment environment. +* Refined logging configuration with configuration override-able external logging config file location and command line options to set logging levels for major packages in OpenTSDB and associated libraries. + + +### Changes and Additions + +#### Modified Classes and Files + +* **.gitignore**: Added transient artifacts created by FatJar builds. +* **Makefile.am**: Added target **fat-jar-pom.xml** that generates the fat-jar symbolic source overlay and fat-jar-pom.xml. Added the new source and test files to **tsdb_SRC** and **test_SRC**. +* **src/tsd/GraphHandler.java**: FatJar introduced a class GnuplotInstaller which locates the gnuplot executable by scanning the path and installs the Gnuplot wrapper script. This modification integrates GnuplotInstaller into the GraphHandler. +* **src/tsd/PipelineFactory.java**: The pipeline's idle-timeout handler HashedWheelTimer was using a non-daemon thread and preventing an orderly shutdown. This is also a fix for #455. +* **src/utils/Config.java**: Made ```loadStaticVariables()``` public. + +#### New Classes and Files + +* **fat-jar/create-src-dir-overlay.sh**: Creates a maven oriented source overlay and generates a **fat-jar-pom.xml**. +* **fat-jar/fat-jar-pom.xml.in**: The input template for **fat-jar-pom.xml**. +* **fat-jar/logback.xml**: The FatJar OpenTSDB boot time and default logging configuration. +* **fat-jar/file-logback.xml**: The parameterized file based logging and file-rolling configuration if a log file name is configured. +* **fat-jar/test-logback.xml**: Test resource logback configuration. Pretty much like **fat-jar/logback.xml** but without the jmx component enabled since it messes up the use of PowerMock. +* **fat-jar/opentsdb.conf.json**: The configuration parameter definition repository +* **src/tools/ArgValueValidator.java**: Interface that defines a class that validates a type of configuration parameter. +* **src/tools/ConfigArgP.java**: Configuration parameter manager. Conceptually a merge of Config and ArgP. Contains a variety of testing hooks. +* **src/tools/ConfigMetaType.java**: Enumeration of configuration parameter types, each with a link to an **ArgValueValidator** so that each parameter value can be validated as accurately as possible. +* **src/tools/GnuplotInstaller.java**: Validates the presence of the gnuplot executable and installs the OpenTSDB gnuplot invocation wrapper shell script. +* **src/tools/Main.java**: The FatJar substitute for **TSDMain**, serving as the boot entry point for the TSD as well as the command line tools. +* **test/tools/TestConfigArgP.java**: Test suite for the **ConfigArgP** class. + +#### New Config Parameters + +* **tsd.logback.file**: The name of the log file the OpenTSDB logback configuration will log to when running the FatJar. +* **tsd.logback.rollpattern**: The logback rolling file appender roll pattern used to roll the file defined in **tsd.logback.file**. +* **tsd.logback.console**: If specified, logback will log to the configured file and keep logging to the console, otherwise, when the file appender is activated, the console appender is removed. +* **tsd.logback.config**: Points to an external [outside the FatJar] logback config file, in which case the internal file based logging configuration is ignored once the config has been loaded. +* **tsd.process.pid.file**: The name and location of the OpenTSDB PID file. +* **tsd.process.pid.ignore.existing**: Normally, if the PID file defined in **tsd.process.pid.file** is found at startup, startup will be aborted. This directive ignores the existing file and overwrites it. +* **tsd.core.config**: Points to a core configuration file which should be used as the base line config. This means the built in config can simply contain this item and point to a shared file or HTTP based location for the main configuration. +* **tsd.ui.noexport**: Disables the automatic UI content export to the local file system. +* **tsd.core.config.include**: Specifies a file or HTTP based config that will override the config specified in the core configuration. (See Presedence below) +* **help**: Not really a configuration, but present to flag the word as an OpenTSDB processable command. +* Default Bindings: These are default JS bindings which can be useful if scripting dynamic configuration values for thread counts and/or cache sizes. Note that all system properties and environmental variables are automatically available in the JavaScript context. + * **processors**: The number of cores available to the JVM + * **maxbytes**: The maximum amount of memory available to the JVM + +#### Configuration Presedence + +In order of presedence: +1. Command line parameters +2. Included parameters +3. Core parameters + +#### Installing the UI content + +The FATJar supports a command line tool to install the UI static content to a specified location: + +``` +Usage: java -jar exportui --d [--p] + --d=DIR The directory to export the UI content to + --p Create the directory if it does not exist +``` +Example: + +```java -jar opentsdb-2.1.0RC1.jar exportui --d /tmp/opentsdb/static-content --p``` + +Output: + +``` +2015-03-15 16:48:14,988 INFO [main] Main: Created exportui directory [/tmp/opentsdb/static-content] +2015-03-15 16:48:15,258 INFO [main] Main: + + =================================================== + Static Root Directory:[/tmp/opentsdb/static-content] + Total Files Written:24 + Total Bytes Written:1162522 + File Write Failures:0 + Existing File Newer Than Content:0 + Elapsed (ms):270 + =================================================== +``` + +#### Command Line Help + +The FatJar supports a slightly modified command line help model. The command help will print the main “menu” of options for the FatJar. + +``` +java -jar opentsdb-x.x.x.jar help +Usage: java -jar [opentsdb.jar] [command] [args] +Valid commands: + tsd: Starts a new TSDB instance + fsck: Searches for and optionally fixes corrupted data in a TSDB + import: Imports data from a file into HBase through a TSDB + mkmetric: Creates a new metric + query: Queries time series data from a TSDB + scan: Dumps data straight from HBase + uid: Provides various functions to search or modify information in the tsdb-uid table. + exportui: Exports the OpenTSDB UI static content + + Use help for details on a command +``` + +Using java -jar opentsdb-x.x.x.jar help for all CLI tools will print the tool's existing usage message. Help for tsd has a couple of additional options. Using the call above, with tsd as the command, the standard usage will be printed. With the additional argument of the keyword **extended**, the full set of configuration options is printed. Full example [here](https://gist.github.com/nickman/673852d8d88675043f45). + +#### Eclipse Support + +An [almost] incidental benefit for users of the Eclipse IDE is that the FatJar source overlay simplifies the setup of a "working-out-of-the-box" Eclipse project using maven. Just do this: + +```mvn -f fat-jar-pom.xml eclipse:eclipse``` + +Then load up the project in Eclipse and launch **net.opentsdb.tools.Main** with the parameter **tsd"**. + +All at no cost to you. + +#### Last Note + +There's some big changes here. I recognize it's not perfect. Please let me have any feedback. It will not be taken personally, but only to improve OpenTSDB. + + + + + + diff --git a/fat-jar/file-logback.xml b/fat-jar/file-logback.xml new file mode 100644 index 0000000000..dc8d45edb2 --- /dev/null +++ b/fat-jar/file-logback.xml @@ -0,0 +1,72 @@ + + + + + + %d{ISO8601} %-5level [%thread] %logger{0}: %msg%n + + + + + 1024 + + + + ${tsdb.logback.file} + true + + + ${tsd.logback.rollpattern} + + + 5MB + + + 30 + + + + UTF-8 + %d %-4relative [%thread] %-5level %logger{35} - %msg%n + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/fat-jar/logback.xml b/fat-jar/logback.xml new file mode 100644 index 0000000000..7c0877cb71 --- /dev/null +++ b/fat-jar/logback.xml @@ -0,0 +1,50 @@ + + + + + + + %d{ISO8601} %-5level [%thread] %logger{0}: %msg%n + + + + + 1024 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/fat-jar/opentsdb.conf.json b/fat-jar/opentsdb.conf.json new file mode 100644 index 0000000000..f1ade767c6 --- /dev/null +++ b/fat-jar/opentsdb.conf.json @@ -0,0 +1,596 @@ +{ + "config-items": [ + { + "key": "tsd.core.auto_create_metrics", + "cl-option": "--auto-metric", + "defaultValue": "false", + "description": "Whether or not a data point with a new metric will assign a UID to the metric. When false, a data point with a metric that is not in the database will be rejected and an exception will be thrown", + "help": "default", + "meta": "BOOL" + }, + { + "key": "tsd.core.auto_create_tagks", + "cl-option": "--auto-tagk", + "defaultValue": "true", + "description": "Whether or not a data point with a new tag name will assign a UID to the tagk. When false, a data point with a tag name that is not in the database will be rejected and an exception will be thrown", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.core.auto_create_tagvs", + "cl-option": "--auto-tagv", + "defaultValue": "true", + "description": "Whether or not a data point with a new tag value will assign a UID to the tagv. When false, a data point with a tag value that is not in the database will be rejected and an exception will be thrown", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.core.connections.limit", + "cl-option": "--max-connection", + "defaultValue": 0, + "description": "Sets the maximum number of connections a TSD will handle, additional connections are immediately closed", + "help": "default", + "meta": "POSINT" + }, + { + "key": "tsd.core.meta.enable_realtime_ts", + "cl-option": "--realtime-ts", + "defaultValue": "false", + "description": "Whether or not to enable real-time TSMeta object creation", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.core.meta.enable_realtime_uid", + "cl-option": "--realtime-uid", + "defaultValue": "false", + "description": "Whether or not to enable real-time UIDMeta object creation", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.core.meta.enable_tsuid_incrementing", + "cl-option": "--tsuid-incr", + "defaultValue": "false", + "description": "Whether or not to enable tracking of TSUIDs by incrementing a counter every time a data point is recorded. (Overrides tsd.core.meta.enable_tsuid_tracking)", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.core.meta.enable_tsuid_tracking", + "cl-option": "--tsuid-tracking", + "defaultValue": "false", + "description": "Whether or not to enable tracking of TSUIDs by storing a 1 with the current timestamp every time a data point is recorded", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.core.plugin_path", + "cl-option": "--plugin-path", + "description": "A path to search for plugins when the TSD starts. If the path is invalid, the TSD will fail to start. Plugins can still be enabled if they are in the class path", + "help": "extended", + "meta": "CLASSPATH" + }, + { + "key": "tsd.core.preload_uid_cache", + "cl-option": "--tsd-preloaduids", + "defaultValue": "false", + "description": "Enables pre-population of the UID caches when starting a TSD", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.core.preload_uid_cache.max_entries", + "cl-option": "--tsd-maxuids", + "defaultValue": 300000, + "description": "The number of rows to scan for UID pre-loading", + "help": "extended", + "meta": "POSINT" + }, + { + "key": "tsd.core.storage_exception_handler.enable", + "cl-option": "--enable-exhandler", + "defaultValue": "false", + "description": "Whether or not to enable the configured storage exception handler plugin", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.core.storage_exception_handler.plugin ", + "cl-option": "--exhandler-plugin", + "description": "The full class name of the storage exception handler plugin you wish to use", + "help": "extended", + "meta": "CLASS" + }, + { + "key": "tsd.core.timezone", + "cl-option": "--tsd-timezone", + "defaultValue": "${user.timezone}", + "description": "A localized timezone identification string used to override the local system timezone used when converting absolute times to UTC when executing a query. This does not affect incoming data timestamps. E.g. America/Los_Angeles", + "help": "extended", + "meta": "TIMEZONE" + }, + { + "key": "tsd.core.tree.enable_processing", + "cl-option": "--enable-tree", + "defaultValue": "false", + "description": "Whether or not to enable processing new/edited TSMeta through tree rule sets", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.http.cachedir", + "cl-option": "--cachedir", + "defaultValue": "${java.io.tmpdir}/.tsdb/http-cache/", + "description": "The full path to a location where temporary files can be written. e.g. /tmp/opentsdb", + "help": "default", + "meta": "DIR" + }, + { + "key": "tsd.http.request.cors_domains", + "cl-option": "--cors-domains", + "description": "A comma separated list of domain names to allow access to OpenTSDB when the Origin header is specified by the client. If empty, CORS requests are passed through without validation. The list may not contain the public wildcard * and specific domains at the same time", + "help": "extended", + "meta": "LIST" + }, + { + "key": "tsd.http.request.cors_headers", + "cl-option": "--cors-headers", + "defaultValue": "Authorization, Content-Type, Accept, Origin, User-Agent, DNT, Cache-Control, X-Mx-ReqToken, Keep-Alive, X-Requested-With, If-Modified-Since", + "description": "A comma separated list of headers sent to clients when executing a CORs request. The literal value of this option will be passed to clients", + "help": "extended", + "meta": "LIST" + }, + { + "key": "tsd.http.request.enable_chunked", + "cl-option": "--enable-chunked", + "defaultValue": "false", + "description": "Whether or not to enable incoming chunk support for the HTTP RPC", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.http.request.max_chunk", + "cl-option": "--max-chunk", + "defaultValue": "4096", + "description": "The maximum request body size to support for incoming HTTP requests when chunking is enabled", + "help": "extended", + "meta": "GTZEROINT" + }, + { + "key": "tsd.http.show_stack_trace", + "cl-option": "--show-stack", + "defaultValue": "true", + "description": "Whether or not to return the stack trace with an API query response when an exception occurs", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.http.staticroot", + "cl-option": "--staticroot", + "defaultValue": "${java.io.tmpdir}/.tsdb/static-content", + "description": "Location of a directory where static files, such as JavaScript files for the web interface, are located. e.g. /opt/opentsdb/staticroot", + "help": "default", + "meta": "DIR" + }, + { + "key": "tsd.mode", + "cl-option": "--tsd-mode", + "defaultValue": "rw", + "description": "Whether or not the TSD will allow writing data points. Must be either rw to allow writing data or ro to block writes", + "help": "extended", + "meta": "RWMODE" + }, + { + "key": "tsd.network.async_io", + "cl-option": "--async-io", + "defaultValue": "true", + "description": "Whether or not to use NIO or tradditional blocking IO", + "help": "default", + "meta": "BOOL" + }, + { + "key": "tsd.network.backlog", + "cl-option": "--backlog", + "defaultValue": "3072", + "description": "The connection queue depth for completed or incomplete connection requests depending on OS. The default may be limited by the 'somaxconn' kernel setting or set by Netty to 3072", + "help": "default", + "meta": "GTZEROINT" + }, + { + "key": "tsd.network.bind", + "cl-option": "--bind", + "defaultValue": "0.0.0.0", + "description": "An IPv4 address to bind to for incoming requests. The default is to listen on all interfaces. e.g. 127.0.0.1", + "help": "default", + "meta": "ADDR" + }, + { + "key": "tsd.network.keep_alive", + "cl-option": "--keep-alive", + "defaultValue": "true", + "description": "Whether or not to allow keep-alive connections", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.network.port", + "cl-option": "--port", + "defaultValue": "4242", + "description": "The TCP port to use for accepting connections", + "help": "default", + "meta": "POSINT" + }, + { + "key": "tsd.network.reuse_address", + "cl-option": "--reuse-address", + "defaultValue": "true", + "description": "Whether or not to allow reuse of the bound port within Netty", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.network.tcp_no_delay", + "cl-option": "--tcp-no-delay", + "defaultValue": "true", + "description": "Whether or not to disable TCP buffering before sending data", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.network.worker_threads", + "cl-option": "--worker-threads", + "defaultValue": "$[new Integer(cores * 2)]", + "description": "The number of asynchronous IO worker threads for Netty", + "help": "default", + "meta": "GTZEROINT" + }, + { + "key": "tsd.core.socket.timeout", + "cl-option": "--socket-timeout", + "defaultValue": 0, + "description": "The idle time timeout for allocated sockets (seconds)", + "help": "extended", + "meta": "POSINT" + }, + { + "key": "tsd.no_diediedie", + "cl-option": "--tsd-nodie", + "defaultValue": "false", + "description": "Enable or disable the diediedie HTML and ASCII commands to shutdown a TSD", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.query.allow_simultaneous_duplicates", + "cl-option": "--tsd-allowsimdups", + "defaultValue": "false", + "description": "Whether or not to allow simultaneous duplicate queries from the same host. If disabled, a second query that comes in matching one already running will receive an exception", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.query.filter.expansion_limit", + "cl-option": "--tsd-expansionlimit", + "defaultValue": "4096", + "description": "The maximum number of tag values to include in the regular expression sent to storage during scanning for data. A larger value means more computation on the HBase region servers", + "help": "extended", + "meta": "POSINT" + }, + { + "key": "tsd.query.skip_unresolved_tagvs", + "cl-option": "--tsd-skip-unresolved", + "defaultValue": "false", + "description": "Whether or not to continue querying when the query includes a tag value that hasn't been assigned a UID yet and may not exist", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.query.timeout", + "cl-option": "--tsd-query-timeout", + "defaultValue": "0", + "description": "How long, in milliseconds, before canceling a running query. A value of 0 means queries will not timeout", + "help": "extended", + "meta": "POSINT" + }, + { + "key": "tsd.rpc.plugins", + "cl-option": "--rpc-plugins", + "description": "A comma delimited list of RPC plugins to load when starting a TSD. Must contain the entire class name", + "help": "extended", + "meta": "LIST" + }, + { + "key": "tsd.rtpublisher.enable", + "cl-option": "--rtplublisher", + "defaultValue": "false", + "description": "Whether or not to enable a real time publishing plugin. If true, you must supply a valid tsd.rtpublisher.plugin class name", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.rtpublisher.plugin", + "cl-option": "--rtplublisher-plugin", + "description": "The class name of a real time publishing plugin to instantiate. If tsd.rtpublisher.enable is set to false, this value is ignored. e.g. net.opentsdb.tsd.RabbitMQPublisher", + "help": "extended", + "meta": "CLASS" + }, + { + "key": "tsd.search.enable", + "cl-option": "--search", + "defaultValue": "false", + "description": "Whether or not to enable search functionality. If true, you must supply a valid tsd.search.plugin class name", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.search.plugin", + "cl-option": "--search-plugin", + "description": "The class name of a search plugin to instantiate. If tsd.search.enable is set to false, this value is ignored. e.g. net.opentsdb.search.ElasticSearch", + "help": "extended", + "meta": "CLASS" + }, + { + "key": "tsd.stats.canonical", + "cl-option": "--stats-canonical", + "defaultValue": "false", + "description": "Whether or not the FQDN should be returned with statistics requests. The default stats are returned with host= which is not guaranteed to perform a lookup and return the FQDN. Setting this to true will perform a name lookup and return the FQDN if found, otherwise it may return the IP. The stats output should be fqdn=", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.storage.compaction.flush_interval", + "cl-option": "--tsd-comp-flush-interval", + "defaultValue": "10", + "description": "How long, in seconds, to wait in between compaction queue flush calls", + "help": "extended", + "meta": "POSINT" + }, + { + "key": "tsd.storage.compaction.flush_speed", + "cl-option": "--tsd-comp-flush-speed", + "defaultValue": "2", + "description": "A multiplier used to determine how quickly to attempt flushing the compaction queue. E.g. a value of 2 means it will try to flush the entire queue within 30 minutes. A value of 1 would take an hour", + "help": "extended", + "meta": "POSINT" + }, + { + "key": "tsd.storage.compaction.max_concurrent_flushes", + "cl-option": "--tsd-comp-flush-maxconc", + "defaultValue": "10000", + "description": "The maximum number of compaction calls inflight to HBase at any given time", + "help": "extended", + "meta": "POSINT" + }, + { + "key": "tsd.storage.compaction.min_flush_threshold", + "cl-option": "--tsd-comp-flush-minflush", + "defaultValue": "100", + "description": "Size of the compaction queue that must be exceeded before flushing is triggered", + "help": "extended", + "meta": "POSINT" + }, + { + "key": "tsd.storage.enable_appends", + "cl-option": "--tsd-enable-appends", + "defaultValue": "false", + "description": "Whether or not to append data to columns when writing data points instead of creating new columns for each value. Avoids the need for compactions after each hour but can use more resources on HBase", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.storage.enable_compaction", + "cl-option": "--compaction", + "defaultValue": "true", + "description": "Whether or not to enable compactions", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.storage.fix_duplicates", + "cl-option": "--fixdups", + "defaultValue": "false", + "description": "Whether or not to accept the last written value when parsing data points with duplicate timestamps. When enabled in conjunction with compactions, a compacted column will be written with the latest data points", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.storage.flush_interval", + "cl-option": "--flush-interval", + "defaultValue": 1000, + "description": "How often, in milliseconds, to flush the data point storage write buffer", + "help": "default", + "meta": "GTZEROINT" + }, + { + "key": "tsd.storage.hbase.data_table", + "cl-option": "--table", + "defaultValue": "tsdb", + "description": "Name of the HBase table where data points are stored", + "help": "default", + "meta": "TABLE" + }, + { + "key": "tsd.storage.hbase.meta_table", + "cl-option": "--metatable", + "defaultValue": "tsdb-meta", + "description": "Name of the HBase table where meta data are stored", + "help": "extended", + "meta": "TABLE" + }, + { + "key": "tsd.storage.hbase.prefetch_meta", + "cl-option": "--tsd-prefetch-meta", + "defaultValue": "false", + "description": "Whether or not to prefetch the regions for the TSDB tables before starting the network interface. This can improve performance", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.storage.hbase.tree_table", + "cl-option": "--treetable", + "defaultValue": "tsdb-tree", + "description": "Name of the HBase table where tree data are stored", + "help": "extended", + "meta": "TABLE" + }, + { + "key": "tsd.storage.hbase.uid_table", + "cl-option": "--uidtable", + "defaultValue": "tsdb-uid", + "description": "Name of the HBase table where UID information is stored", + "help": "extended", + "meta": "TABLE" + }, + { + "key": "tsd.storage.hbase.zk_basedir", + "cl-option": "--zkbasedir", + "defaultValue": "/hbase", + "description": "Path under which is the znode for the -ROOT- region (default: /hbase)", + "help": "default", + "meta": "ZPATH" + }, + { + "key": "tsd.storage.hbase.zk_quorum", + "cl-option": "--zkquorum", + "defaultValue": "localhost", + "description": "A comma-separated list of ZooKeeper hosts to connect to, with or without port specifiers. e.g. 192.168.1.1:2181,192.168.1.2:2181 (default: localhost)", + "help": "default", + "meta": "SPEC" + }, + { + "key": "tsd.storage.repair_appends", + "cl-option": "--tsd-repair-appends", + "defaultValue": "false", + "description": "Whether or not to re-write appended data point columns at query time when the columns contain duplicate or out of order data", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.storage.max_tags", + "cl-option": "--tsd-maxtags", + "defaultValue": "8", + "description": "The maximum number of tags allowed per data point. NOTE Please be aware of the performance tradeoffs of overusing tags", + "help": "extended", + "meta": "POSINT" + }, + { + "key": "tsd.storage.salt.buckets", + "cl-option": "--tsd-salt-buckets", + "description": "The number of salt buckets used to distribute load across regions. NOTE Changing this value after writing data may cause TSUID based queries to fail", + "help": "extended", + "meta": "GTZEROINT" + }, + { + "key": "tsd.storage.salt.width", + "cl-option": "--tsd-salt-width", + "description": "The width, in bytes, of the salt prefix used to indicate which bucket a time series belongs in. A value of 0 means salting is disabled. WARNING Do not change after writing data to HBase or you will corrupt your tables and not be able to query any more", + "help": "extended", + "meta": "GTZEROINT" + }, + { + "key": "tsd.storage.uid.width.metric", + "cl-option": "--tsd-width-metric", + "defaultValue": "3", + "description": "The width, in bytes, of metric UIDs. WARNING Do not change after writing data to HBase or you will corrupt your tables and not be able to query any more", + "help": "extended", + "meta": "GTZEROINT" + }, + { + "key": "tsd.storage.uid.width.tagk", + "cl-option": "--tsd-width-tagk", + "defaultValue": "3", + "description": "The width, in bytes, of tag key UIDs. WARNING Do not change after writing data to HBase or you will corrupt your tables and not be able to query any more", + "help": "extended", + "meta": "GTZEROINT" + }, + { + "key": "tsd.storage.uid.width.tagv", + "cl-option": "--tsd-width-tagv", + "defaultValue": "3", + "description": "The width, in bytes, of tag value UIDs. WARNING Do not change after writing data to HBase or you will corrupt your tables and not be able to query any more", + "help": "extended", + "meta": "GTZEROINT" + }, + { + "key": "tsd.logback.file", + "cl-option": "--logback", + "description": "The file name that the tsd will log to using logback", + "help": "default", + "meta": "EDIRFILE" + }, + { + "key": "tsd.logback.rollpattern", + "cl-option": "--logback-roll", + "description": "The pattern specifying the rolling file pattern, inserted between the file name base and extension", + "defaultValue" : "_%d{yyyy-MM-dd}.%i", + "help": "default" + }, + { + "key": "tsd.logback.console", + "cl-option": "--logback-console", + "description": "If specified, logback will log to the configured file and keep logging to the console", + "defaultValue": "false", + "help": "default", + "meta": "BOOL" + }, + { + "key": "tsd.logback.config", + "cl-option": "--logback-config", + "description": "The file name of an external logback configuration", + "help": "default", + "meta": "EFILE" + }, + + { + "key": "tsd.process.pid.file", + "cl-option": "--pid-file", + "defaultValue": "${java.io.tmpdir}/.tsdb/opentsdb.pid", + "description": "The file to write the process PID to. Defaults to [${user.home}.tsdb/opentsdb.pid]", + "help": "default", + "meta": "FILE" + }, + { + "key": "help", + "cl-option": "--help", + "defaultValue": "", + "description": "Prints the default command line usage options, or the extended if 'extended' is passed as an arg", + "help": "default" + }, + { + "key": "tsd.core.config", + "cl-option": "--config", + "description": "The core config file overlayed on this default", + "help": "default", + "meta": "URLORFILE" + }, + { + "key": "tsd.ui.noexport", + "cl-option": "--no-uiexport", + "defaultValue": "false", + "description": "Skips the boot time export of static UI content to tsd.http.staticroot", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.process.pid.ignore.existing", + "cl-option": "--ignore-existing-pid", + "defaultValue": "false", + "description": "If true, ignores and overwrites an existing pid file on startup", + "help": "extended", + "meta": "BOOL" + }, + { + "key": "tsd.core.config.include", + "cl-option": "--include", + "description": "An additional config overlay useful when the --config file is fairly static", + "help": "extended", + "meta": "URLORFILE" + } + ], + "bindings": [ + "//importPackage(Packages.java.lang);", + "var processors = java.lang.Runtime.getRuntime().availableProcessors();", + "var maxbytes = java.lang.Runtime.getRuntime().maxMemory();" + ] +} diff --git a/fat-jar/test-logback.xml b/fat-jar/test-logback.xml new file mode 100644 index 0000000000..c0f172d9e9 --- /dev/null +++ b/fat-jar/test-logback.xml @@ -0,0 +1,45 @@ + + + + + + %d{ISO8601} %-5level [%thread] %logger{0}: %msg%n + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/opentsdb.spec.in b/opentsdb.spec.in index 779f5320bf..5943b7f66e 100644 --- a/opentsdb.spec.in +++ b/opentsdb.spec.in @@ -4,7 +4,7 @@ %define _sourcedir %(echo $PWD) Name: @PACKAGE@ -Version: @VERSION@ +Version: %(echo @VERSION@ | sed 's/-/_/g') Release: 1 Summary: A scalable, distributed Time Series Database Packager: @PACKAGE_BUGREPORT@ @@ -45,7 +45,7 @@ seconds). OpenTSDB will never delete or downsample data and can easily store billions of data points. %prep -%setup -q +%setup -q -n @PACKAGE@-@VERSION@ %build @@ -55,7 +55,9 @@ make %install rm -rf %{buildroot} make install DESTDIR=%{buildroot} -mkdir -p %{buildroot}/var/cache/opentsdb +mkdir -p %{buildroot}%{_localstatedir}/cache/opentsdb +mkdir -p %{buildroot}%{_localstatedir}/log/opentsdb +mkdir -p %{buildroot}%{_localstatedir}/tmp/opentsdb mkdir -p %{buildroot}%{_datarootdir}/opentsdb/plugins # TODO: Use alternatives to manage the init script and configuration. @@ -70,29 +72,44 @@ rm -rf %{buildroot} %attr(0755,root,root) %{_datarootdir}/opentsdb/plugins %attr(0755,root,root) %{_datarootdir}/opentsdb/tools/* %attr(0755,root,root) %{_datarootdir}/opentsdb/etc/init.d/opentsdb +%attr(0644,root,root) %{_datarootdir}/opentsdb/etc/systemd/system/opentsdb@.service %config %{_datarootdir}/opentsdb/etc/opentsdb/opentsdb.conf %config %{_datarootdir}/opentsdb/etc/opentsdb/logback.xml %doc %{_datarootdir}/opentsdb %{_bindir}/tsdb -%dir %{_localstatedir}/cache/opentsdb +%dir %attr(0755,opentsdb,opentsdb) %{_tmppath}/opentsdb +%dir %attr(0755,opentsdb,opentsdb) %{_localstatedir}/cache/opentsdb +%dir %attr(0755,opentsdb,opentsdb) %{_localstatedir}/log/opentsdb +%dir %attr(0755,opentsdb,opentsdb) %{_localstatedir}/tmp/opentsdb %changelog -%post +%pre +getent group opentsdb 2>/dev/null >/dev/null || /usr/sbin/groupadd -r opentsdb +getent passwd opentsbd 2>&1 > /dev/null || /usr/sbin/useradd -c "OpenTSDB" -s /sbin/nologin -g opentsdb -r -d %{_datarootdir}/opentsdb opentsdb 2> /dev/null || : +%post if [ $1 -eq 1 ]; then # we're installing the first version of this package ln -s %{_datarootdir}/opentsdb/etc/opentsdb /etc/opentsdb - ln -s %{_datarootdir}/opentsdb/etc/init.d/opentsdb /etc/init.d/opentsdb + if [ -d /run/systemd/system ]; then + ln -s %{_datarootdir}/opentsdb/etc/systemd/system/opentsdb.service /lib/systemd/system + ln -s %{_datarootdir}/opentsdb/etc/systemd/system/opentsdb@.service /lib/systemd/system + systemctl daemon-reload + else + ln -s %{_datarootdir}/opentsdb/etc/init.d/opentsdb /etc/init.d/opentsdb + fi fi exit 0 %postun - if [ $1 -eq 0 ]; then # we're removing last version of this package - rm -rf /etc/opentsdb - rm -rf /etc/init.d/opentsdb + [ -L /etc/opentsdb ] && rm -f /etc/opentsdb + [ -L /etc/init.d/opentsdb ] && rm -f /etc/init.d/opentsdb + [ -L /lib/systemd/system/opentsdb.service ] && rm -f /lib/systemd/system/opentsdb.service + [ -L /lib/systemd/system/opentsdb@.service ] && rm -f /lib/systemd/system/opentsdb@.service + [ -d /run/systemd/system ] && systemctl daemon-reload fi exit 0 diff --git a/pom.xml.in b/pom.xml.in index 5a5c7016eb..7c1eadee68 100644 --- a/pom.xml.in +++ b/pom.xml.in @@ -70,8 +70,8 @@ maven-compiler-plugin 2.5.1 - 1.6 - 1.6 + 1.8 + 1.8 -Xlint **/client/*.java @@ -93,8 +93,8 @@ build-aux/gen_build_data.sh - target/generated-sources/net/opentsdb/BuildData.java - net.opentsdb + target/generated-sources/net/opentsdb/tools/BuildData.java + net.opentsdb.tools BuildData @@ -126,11 +126,17 @@ net/opentsdb/tsd/DummyHttpSerializer.class -C target/test-classes + net/opentsdb/tsd/DummyHttpRpcPlugin.class + -C + target/test-classes net/opentsdb/tsd/DummyRpcPlugin.class -C target/test-classes net/opentsdb/tsd/DummyRTPublisher.class -C + target/test-classes + net/opentsdb/tsd/DummySEHPlugin.class + -C test META-INF/services/net.opentsdb.plugin.DummyPlugin -C @@ -141,10 +147,16 @@ META-INF/services/net.opentsdb.tsd.HttpSerializer -C test + META-INF/services/net.opentsdb.tsd.HttpRpcPlugin + -C + test META-INF/services/net.opentsdb.tsd.RpcPlugin -C test META-INF/services/net.opentsdb.tsd.RTPublisher + -C + test + META-INF/services/net.opentsdb.tsd.StorageExceptionHandler test-compile @@ -278,7 +290,7 @@ org.apache.maven.plugins maven-gpg-plugin - 1.4 + 1.5 sign-artifacts @@ -288,11 +300,55 @@ + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.3 + true - tsunanet@gmail.com + ossrh + https://oss.sonatype.org/ + false + + com.helger.maven + ph-javacc-maven-plugin + 2.8.2 + + + jjc + generate-sources + + javacc + + + 1.8 + true + net.opentsdb.query.expression.parser + ${basedir}/src/ + ${project.build.directory}/generated-sources/ + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.0.2 + + + + test-jar + + + + + @@ -334,30 +390,6 @@ @SUASYNC_VERSION@ - - org.apache.zookeeper - zookeeper - @ZOOKEEPER_VERSION@ - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - jline - jline - - - junit - junit - - - - org.slf4j slf4j-api @@ -365,9 +397,27 @@ - org.hbase - asynchbase - @ASYNCHBASE_VERSION@ + org.apache.commons + commons-math3 + @APACHE_MATH_VERSION@ + + + + org.apache.commons + commons-jexl + @JEXL_VERSION@ + + + + org.jgrapht + jgrapht-core + @JGRAPHT_VERSION@ + + + + com.esotericsoftware.kryo + kryo + 2.21.1 @@ -448,13 +498,97 @@ gwt-user @GWT_VERSION@ + + + net.opentsdb + opentsdb_gwt_theme + @GWT_THEME_VERSION@ + UTF-8 - + + + + + hbase + + @maven_profile_hbase@ + + + + + org.hbase + asynchbase + @ASYNCHBASE_VERSION@ + + + + org.apache.zookeeper + zookeeper + @ZOOKEEPER_VERSION@ + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + jline + jline + + + junit + junit + + + + + + + + + bigtable + + @maven_profile_bigtable@ + + + + + com.pythian.opentsdb + asyncbigtable + @ASYNCBIGTABLE_VERSION@ + jar-with-dependencies + + + + + + + + cassandra + + @maven_profile_cassandra@ + + + + + net.opentsdb + asynccassandra + @ASYNCCASSANDRA_VERSION@ + jar-with-dependencies + + + + + + org.sonatype.oss oss-parent diff --git a/screwdriver.yaml b/screwdriver.yaml new file mode 100644 index 0000000000..0f5a502801 --- /dev/null +++ b/screwdriver.yaml @@ -0,0 +1,10 @@ +shared: + image: maven + +jobs: + pr: + steps: + - run_arbitrary_script: apt-get update && apt-get install autoconf make python -y && ./build.sh pom.xml && mvn clean test --quiet + main: + steps: + - run_arbitrary_script: apt-get update && apt-get install autoconf make python -y && ./build.sh pom.xml && mvn clean test --quiet diff --git a/src/META-INF/MANIFEST.MF b/src/META-INF/MANIFEST.MF new file mode 100644 index 0000000000..348f1bdd38 --- /dev/null +++ b/src/META-INF/MANIFEST.MF @@ -0,0 +1 @@ +Manifest-Version: 1.0 \ No newline at end of file diff --git a/src/META-INF/services/net.opentsdb.uid.UniqueIdFilterPlugin b/src/META-INF/services/net.opentsdb.uid.UniqueIdFilterPlugin new file mode 100644 index 0000000000..010e95484c --- /dev/null +++ b/src/META-INF/services/net.opentsdb.uid.UniqueIdFilterPlugin @@ -0,0 +1 @@ +net.opentsdb.uid.UniqueIdWhitelistFilter \ No newline at end of file diff --git a/src/auth/AllowAllAuthenticatingAuthorizer.java b/src/auth/AllowAllAuthenticatingAuthorizer.java new file mode 100644 index 0000000000..e47d6d33fd --- /dev/null +++ b/src/auth/AllowAllAuthenticatingAuthorizer.java @@ -0,0 +1,221 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . + +package net.opentsdb.auth; + +import com.stumbleupon.async.Deferred; +import net.opentsdb.core.TSDB; +import net.opentsdb.core.TSQuery; +import net.opentsdb.query.pojo.Query; +import net.opentsdb.stats.StatsCollector; +import net.opentsdb.tsd.BadRequestException; +import org.jboss.netty.channel.Channel; +import org.jboss.netty.handler.codec.http.HttpRequest; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import java.util.concurrent.atomic.AtomicLong; + +/** + * The default authentication and authorization plugin. This plugin allows all users with no real authentication + * or authorization. By default, users are considered in the ADMINISTRATOR role and have all of the + * net.opentsdb.auth.Permissions + * + * @author jonathan.creasy + * @since 2.4.0 + */ +@SuppressWarnings("unused") +public class AllowAllAuthenticatingAuthorizer implements Authentication, Authorization { + private Roles roles; + + static AuthState accessDenied = new AuthState() { + Channel channel; + + @Override + public String getUser() { + return "guest"; + } + + @Override + public AuthStatus getStatus() { + return AuthStatus.FORBIDDEN; + } + + @Override + public String getMessage() { + return "Guest User forbidden by AllowAllAuthenticatingAuthorizer"; + } + + @Override + public Throwable getException() { + return null; + } + + @Override + public void setChannel(Channel channel) { + this.channel = channel; + } + + @Override + public byte[] getToken() { + return new byte[0]; + } + }; + static AuthState accessGranted = new AuthState() { + Channel channel; + + @Override + public String getUser() { + return "guest"; + } + + @Override + public AuthStatus getStatus() { + return AuthStatus.SUCCESS; + } + + @Override + public String getMessage() { + return "Guest User allowed by AllowAllAuthenticatingAuthorizer"; + } + + @Override + public Throwable getException() { + return null; + } + + @Override + public void setChannel(Channel channel) { + this.channel = channel; + } + + @Override + public byte[] getToken() { + return new byte[0]; + } + }; + + private static final AtomicLong queries_allowed = new AtomicLong(); + private static final AtomicLong queries_denied = new AtomicLong(); + private static final AtomicLong authentication_http_allowed = new AtomicLong(); + private static final AtomicLong authentication_http_denied = new AtomicLong(); + private static final AtomicLong authentication_telnet_allowed = new AtomicLong(); + private static final AtomicLong authentication_telnet_denied = new AtomicLong(); + private static final AtomicLong authorization_role_allowed = new AtomicLong(); + private static final AtomicLong authorization_permission_allowed = new AtomicLong(); + private static final AtomicLong authorization_role_denied = new AtomicLong(); + private static final AtomicLong authorization_permission_denied = new AtomicLong(); + + public Roles getRoles() { + return roles; + } + + public void setRoles(Roles roles) { + this.roles = roles; + } + + public AuthState getAccessDenied() { + return accessDenied; + } + + public AuthState getAccessGranted() { + return accessGranted; + } + + @Override + public void initialize(final TSDB tsdb) { + } + + @Override + public Deferred shutdown() { + return null; + } + + @Override + public String version() { + return "2.4.0"; + } + + @Override + public void collectStats(final StatsCollector collector) { + collector.record("authorization.queries.allowed", queries_allowed); + collector.record("authorization.queries.denied", queries_denied); + collector.record("authorization.allowed", authorization_role_allowed, "type=role"); + collector.record("authorization.denied", authorization_role_denied, "type=role"); + collector.record("authorization.allowed", authorization_permission_allowed, "type=permission"); + collector.record("authorization.denied", authorization_permission_denied, "type=permission"); + collector.record("authentication.succeeded", authentication_http_allowed, "type=http"); + collector.record("authentication.denied", authentication_http_denied, "type=http"); + collector.record("authentication.succeeded", authentication_telnet_allowed, "type=telnet"); + collector.record("authentication.denied", authentication_telnet_denied, "type=telnet"); + } + + @Override + public AuthState authenticateTelnet(final Channel channel, final String[] command) { + authentication_telnet_allowed.getAndIncrement(); + return accessGranted; + } + + @Override + public AuthState authenticateHTTP(final Channel channel, final HttpRequest req) { + authentication_http_allowed.getAndIncrement(); + return accessGranted; + } + + @Override + public Authorization authorization() { + return null; + } + + @Override + public boolean isReady(final TSDB tsdb, final Channel chan) { + if (tsdb.getAuth() != null && tsdb.getAuth().authorization() != null) { + if (chan.getAttachment() == null || !(chan.getAttachment() instanceof AuthState)) { + throw new BadRequestException(HttpResponseStatus.INTERNAL_SERVER_ERROR, + "Authentication was enabled but the authentication state for " + + "this channel was not set properly"); + } + return true; + } else { + return false; + } + } + + @Override + public AuthState hasPermission(final AuthState state, final Permissions permission) { + if (this.roles != null && this.roles.hasPermission(permission)) { + authorization_permission_allowed.getAndIncrement(); + return accessGranted; + } + authorization_permission_denied.getAndIncrement(); + return accessDenied; + } + + @Override + public AuthState allowQuery(final AuthState state, final TSQuery query) { + if (this.roles != null && this.roles.hasPermission(Permissions.HTTP_QUERY)) { + queries_allowed.getAndIncrement(); + return accessGranted; + } + queries_denied.getAndIncrement(); + return accessDenied; + } + + @Override + public AuthState allowQuery(final AuthState state, final Query query) { + if (this.roles != null && this.roles.hasPermission(Permissions.HTTP_QUERY)) { + queries_allowed.getAndIncrement(); + return accessGranted; + } + queries_denied.getAndIncrement(); + return accessDenied; + } +} \ No newline at end of file diff --git a/src/auth/AuthState.java b/src/auth/AuthState.java new file mode 100644 index 0000000000..4dee5fd0d4 --- /dev/null +++ b/src/auth/AuthState.java @@ -0,0 +1,83 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.auth; + +import org.jboss.netty.channel.Channel; + +/** + * An interface for use in authentication and authorization of calls. + *

+ * When used for successful authentication, this stage object will be attached + * to the Netty Channel via it's attachment method. It can then be read out of + * the channel and used for authorization calls. + * + * @since 2.4 + */ +public interface AuthState { + + /** + * A list of various authentication and authorization states. + */ + public enum AuthStatus { + SUCCESS, + UNAUTHORIZED, + FORBIDDEN, + REDIRECTED, + ERROR, + REVOKED + } + + /** + * Returns the user associated with this state as a string object. This should + * always return a non-null value. If authentication is enabled but anonymous + * users or unauthenticated users are allowed, return something like "anonymous". + * @return A non-null user ID. + */ + public String getUser(); + + /** + * The status associated with this authentication or authorization state. + * @return A non-null auth status enumerator value. + */ + public AuthStatus getStatus(); + + /** + * An optional message associated with the authentication or authorization + * attempt. If successful, this may be null. On an error or failure, this + * message should be populated. + * @return A useful message regarding the action taken. May be null. + */ + public String getMessage(); + + /** + * An optional exception if an error occurred during the operation. + * @return An optional exception; may be null. + */ + public Throwable getException(); + + /** + * Sets the channel associated with this state when used for authentication + * and attached to a channel. Useful for associating connection information + * (IP and port) with the user. + * @param channel A non-null channel. + */ + public void setChannel(final Channel channel); + + /** + * An optional token to use with an authentication system for validating that + * the user key is still valid. + * @return A byte array encoded depending on the authentication plugin. May + * be null. + */ + public byte[] getToken(); +} diff --git a/src/auth/Authentication.java b/src/auth/Authentication.java new file mode 100644 index 0000000000..acf16fe050 --- /dev/null +++ b/src/auth/Authentication.java @@ -0,0 +1,128 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.auth; + +import net.opentsdb.core.TSDB; +import net.opentsdb.stats.StatsCollector; +import com.stumbleupon.async.Deferred; + +import org.jboss.netty.channel.Channel; +import org.jboss.netty.handler.codec.http.HttpRequest; + +/** + * A plugin interface for performing authentication for OpenTSDB API access. + * The plugin is embedded within the Netty pipeline and intercepts requests + * on new channels. Once a channel is authenticated successfully, the plugin is + * removed from the pipeline so further calls on that channel are not evaluated. + *

+ * An AuthState object is attached to the channel for evaluation later in the + * pipeline. This state cannot be changed but cane be replaced. + *

+ * The plugin also includes an acessor to an Authorization plugin to allow or + * disallow operations per user. + * + * @since 2.4 + */ +public interface Authentication { + + /** + * Called by TSDB to initialize the plugin + * Implementations are responsible for setting up any IO they need as well + * as starting any required background threads. + * Note: Implementations should throw exceptions if they can't start + * up properly. The TSD will then shutdown so the operator can fix the + * problem. Please use IllegalArgumentException for configuration issues. + * + * @param tsdb The parent TSDB object + * @throws IllegalArgumentException if required configuration parameters are + * missing + * @throws RuntimeException if something else goes wrong + */ + public abstract void initialize(final TSDB tsdb); + + /** + * Called to gracefully shutdown the plugin. Implementations should close + * any IO they have open + * + * @return A deferred object that indicates the completion of the request. + * The {@link Object} has not special meaning and can be {@code null} + * (think of it as {@code Deferred}). + */ + public abstract Deferred shutdown(); + + /** + * Should return the version of this plugin in the format: + * MAJOR.MINOR.MAINT, e.g. 2.0.1. The MAJOR version should match the major + * version of OpenTSDB the plugin is meant to work with. + * + * @return A version string used to log the loaded version + */ + public abstract String version(); + + /** + * Called by the TSD when a request for statistics collection has come in. The + * implementation may provide one or more statistics. If no statistics are + * available for the implementation, simply stub the method. + * + * @param collector The collector used for emitting statistics + */ + public abstract void collectStats(final StatsCollector collector); + + /** + * Authenticate Telnet connections, provides the first line of the incoming + * connection. + *

+ * NOTE: This method should not throw exceptions, rather return a state object + * with the AuthStatus.ERROR status. + * + * @param channel A non-null Netty channel to associate with the request. + * @param command A non-null list of "words" from a Telnet style command + * (strings or numbers separated by spaces) + * @return A non-null AuthState object with a valid AuthStatus to evaluate for + * a successful or unsuccessful authentication. + */ + public abstract AuthState authenticateTelnet(final Channel channel, + final String[] command); + + /** + * Authenticate HTTP connections, provides the HTTPRequest object for the + * incoming connection. + *

+ * NOTE: This method should not throw exceptions, rather return a state object + * with the AuthStatus.ERROR status. + * + * @param channel A non-null Netty channel to associate with the request. + * @param req A non-null HTTP request. + * @return A non-null AuthState object with a valid AuthStatus to evaluate for + * a successful or unsuccessful authentication. + */ + public abstract AuthState authenticateHTTP(final Channel channel, + final HttpRequest req); + + /** + * An optional authorization object. If authorization is not enabled, this + * call may return null. + * + * @return An authorization object or null; + */ + public abstract Authorization authorization(); + + /** + * Function to determine if the authorization and authentication system is valid for this channel. + * + * @param tsdb the current tsdb + * @param chan the current channel + * @return returns true if the authc/authz state is valid for the provided channel + */ + public boolean isReady(final TSDB tsdb, final Channel chan); +} \ No newline at end of file diff --git a/src/auth/AuthenticationChannelHandler.java b/src/auth/AuthenticationChannelHandler.java new file mode 100644 index 0000000000..1ce19772d5 --- /dev/null +++ b/src/auth/AuthenticationChannelHandler.java @@ -0,0 +1,161 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . + +package net.opentsdb.auth; + +import net.opentsdb.auth.AuthState.AuthStatus; +import net.opentsdb.core.Const; +import net.opentsdb.core.TSDB; +import org.jboss.netty.channel.ChannelHandlerContext; +import org.jboss.netty.channel.MessageEvent; +import org.jboss.netty.channel.SimpleChannelUpstreamHandler; +import org.jboss.netty.handler.codec.http.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Strings; + +import org.jboss.netty.channel.ExceptionEvent; + +import static org.jboss.netty.handler.codec.http.HttpVersion.HTTP_1_1; + +import org.jboss.netty.buffer.ChannelBuffers; + +/** + * This is a simple authentication handler that intercepts the initial calls + * for a new channel and attempts to authenticate the caller. + *

+ * On a successful authentication, the authentication state is attached to the + * Netty channel for other components to use and the handler is removed so that + * subsequent calls across the channel do not have to go through authentication. + * At any time, the pipeline can check the auth state to determine if actions + * are allowed or if the state should be revoked (e.g. token timing out) + *

+ * On a failed auth, an error is returned to the user (message via Telnet or + * 401, 403 or 500 for HTTP). The channel is left open so callers can try again + * but if an unknown error occurs then the channel is closed. + * + * @since 2.4 + */ +public class AuthenticationChannelHandler extends SimpleChannelUpstreamHandler { + private static final Logger LOG = LoggerFactory.getLogger( + AuthenticationChannelHandler.class); + + public static final String TELNET_AUTH_FAILURE = "AUTH_FAIL\r\n"; + public static final String TELNET_AUTH_SUCCESS = "AUTH_SUCCESS\r\n"; + + /** The authentication object, pulled from TSDB. */ + private final Authentication authentication; + + /** + * Default ctor. + * @param tsdb Non-null TSDB object from which to fetch the auth plugin. + * @throws IllegalArgumentException if the TSDB object is null or the auth + * object the TSDB contains is null. + */ + public AuthenticationChannelHandler(final TSDB tsdb) { + if (tsdb == null) { + throw new IllegalArgumentException("TSDB object cannot be null"); + } + if (tsdb.getAuth() == null) { + throw new IllegalArgumentException("Attempted to instantiate an " + + "authentication handler but it was not configured in TSDB."); + } + authentication = tsdb.getAuth(); + LOG.info("Set up AuthenticationChannelHandler: " + authentication.getClass()); + } + + @Override + public void exceptionCaught(final ChannelHandlerContext ctx, + final ExceptionEvent e) { + LOG.error("Unexpected exception in AuthenticationChannelHandler for " + + "channel: " + e.getChannel(), e.getCause()); + e.getChannel().close(); + } + + @Override + public void messageReceived(final ChannelHandlerContext ctx, + final MessageEvent authEvent) { + try { + final Object authCommand = authEvent.getMessage(); + + // Telnet Auth + if (authCommand instanceof String[]) { + if (LOG.isDebugEnabled()) { + LOG.debug("Authenticating Telnet command from channel: " + + authEvent.getChannel()); + } + String auth_response = TELNET_AUTH_FAILURE; + final AuthState state = authentication.authenticateTelnet( + authEvent.getChannel(), (String[]) authCommand); + + if (state.getStatus() == AuthStatus.SUCCESS) { + auth_response = TELNET_AUTH_SUCCESS; + ctx.getPipeline().remove(this); + authEvent.getChannel().setAttachment(state); + state.setChannel(authEvent.getChannel()); + } + authEvent.getChannel().write(auth_response); + + // HTTTP Auth + } else if (authCommand instanceof HttpRequest) { + if (LOG.isDebugEnabled()) { + LOG.debug("Authenticating HTTP request from channel: " + + authEvent.getChannel()); + } + HttpResponseStatus status; + final AuthState state = authentication.authenticateHTTP( + authEvent.getChannel(), (HttpRequest) authCommand); + if (state.getStatus() == AuthStatus.SUCCESS) { + ctx.getPipeline().remove(this); + authEvent.getChannel().setAttachment(state); + state.setChannel(authEvent.getChannel()); + // pass it down! + super.messageReceived(ctx, authEvent); + } else if (state.getStatus() == AuthStatus.REDIRECTED) { + // do nothing here as the plugin sent the redirect answer. We want to + // keep auth inline so the next call can process the authentication. + } else { + switch (state.getStatus()) { + case UNAUTHORIZED: + status = HttpResponseStatus.UNAUTHORIZED; + break; + case FORBIDDEN: + status = HttpResponseStatus.FORBIDDEN; + break; + default: + status = HttpResponseStatus.INTERNAL_SERVER_ERROR; + break; + } + + final HttpResponse response = new DefaultHttpResponse(HTTP_1_1, status); + if (!Strings.isNullOrEmpty(state.getMessage())) { + // TODO - JSONify or something + response.setContent(ChannelBuffers.copiedBuffer( + state.getMessage(), Const.UTF8_CHARSET)); + } + authEvent.getChannel().write(response); + } + // Unknown Authentication. Log and close the connection. + } else { + LOG.error("Unexpected message type " + authCommand.getClass() + ": " + + authCommand + " from channel: " + authEvent.getChannel()); + authEvent.getChannel().close(); + } + } catch (Throwable t) { + LOG.error("Unexpected exception caught while serving channel: " + + authEvent.getChannel(), t); + authEvent.getChannel().close(); + } + } +} \ No newline at end of file diff --git a/src/auth/Authorization.java b/src/auth/Authorization.java new file mode 100644 index 0000000000..b6ad5e2067 --- /dev/null +++ b/src/auth/Authorization.java @@ -0,0 +1,104 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.auth; + +import com.stumbleupon.async.Deferred; + +import net.opentsdb.core.TSDB; +import net.opentsdb.core.TSQuery; +import net.opentsdb.query.pojo.Query; +import net.opentsdb.stats.StatsCollector; + +import java.util.EnumSet; + +/** + * A plugin interface for authorization calls, allowing or disallowing operations + * in OpenTSDB. + * + * @since 2.4 + */ +public interface Authorization { + + /** + * Called by TSDB to initialize the plugin + * Implementations are responsible for setting up any IO they need as well + * as starting any required background threads. + * Note: Implementations should throw exceptions if they can't start + * up properly. The TSD will then shutdown so the operator can fix the + * problem. Please use IllegalArgumentException for configuration issues. + * @param tsdb The parent TSDB object + * @throws IllegalArgumentException if required configuration parameters are + * missing + * @throws RuntimeException if something else goes wrong + */ + public abstract void initialize(final TSDB tsdb); + + /** + * Called to gracefully shutdown the plugin. Implementations should close + * any IO they have open + * @return A deferred object that indicates the completion of the request. + * The {@link Object} has not special meaning and can be {@code null} + * (think of it as {@code Deferred}). + */ + public abstract Deferred shutdown(); + + /** + * Should return the version of this plugin in the format: + * MAJOR.MINOR.MAINT, e.g. 2.0.1. The MAJOR version should match the major + * version of OpenTSDB the plugin is meant to work with. + * @return A version string used to log the loaded version + */ + public abstract String version(); + + /** + * Called by the TSD when a request for statistics collection has come in. The + * implementation may provide one or more statistics. If no statistics are + * available for the implementation, simply stub the method. + * @param collector The collector used for emitting statistics + */ + public abstract void collectStats(final StatsCollector collector); + + /** + * Determines if the user has a specified permission + * @param state + * @param permission + * @return + */ + public abstract AuthState hasPermission(final AuthState state, final Permissions permission); + + /** + * Determines if the user is allowed to execute the given query. + * The returned state contains a status code regarding whether or not the query + * is allowed. If the query IS allowed, the same user state passed as an + * argument maybe returned. + * @param state A non-null auth state with the user and AuthStatus.SUCCESS. + * @param query A non-null query. + * @return An AuthState object with a valid AuthStatus to evaluate for + * permission. + */ + public abstract AuthState allowQuery(final AuthState state, + final TSQuery query); + + /** + * Determines if the user is allowed to execute the given query. + * The returned state contains a status code regarding whether or not the query + * is allowed. If the query IS allowed, the same user state passed as an + * argument maybe returned. + * @param state A non-null auth state with the user and AuthStatus.SUCCESS. + * @param query A non-null query. + * @return An AuthState object with a valid AuthStatus to evaluate for + * permission. + */ + public abstract AuthState allowQuery(final AuthState state, + final Query query); +} diff --git a/src/auth/Permissions.java b/src/auth/Permissions.java new file mode 100644 index 0000000000..edee4acce0 --- /dev/null +++ b/src/auth/Permissions.java @@ -0,0 +1,28 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . + +package net.opentsdb.auth; + +/** + * The Permissions used within the OpenTSDB Code. + * Any authorization plugins need to be able to respond to inquiries on these permissions. Plugins and other + * third-party code may implement additional permissions. + * + * @author jonathan.creasy + * @since 2.4.0 + * + */ +public enum Permissions { + TELNET_PUT, HTTP_PUT, HTTP_QUERY, + CREATE_TAGK,CREATE_TAGV, CREATE_METRIC; +} diff --git a/src/auth/Roles.java b/src/auth/Roles.java new file mode 100644 index 0000000000..22e35cfbcd --- /dev/null +++ b/src/auth/Roles.java @@ -0,0 +1,56 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . + +package net.opentsdb.auth; + +import java.util.*; + +/** + * Suggested standard Roles for OpenTSDB Users + * + * @author jonathan.creasy + * @since 2.4.0 + */ +@SuppressWarnings({"WeakerAccess", "unused"}) +public class Roles { + final static EnumSet ADMINISTRATOR = EnumSet.allOf(Permissions.class); + final static EnumSet PUTONLY = EnumSet.of(Permissions.HTTP_PUT, Permissions.TELNET_PUT); + final static EnumSet WRITER = EnumSet.of(Permissions.HTTP_PUT, Permissions.TELNET_PUT, Permissions.CREATE_TAGV); + final static EnumSet READER = EnumSet.of(Permissions.HTTP_QUERY); + final static EnumSet CREATOR = EnumSet.of(Permissions.CREATE_METRIC, Permissions.CREATE_TAGK, Permissions.CREATE_TAGV); + final static EnumSet GUEST = EnumSet.noneOf(Permissions.class); + + @SuppressWarnings("Convert2Diamond") + private final Set> grantedPermissions = new HashSet>(); + + public Roles() { + this.grantedPermissions.add(GUEST); + } + + public Roles(final EnumSet permissions) { + this.grantedPermissions.add(permissions); + } + + public void grantPermissions(final EnumSet permissions) { + grantedPermissions.add(permissions); + } + + public Boolean hasPermission(final Permissions permission) { + for (EnumSet permissions : this.grantedPermissions) { + if (permissions.contains(permission)) { + return true; + } + } + return false; + } +} diff --git a/src/core/AbstractQuery.java b/src/core/AbstractQuery.java new file mode 100644 index 0000000000..5a37c50f4a --- /dev/null +++ b/src/core/AbstractQuery.java @@ -0,0 +1,66 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2010-2012 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import org.hbase.async.HBaseException; + +public abstract class AbstractQuery implements Query { + /** + * Runs this query. + * + * @return The data points matched by this query. + *

+ * Each element in the non-{@code null} but possibly empty array returned + * corresponds to one time series for which some data points have been + * matched by the query. + * @throws HBaseException if there was a problem communicating with HBase to + * perform the search. + */ + @Override + public DataPoints[] run() throws HBaseException { + try { + return runAsync().joinUninterruptibly(); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Should never be here", e); + } + } + + /** + * Runs this query. + * + * @return The data points matched by this query and applied with percentile calculation + *

+ * Each element in the non-{@code null} but possibly empty array returned + * corresponds to one time series for which some data points have been + * matched by the query. + * @throws HBaseException if there was a problem communicating with HBase to + * perform the search. + * @throws IllegalStateException if the query is not a histogram query + */ + @Override + public DataPoints[] runHistogram() throws HBaseException { + if (!isHistogramQuery()) { + throw new RuntimeException("Should never be here"); + } + + try { + return runHistogramAsync().joinUninterruptibly(); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Should never be here", e); + } + } +} diff --git a/src/core/AbstractSpanGroup.java b/src/core/AbstractSpanGroup.java new file mode 100644 index 0000000000..f9605a6b05 --- /dev/null +++ b/src/core/AbstractSpanGroup.java @@ -0,0 +1,70 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2010-2012 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import java.util.*; + +import org.hbase.async.Bytes; +import org.hbase.async.Bytes.ByteMap; + +import com.stumbleupon.async.Callback; +import com.stumbleupon.async.Deferred; + +import net.opentsdb.meta.Annotation; +import net.opentsdb.rollup.RollupQuery; + +/** + * Groups multiple spans together and offers a dynamic "view" on them. + *

+ * This is used for queries to the TSDB, where we might group multiple + * {@link Span}s that are for the same time series but different tags + * together. We need to "hide" data points that are outside of the + * time period of the query and do on-the-fly aggregation of the data + * points coming from the different Spans, using an {@link Aggregator}. + * Since not all the Spans will have their data points at exactly the + * same time, we also do on-the-fly linear interpolation. If needed, + * this view can also return the rate of change instead of the actual + * data points. + *

+ * This is one of the rare (if not the only) implementations of + * {@link DataPoints} for which {@link #getTags} can potentially return + * an empty map. + *

+ * The implementation can also dynamically downsample the data when a + * sampling interval a downsampling function (in the form of an + * {@link Aggregator}) are given. This is done by using a special + * iterator when using the {@link Span.DownsamplingIterator}. + */ +abstract class AbstractSpanGroup implements DataPoints { + /** + * Finds the {@code i}th data point of this group in {@code O(n)}. + * Where {@code n} is the number of data points in this group. + */ + protected DataPoint getDataPoint(int i) { + if (i < 0) { + throw new IndexOutOfBoundsException("negative index: " + i); + } + final int saved_i = i; + final SeekableView it = iterator(); + DataPoint dp = null; + while (it.hasNext() && i >= 0) { + dp = it.next(); + i--; + } + if (i != -1 || dp == null) { + throw new IndexOutOfBoundsException("index " + saved_i + + " too large (it's >= " + size() + ") for " + this); + } + return dp; + } +} diff --git a/src/core/AggregationIterator.java b/src/core/AggregationIterator.java index 2c6f6e3433..7e2210c721 100644 --- a/src/core/AggregationIterator.java +++ b/src/core/AggregationIterator.java @@ -19,6 +19,8 @@ import com.google.common.annotations.VisibleForTesting; import net.opentsdb.core.Aggregators.Interpolation; +import net.opentsdb.rollup.RollupQuery; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -115,7 +117,7 @@ * to a special, really large value (too large to be a valid timestamp). *

*/ -final class AggregationIterator implements SeekableView, DataPoint, +public class AggregationIterator implements SeekableView, DataPoint, Aggregator.Longs, Aggregator.Doubles { private static final Logger LOG = @@ -129,7 +131,7 @@ final class AggregationIterator implements SeekableView, DataPoint, * possibly store, provided that the most significant bit is reserved by * FLAG_FLOAT. */ - private static final long TIME_MASK = 0x7FFFFFFFFFFFFFFFL; + protected static final long TIME_MASK = 0x7FFFFFFFFFFFFFFFL; /** Aggregator to use to aggregate data points from different Spans. */ private final Aggregator aggregator; @@ -148,13 +150,13 @@ final class AggregationIterator implements SeekableView, DataPoint, * Once we reach the end of a Span, we'll null out its iterator from this * array. */ - private final SeekableView[] iterators; + protected final SeekableView[] iterators; /** Start time (UNIX timestamp in seconds or ms) on 32 bits ("unsigned" int). */ - private final long start_time; + protected final long start_time; /** End time (UNIX timestamp in seconds or ms) on 32 bits ("unsigned" int). */ - private final long end_time; + protected final long end_time; /** * The current and previous timestamps for the data points being used. @@ -164,7 +166,7 @@ final class AggregationIterator implements SeekableView, DataPoint, *

  • No: for {@code iterators[i]} the timestamp of the current data * point is {@code timestamps[i]} and the timestamp of the next data * point is {@code timestamps[iterators.length + i]}.
    • - * + * *

    * Each timestamp can have the {@code FLAG_FLOAT} applied so it's important * to use the {@code TIME_MASK} when getting the actual timestamp value @@ -178,7 +180,7 @@ final class AggregationIterator implements SeekableView, DataPoint, * linear interpolation anymore. * */ - private final long[] timestamps; // 32 bit unsigned + flag + protected final long[] timestamps; // 32 bit unsigned + flag /** * The current and next values for the data points being used. @@ -186,7 +188,7 @@ final class AggregationIterator implements SeekableView, DataPoint, * This array is also used to store floating point values, in which case * their binary representation just happens to be stored in a {@code long}. */ - private final long[] values; + protected final long[] values; /** The index in {@link #iterators} of the current Span being used. */ private int current; @@ -222,6 +224,42 @@ public static AggregationIterator create(final List spans, final long sample_interval_ms, final boolean rate, final RateOptions rate_options) { + return create(spans, start_time, end_time, aggregator, method, downsampler, + sample_interval_ms, rate, rate_options, null); + } + + /** + * Creates a new iterator for a {@link SpanGroup}. + * @param spans Spans in a group. + * @param start_time Any data point strictly before this timestamp will be + * ignored. + * @param end_time Any data point strictly after this timestamp will be + * ignored. + * @param aggregator The aggregation function to use. + * @param method Interpolation method to use when aggregating time series + * @param downsampler Aggregation function to use to group data points + * within an interval. + * @param sample_interval_ms Number of milliseconds wanted between each data + * point. + * @param rate If {@code true}, the rate of the series will be used instead + * of the actual values. + * @param rate_options Specifies the optional additional rate calculation + * options. + * @param fill_policy Policy specifying whether to interpolate or to fill + * missing intervals with special values. + * @return An {@link AggregationIterator} object. + * @since 2.2 + */ + public static AggregationIterator create(final List spans, + final long start_time, + final long end_time, + final Aggregator aggregator, + final Interpolation method, + final Aggregator downsampler, + final long sample_interval_ms, + final boolean rate, + final RateOptions rate_options, + final FillPolicy fill_policy) { final int size = spans.size(); final SeekableView[] iterators = new SeekableView[size]; for (int i = 0; i < size; i++) { @@ -229,7 +267,8 @@ public static AggregationIterator create(final List spans, if (downsampler == null) { it = spans.get(i).spanIterator(); } else { - it = spans.get(i).downsampler(sample_interval_ms, downsampler); + it = spans.get(i).downsampler(start_time, end_time, sample_interval_ms, + downsampler, fill_policy); } if (rate) { it = new RateSpan(it, rate_options); @@ -240,6 +279,106 @@ public static AggregationIterator create(final List spans, method, rate); } + /** + * Creates a new iterator for a {@link SpanGroup}. + * @param spans Spans in a group. + * @param start_time Any data point strictly before this timestamp will be + * ignored. + * @param end_time Any data point strictly after this timestamp will be + * ignored. + * @param aggregator The aggregation function to use. + * @param method Interpolation method to use when aggregating time series + * @param downsampler The downsampling specifier to use (cannot be null) + * @param query_start Start of the actual query + * @param query_end End of the actual query + * @param rate If {@code true}, the rate of the series will be used instead + * of the actual values. + * @param rate_options Specifies the optional additional rate calculation + * options. + * @return an AggregationIterator + * @since 2.3 + */ + public static AggregationIterator create(final List spans, + final long start_time, + final long end_time, + final Aggregator aggregator, + final Interpolation method, + final DownsamplingSpecification downsampler, + final long query_start, + final long query_end, + final boolean rate, + final RateOptions rate_options) { + final int size = spans.size(); + final SeekableView[] iterators = new SeekableView[size]; + for (int i = 0; i < size; i++) { + SeekableView it; + if (downsampler == null || + downsampler == DownsamplingSpecification.NO_DOWNSAMPLER) { + it = spans.get(i).spanIterator(); + } else { + it = spans.get(i).downsampler(start_time, end_time, downsampler, + query_start, query_end); + } + if (rate) { + it = new RateSpan(it, rate_options); + } + iterators[i] = it; + } + return new AggregationIterator(iterators, start_time, end_time, aggregator, + method, rate); + } + + /** + * Creates a new iterator for a {@link SpanGroup}. + * @param spans Spans in a group. + * @param start_time Any data point strictly before this timestamp will be + * ignored. + * @param end_time Any data point strictly after this timestamp will be + * ignored. + * @param aggregator The aggregation function to use. + * @param method Interpolation method to use when aggregating time series + * @param downsampler The downsampling specifier to use (cannot be null) + * @param query_start Start of the actual query + * @param query_end End of the actual query + * @param rate If {@code true}, the rate of the series will be used instead + * of the actual values. + * @param rate_options Specifies the optional additional rate calculation + * options. + * @param rollup_query An optional rollup query. + * @return an AggregationIterator + * @since 2.4 + */ + public static AggregationIterator create(final List spans, + final long start_time, + final long end_time, + final Aggregator aggregator, + final Interpolation method, + final DownsamplingSpecification downsampler, + final long query_start, + final long query_end, + final boolean rate, + final RateOptions rate_options, + final RollupQuery rollup_query) { + final int size = spans.size(); + final SeekableView[] iterators = new SeekableView[size]; + for (int i = 0; i < size; i++) { + SeekableView it; + if (downsampler == null || + downsampler == DownsamplingSpecification.NO_DOWNSAMPLER) { + it = spans.get(i).spanIterator(); + } else { + it = spans.get(i).downsampler(start_time, end_time, downsampler, + query_start, query_end, rollup_query); + } + if (rate) { + it = new RateSpan(it, rate_options); + } + iterators[i] = it; + } + return new AggregationIterator(iterators, start_time, end_time, aggregator, + method, rate); + } + /** * Creates an aggregation iterator for a group of data point iterators. * @param iterators An array of Seekable views of spans in a group. Ignored @@ -253,7 +392,7 @@ public static AggregationIterator create(final List spans, * @param rate If {@code true}, the rate of the series will be used instead * of the actual values. */ - private AggregationIterator(final SeekableView[] iterators, + public AggregationIterator(final SeekableView[] iterators, final long start_time, final long end_time, final Aggregator aggregator, @@ -275,30 +414,36 @@ private AggregationIterator(final SeekableView[] iterators, for (int i = 0; i < size; i++) { SeekableView it = iterators[i]; it.seek(start_time); - final DataPoint dp; - try { - dp = it.next(); - } catch (NoSuchElementException e) { - // It should be rare but could happen after downsampling when - // we throw away some data points at the beginning after aligning - // start time by downsmpling interval and there are no data points - // left for the current span. + DataPoint dp; + if (!it.hasNext()) { ++num_empty_spans; endReached(i); continue; } + dp = it.next(); //LOG.debug("Creating iterator #" + i); if (dp.timestamp() >= start_time) { //LOG.debug("First DP in range for #" + i + ": " // + dp.timestamp() + " >= " + start_time); putDataPoint(size + i, dp); } else { - if (LOG.isDebugEnabled()) { - LOG.debug(String.format("No DP in range for #%d: %d < %d", i, - dp.timestamp(), start_time)); + // if there is data, advance to the start time if applicable. + while (dp != null && dp.timestamp() < start_time) { + if (it.hasNext()) { + dp = it.next(); + } else { + dp = null; + } } - endReached(i); - continue; + if (dp == null) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("No DP in range for #%d: start time %d", i, + start_time)); + } + endReached(i); + continue; + } + putDataPoint(size + i, dp); } if (rate) { // The first rate against the time zero should be populated @@ -492,8 +637,8 @@ public double doubleValue() { pos = -1; final double value = aggregator.runDouble(this); //LOG.debug("aggregator returned " + value); - if (value != value || Double.isInfinite(value)) { - throw new IllegalStateException("Got NaN or Infinity: " + if (Double.isInfinite(value)) { + throw new IllegalStateException("Got Infinity: " + value + " in this " + this); } return value; @@ -572,8 +717,11 @@ public long nextLongValue() { case MIN: r = Long.MIN_VALUE; break; + case PREV: + r = y0; + break; default: - throw new IllegalDataException("Invalid interploation somehow??"); + throw new IllegalDataException("Invalid interpolation somehow??"); } return r; } @@ -635,7 +783,10 @@ public double nextDoubleValue() { r = Double.MAX_VALUE; break; case MIN: - r = Double.MIN_VALUE; + r = -Double.MAX_VALUE; + break; + case PREV: + r = y0; break; default: throw new IllegalDataException("Invalid interploation somehow??"); @@ -686,4 +837,10 @@ static AggregationIterator createForTesting(final SeekableView[] iterators, return new AggregationIterator(iterators, start_time, end_time, aggregator, method, rate); } + + @Override + public long valueCount() { + // TODO don't know if this is right + return values.length; + } } diff --git a/src/core/Aggregator.java b/src/core/Aggregator.java index bb2c1124ef..0b6a1a7818 100644 --- a/src/core/Aggregator.java +++ b/src/core/Aggregator.java @@ -23,8 +23,24 @@ * sequence of {@link Longs Longs} or {@link Doubles Doubles} and return an * aggregated value. */ -public interface Aggregator { +public abstract class Aggregator { + + /** Interpolation method this aggregator uses across time series */ + private final Interpolation interpolation_method; + + /** String name of the aggregator */ + private final String name; + /** + * Create a new instance of this class. + * @param interpolationMethod The interpolation method to use. + * @param name The name of this aggregator. + */ + protected Aggregator(final Interpolation interpolationMethod, final String name) { + this.interpolation_method = interpolationMethod; + this.name = name; + } + /** * A sequence of {@code long}s. *

    @@ -76,19 +92,26 @@ public interface Doubles { * @param values The sequence to aggregate. * @return The aggregated value. */ - long runLong(Longs values); + public abstract long runLong(Longs values); /** * Aggregates a sequence of {@code double}s. * @param values The sequence to aggregate. * @return The aggregated value. */ - double runDouble(Doubles values); + public abstract double runDouble(Doubles values); /** * Returns the interpolation method to use when working with data points * across time series. * @return The interpolation method to use */ - Interpolation interpolationMethod(); + Interpolation interpolationMethod() { + return interpolation_method; + } + + @Override + public String toString() { + return name; + } } diff --git a/src/core/Aggregators.java b/src/core/Aggregators.java index c387410f15..531f0121c9 100644 --- a/src/core/Aggregators.java +++ b/src/core/Aggregators.java @@ -1,5 +1,5 @@ // This file is part of OpenTSDB. -// Copyright (C) 2010-2012 The OpenTSDB Authors. +// Copyright (C) 2010-2016 The OpenTSDB Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU Lesser General Public License as published by @@ -12,9 +12,20 @@ // see . package net.opentsdb.core; +import java.util.Collections; import java.util.HashMap; import java.util.NoSuchElementException; import java.util.Set; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; + +import org.apache.commons.math3.stat.descriptive.rank.Percentile; +import org.apache.commons.math3.stat.descriptive.rank.Percentile.EstimationType; +import org.apache.commons.math3.util.ResizableDoubleArray; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; /** * Utility class that provides common, generally useful aggregators. @@ -28,13 +39,21 @@ public enum Interpolation { LERP, /* Regular linear interpolation */ ZIM, /* Returns 0 when a data point is missing */ MAX, /* Returns the .MaxValue when a data point is missing */ - MIN /* Returns the .MinValue when a data point is missing */ + MIN, /* Returns the .MinValue when a data point is missing */ + PREV /* Returns the previous value stored, when a data point is missing */ } /** Aggregator that sums up all the data points. */ public static final Aggregator SUM = new Sum( Interpolation.LERP, "sum"); + /** + * Aggregator that sums up all the data points,and uses interpolation where + * previous value is used for data point missing. + */ + public static final Aggregator PFSUM= new Sum( + Interpolation.PREV, "pfsum"); + /** Aggregator that returns the minimum data point. */ public static final Aggregator MIN = new Min( Interpolation.LERP, "min"); @@ -47,38 +66,140 @@ public enum Interpolation { public static final Aggregator AVG = new Avg( Interpolation.LERP, "avg"); + /** Aggregator that returns the emedian of the data points. */ + public static final Aggregator MEDIAN = new Median(Interpolation.LERP, + "median"); + + /** Aggregator that skips aggregation/interpolation and/or downsampling. */ + public static final Aggregator NONE = new None(Interpolation.ZIM, "raw"); + + /** Return the product of two time series + * @since 2.3 */ + public static final Aggregator MULTIPLY = new Multiply( + Interpolation.LERP, "multiply"); + /** Aggregator that returns the Standard Deviation of the data points. */ public static final Aggregator DEV = new StdDev( Interpolation.LERP, "dev"); - /** Sums data points but will cause the SpanGroup to return a 0 if timesamps + /** Aggregator that returns the difference of the first value and the + * last value in the data points */ + public static final Aggregator DIFF = new Diff( + Interpolation.LERP, "diff"); + + /** Sums data points but will cause the SpanGroup to return a 0 if timestamps * don't line up instead of interpolating. */ public static final Aggregator ZIMSUM = new Sum( Interpolation.ZIM, "zimsum"); - /** Returns the minimum data point, causing SpanGroup to set .MaxValue + /** Returns the minimum data point, causing SpanGroup to set <type>.MaxValue * if timestamps don't line up instead of interpolating. */ public static final Aggregator MIMMIN = new Min( Interpolation.MAX, "mimmin"); - /** Returns the maximum data point, causing SpanGroup to set .MinValue + /** Returns the maximum data point, causing SpanGroup to set <type>.MinValue * if timestamps don't line up instead of interpolating. */ public static final Aggregator MIMMAX = new Max( Interpolation.MIN, "mimmax"); + + /** Aggregator that returns the square sum of the data point. */ + public static final Aggregator SQUARESUM = new SquareSum(Interpolation.ZIM, "squareSum"); + + /** Aggregator that returns the number of data points. + * WARNING: This currently interpolates with zero-if-missing. In this case + * counts will be off when counting multiple time series. Only use this when + * downsampling until we support NaNs. + * @since 2.2 */ + public static final Aggregator COUNT = new Count(Interpolation.ZIM, "count"); + + /** Aggregator that returns the first data point. */ + public static final Aggregator FIRST = new First(Interpolation.ZIM, "first"); + + /** Aggregator that returns the last data point. */ + public static final Aggregator LAST = new Last(Interpolation.ZIM, "last"); /** Maps an aggregator name to its instance. */ private static final HashMap aggregators; + /** Aggregator that returns 99.9th percentile. */ + public static final PercentileAgg p999 = new PercentileAgg(99.9d, "p999"); + /** Aggregator that returns 99th percentile. */ + public static final PercentileAgg p99 = new PercentileAgg(99d, "p99"); + /** Aggregator that returns 95th percentile. */ + public static final PercentileAgg p95 = new PercentileAgg(95d, "p95"); + /** Aggregator that returns 90th percentile. */ + public static final PercentileAgg p90 = new PercentileAgg(90d, "p90"); + /** Aggregator that returns 75th percentile. */ + public static final PercentileAgg p75 = new PercentileAgg(75d, "p75"); + /** Aggregator that returns 50th percentile. */ + public static final PercentileAgg p50 = new PercentileAgg(50d, "p50"); + + /** Aggregator that returns estimated 99.9th percentile. */ + public static final PercentileAgg ep999r3 = + new PercentileAgg(99.9d, "ep999r3", EstimationType.R_3); + /** Aggregator that returns estimated 99th percentile. */ + public static final PercentileAgg ep99r3 = + new PercentileAgg(99d, "ep99r3", EstimationType.R_3); + /** Aggregator that returns estimated 95th percentile. */ + public static final PercentileAgg ep95r3 = + new PercentileAgg(95d, "ep95r3", EstimationType.R_3); + /** Aggregator that returns estimated 90th percentile. */ + public static final PercentileAgg ep90r3 = + new PercentileAgg(90d, "ep90r3", EstimationType.R_3); + /** Aggregator that returns estimated 75th percentile. */ + public static final PercentileAgg ep75r3 = + new PercentileAgg(75d, "ep75r3", EstimationType.R_3); + /** Aggregator that returns estimated 50th percentile. */ + public static final PercentileAgg ep50r3 = + new PercentileAgg(50d, "ep50r3", EstimationType.R_3); + + /** Aggregator that returns estimated 99.9th percentile. */ + public static final PercentileAgg ep999r7 = + new PercentileAgg(99.9d, "ep999r7", EstimationType.R_7); + /** Aggregator that returns estimated 99th percentile. */ + public static final PercentileAgg ep99r7 = + new PercentileAgg(99d, "ep99r7", EstimationType.R_7); + /** Aggregator that returns estimated 95th percentile. */ + public static final PercentileAgg ep95r7 = + new PercentileAgg(95d, "ep95r7", EstimationType.R_7); + /** Aggregator that returns estimated 90th percentile. */ + public static final PercentileAgg ep90r7 = + new PercentileAgg(90d, "ep90r7", EstimationType.R_7); + /** Aggregator that returns estimated 75th percentile. */ + public static final PercentileAgg ep75r7 = + new PercentileAgg(75d, "ep75r7", EstimationType.R_7); + /** Aggregator that returns estimated 50th percentile. */ + public static final PercentileAgg ep50r7 = + new PercentileAgg(50d, "ep50r7", EstimationType.R_7); + static { aggregators = new HashMap(8); aggregators.put("sum", SUM); aggregators.put("min", MIN); aggregators.put("max", MAX); aggregators.put("avg", AVG); + aggregators.put("none", NONE); + aggregators.put("median", MEDIAN); + aggregators.put("mult", MULTIPLY); aggregators.put("dev", DEV); + aggregators.put("diff", DIFF); + aggregators.put("count", COUNT); aggregators.put("zimsum", ZIMSUM); aggregators.put("mimmin", MIMMIN); aggregators.put("mimmax", MIMMAX); + aggregators.put("first", FIRST); + aggregators.put("last", LAST); + aggregators.put("pfsum", PFSUM); + aggregators.put("squareSum", SQUARESUM); + + PercentileAgg[] percentiles = { + p999, p99, p95, p90, p75, p50, + ep999r3, ep99r3, ep95r3, ep90r3, ep75r3, ep50r3, + ep999r7, ep99r7, ep95r7, ep90r7, ep75r7, ep50r7 + }; + for (PercentileAgg agg : percentiles) { + aggregators.put(agg.toString(), agg); + } } private Aggregators() { @@ -106,15 +227,13 @@ public static Aggregator get(final String name) { throw new NoSuchElementException("No such aggregator: " + name); } - private static final class Sum implements Aggregator { - private final Interpolation method; - private final String name; - + + private static final class Sum extends Aggregator { public Sum(final Interpolation method, final String name) { - this.method = method; - this.name = name; + super(method, name); } - + + @Override public long runLong(final Longs values) { long result = values.nextLongValue(); while (values.hasNextValue()) { @@ -123,33 +242,64 @@ public long runLong(final Longs values) { return result; } + @Override public double runDouble(final Doubles values) { - double result = values.nextDoubleValue(); + double result = 0.; + long n = 0L; + while (values.hasNextValue()) { - result += values.nextDoubleValue(); + final double val = values.nextDoubleValue(); + if (!Double.isNaN(val)) { + result += val; + ++n; + } } - return result; + + return (0L == n) ? Double.NaN : result; } + + } - public String toString() { - return name; + private static final class SquareSum extends Aggregator { + public SquareSum(final Interpolation method, final String name) { + super(method, name); } - public Interpolation interpolationMethod() { - return method; + @Override + public long runLong(final Longs values) { + long a = values.nextLongValue(); + long result = a * a; + while (values.hasNextValue()) { + a = values.nextLongValue(); + result += a * a; + } + return result; } - + + @Override + public double runDouble(final Doubles values) { + double result = 0.; + long n = 0L; + + while (values.hasNextValue()) { + final double val = values.nextDoubleValue(); + if (!Double.isNaN(val)) { + result += val * val; + ++n; + } + } + + return (0L == n) ? Double.NaN : result; + } + } - private static final class Min implements Aggregator { - private final Interpolation method; - private final String name; - + private static final class Min extends Aggregator { public Min(final Interpolation method, final String name) { - this.method = method; - this.name = name; + super(method, name); } - + + @Override public long runLong(final Longs values) { long min = values.nextLongValue(); while (values.hasNextValue()) { @@ -161,36 +311,29 @@ public long runLong(final Longs values) { return min; } + @Override public double runDouble(final Doubles values) { - double min = values.nextDoubleValue(); + final double initial = values.nextDoubleValue(); + double min = Double.isNaN(initial) ? Double.POSITIVE_INFINITY : initial; + while (values.hasNextValue()) { final double val = values.nextDoubleValue(); - if (val < min) { + if (!Double.isNaN(val) && val < min) { min = val; } } - return min; - } - - public String toString() { - return name; - } - public Interpolation interpolationMethod() { - return method; + return (Double.POSITIVE_INFINITY == min) ? Double.NaN : min; } } - private static final class Max implements Aggregator { - private final Interpolation method; - private final String name; - + private static final class Max extends Aggregator { public Max(final Interpolation method, final String name) { - this.method = method; - this.name = name; + super(method, name); } - + + @Override public long runLong(final Longs values) { long max = values.nextLongValue(); while (values.hasNextValue()) { @@ -202,36 +345,29 @@ public long runLong(final Longs values) { return max; } + @Override public double runDouble(final Doubles values) { - double max = values.nextDoubleValue(); + final double initial = values.nextDoubleValue(); + double max = Double.isNaN(initial) ? Double.NEGATIVE_INFINITY : initial; + while (values.hasNextValue()) { final double val = values.nextDoubleValue(); - if (val > max) { + if (!Double.isNaN(val) && val > max) { max = val; } } - return max; - } - public String toString() { - return name; - } - - public Interpolation interpolationMethod() { - return method; + return (Double.NEGATIVE_INFINITY == max) ? Double.NaN : max; } } - private static final class Avg implements Aggregator { - private final Interpolation method; - private final String name; - + private static final class Avg extends Aggregator { public Avg(final Interpolation method, final String name) { - this.method = method; - this.name = name; + super(method, name); } - + + @Override public long runLong(final Longs values) { long result = values.nextLongValue(); int n = 1; @@ -242,26 +378,114 @@ public long runLong(final Longs values) { return result / n; } + @Override public double runDouble(final Doubles values) { - double result = values.nextDoubleValue(); - int n = 1; + double result = 0.; + int n = 0; while (values.hasNextValue()) { - result += values.nextDoubleValue(); - n++; + final double val = values.nextDoubleValue(); + if (!Double.isNaN(val)) { + result += val; + n++; + } } - return result / n; + return (0 == n) ? Double.NaN : result / n; + } + + } + + private static final class Median extends Aggregator { + public Median(final Interpolation method, final String name) { + super(method, name); } - public String toString() { - return name; + @Override + public long runLong(final Longs values) { + final List collection = Lists.newArrayList(); + while (values.hasNextValue()) { + collection.add(values.nextLongValue()); + } + if (collection.isEmpty()) { + throw new IllegalStateException("Shouldn't be here without any data"); + } + Collections.sort(collection); + return collection.get(collection.size() / 2); } + + @Override + public double runDouble(final Doubles values) { + final List collection = Lists.newArrayList(); + while (values.hasNextValue()) { + final double val = values.nextDoubleValue(); + if (!Double.isNaN(val)) { + collection.add(val); + } + } + if (collection.isEmpty()) { + // in this case we may have had lots of NaNs so just drop em. + return Double.NaN; + } + Collections.sort(collection); + return collection.get(collection.size() / 2); + } + } - public Interpolation interpolationMethod() { - return method; + /** + * An aggregator that isn't meant for aggregation. Paradoxical!! + * Really it's used as a flag to indicate that, during sorting and iteration, + * that the pipeline should not perform any aggregation and should emit + * raw time series. + */ + private static final class None extends Aggregator { + public None(final Interpolation method, final String name) { + super(method, name); + } + + @Override + public long runLong(final Longs values) { + final long v = values.nextLongValue(); + if (values.hasNextValue()) { + throw new IllegalDataException("More than one value in aggregator " + values); + } + return v; + } + + @Override + public double runDouble(final Doubles values) { + final double v = values.nextDoubleValue(); + if (values.hasNextValue()) { + throw new IllegalDataException("More than one value in aggregator " + values); + } + return v; } - } + + private static final class Multiply extends Aggregator { + + public Multiply(final Interpolation method, final String name) { + super(method, name); + } + @Override + public long runLong(Longs values) { + long result = values.nextLongValue(); + while (values.hasNextValue()) { + result *= values.nextLongValue(); + } + return result; + } + + @Override + public double runDouble(Doubles values) { + double result = values.nextDoubleValue(); + while (values.hasNextValue()) { + result *= values.nextDoubleValue(); + } + return result; + } + + } + /** * Standard Deviation aggregator. * Can compute without storing all of the data points in memory at the same @@ -271,15 +495,12 @@ public Interpolation interpolationMethod() { * paper by B. P. Welford and is presented in Donald Knuth's Art of * Computer Programming, Vol 2, page 232, 3rd edition */ - private static final class StdDev implements Aggregator { - private final Interpolation method; - private final String name; - + private static final class StdDev extends Aggregator { public StdDev(final Interpolation method, final String name) { - this.method = method; - this.name = name; + super(method, name); } - + + @Override public long runLong(final Longs values) { double old_mean = values.nextLongValue(); @@ -288,48 +509,345 @@ public long runLong(final Longs values) { } long n = 2; - double new_mean = 0; - double variance = 0; + double new_mean = 0.; + double M2 = 0.; do { final double x = values.nextLongValue(); new_mean = old_mean + (x - old_mean) / n; - variance += (x - old_mean) * (x - new_mean); + M2 += (x - old_mean) * (x - new_mean); old_mean = new_mean; n++; } while (values.hasNextValue()); - return (long) Math.sqrt(variance / (n - 1)); + return (long) Math.sqrt(M2 / (n - 1)); } + @Override public double runDouble(final Doubles values) { + // Try to get at least one non-NaN value. double old_mean = values.nextDoubleValue(); + while (Double.isNaN(old_mean) && values.hasNextValue()) { + old_mean = values.nextDoubleValue(); + } + if (Double.isNaN(old_mean)) { + // Couldn't find any non-NaN values. + // The stddev of NaNs is NaN. + return Double.NaN; + } if (!values.hasNextValue()) { - return 0; + // Only found one non-NaN value. + // The stddev of one value is zero. + return 0.; } + // If we got here, then we have one non-NaN value, and there are more + // values to aggregate; however, some or all of these values may be NaNs. + long n = 2; - double new_mean = 0; - double variance = 0; + double new_mean = 0.; + + // This is not strictly the second central moment (i.e., variance), but + // rather a multiple of it. + double M2 = 0.; do { final double x = values.nextDoubleValue(); - new_mean = old_mean + (x - old_mean) / n; - variance += (x - old_mean) * (x - new_mean); - old_mean = new_mean; - n++; + if (!Double.isNaN(x)) { + new_mean = old_mean + (x - old_mean) / n; + M2 += (x - old_mean) * (x - new_mean); + old_mean = new_mean; + n++; + } } while (values.hasNextValue()); - return Math.sqrt(variance / (n - 1)); + // If n is still 2, then we never found another non-NaN value; therefore, + // we should return zero. + // + // Otherwise, we calculate the actual variance, and then we find its + // positive square root, which is the standard deviation. + return (2 == n) ? 0. : Math.sqrt(M2 / (n - 1)); + } + + } + + /** + * Difference of the first value and the last value in multi values aggregator. + */ + private static final class Diff extends net.opentsdb.core.Aggregator { + public Diff(final Interpolation method, final String name) { + super(method, name); } - public String toString() { - return name; + @Override + public long runLong(final Longs values) { + long first_mean = values.nextLongValue(); + + if (!values.hasNextValue()) { + return 0; + } + + long last_mean = 0; + do { + last_mean = values.nextLongValue(); + } while (values.hasNextValue()); + + return last_mean - first_mean; + } + + @Override + public double runDouble(final Doubles values) { + double first_mean = values.nextDoubleValue(); + while (Double.isNaN(first_mean) && values.hasNextValue()) { + first_mean = values.nextDoubleValue(); + } + + if (Double.isNaN(first_mean)) { + return Double.NaN; + } + if (!values.hasNextValue()) { + return 0.; + } + + double last_mean = 0.; + do { + last_mean = values.nextDoubleValue(); + } while (values.hasNextValue()); + + return last_mean - first_mean; + } + } + + private static final class Count extends Aggregator { + public Count(final Interpolation method, final String name) { + super(method, name); } - public Interpolation interpolationMethod() { - return method; + @Override + public long runLong(Longs values) { + long result = 0; + while (values.hasNextValue()) { + values.nextLongValue(); + result++; + } + return result; + } + + @Override + public double runDouble(Doubles values) { + double result = 0; + while (values.hasNextValue()) { + final double val = values.nextDoubleValue(); + if (!Double.isNaN(val)) { + result++; + } + } + return result; + } + + } + + /** + * Percentile aggregator based on apache commons math3 implementation + * The default calculation is: + * index=(N+1)p + * estimate=x⌈h−1/2⌉ + * minLimit=0 + * maxLimit=1 + */ + private static final class PercentileAgg extends Aggregator { + private final Double percentile; + private final EstimationType estimation; + + public PercentileAgg(final Double percentile, final String name) { + this(percentile, name, null); + } + + public PercentileAgg(final Double percentile, final String name, + final EstimationType est) { + super(Aggregators.Interpolation.LERP, name); + Preconditions.checkArgument(percentile > 0 && percentile <= 100, + "Invalid percentile value"); + this.percentile = percentile; + this.estimation = est; + } + + @Override + public long runLong(final Longs values) { + final Percentile percentile = + this.estimation == null + ? new Percentile(this.percentile) + : new Percentile(this.percentile).withEstimationType(estimation); + final ResizableDoubleArray local_values = new ResizableDoubleArray(); + while(values.hasNextValue()) { + local_values.addElement(values.nextLongValue()); + } + percentile.setData(local_values.getElements()); + return (long) percentile.evaluate(); + } + + @Override + public double runDouble(final Doubles values) { + final Percentile percentile = new Percentile(this.percentile); + final ResizableDoubleArray local_values = new ResizableDoubleArray(); + int n = 0; + while(values.hasNextValue()) { + final double val = values.nextDoubleValue(); + if (!Double.isNaN(val)) { + local_values.addElement(val); + n++; + } + } + if (n > 0) { + percentile.setData(local_values.getElements()); + return percentile.evaluate(); + } else { + return Double.NaN; + } + } + + } + public static final class MovingAverage extends Aggregator { + private LinkedList list = new LinkedList(); + private final long numPoints; + private final boolean isTimeUnit; + + public MovingAverage(final Interpolation method, final String name, long numPoints, boolean isTimeUnit) { + super(method, name); + this.numPoints = numPoints; + this.isTimeUnit = isTimeUnit; + } + + public long runLong(final Longs values) { + long sum = values.nextLongValue(); + while (values.hasNextValue()) { + sum += values.nextLongValue(); + } + + if (values instanceof DataPoint) { + long ts = ((DataPoint) values).timestamp(); + list.addFirst(new SumPoint(ts, sum)); + } + + long result = 0; + int count = 0; + + Iterator iter = list.iterator(); + SumPoint first = iter.next(); + boolean conditionMet = false; + + // now sum up the preceeding points + while (iter.hasNext()) { + SumPoint next = iter.next(); + result += (Long) next.val; + count++; + if (!isTimeUnit && count >= numPoints) { + conditionMet = true; + break; + } else if (isTimeUnit && ((first.ts - next.ts) > numPoints)) { + conditionMet = true; + break; + } + } + + if (!conditionMet || count == 0) { + return 0; + } + + return result / count; + } + + @Override + public double runDouble(Doubles values) { + double sum = values.nextDoubleValue(); + while (values.hasNextValue()) { + sum += values.nextDoubleValue(); + } + + if (values instanceof DataPoint) { + long ts = ((DataPoint) values).timestamp(); + list.addFirst(new SumPoint(ts, sum)); + } + + double result = 0; + int count = 0; + + Iterator iter = list.iterator(); + SumPoint first = iter.next(); + boolean conditionMet = false; + + // now sum up the preceeding points + while (iter.hasNext()) { + SumPoint next = iter.next(); + result += (Double) next.val; + count++; + if (!isTimeUnit && count >= numPoints) { + conditionMet = true; + break; + } else if (isTimeUnit && ((first.ts - next.ts) > numPoints)) { + conditionMet = true; + break; + } + } + + if (!conditionMet || count == 0) { + return 0; + } + + return result / count; + } + + class SumPoint { + long ts; + Object val; + + public SumPoint(long ts, Object val) { + this.ts = ts; + this.val = val; + } + } + } + + private static final class First extends Aggregator { + public First(final Interpolation method, final String name) { + super(method, name); } + public long runLong(final Longs values) { + long val = values.nextLongValue(); + while (values.hasNextValue()) { + values.nextLongValue(); + } + return val; + } + + public double runDouble(final Doubles values) { + double val = values.nextDoubleValue(); + while (values.hasNextValue()) { + values.nextDoubleValue(); + } + return val; + } } + + private static final class Last extends Aggregator { + public Last(final Interpolation method, final String name) { + super(method, name); + } + + public long runLong(final Longs values) { + long val = values.nextLongValue(); + while (values.hasNextValue()) { + val = values.nextLongValue(); + } + return val; + } + public double runDouble(final Doubles values) { + double val = values.nextDoubleValue(); + while (values.hasNextValue()) { + val = values.nextDoubleValue(); + } + return val; + } + } } diff --git a/src/core/AppendDataPoints.java b/src/core/AppendDataPoints.java new file mode 100644 index 0000000000..3646e4039b --- /dev/null +++ b/src/core/AppendDataPoints.java @@ -0,0 +1,260 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import java.util.Collection; +import java.util.Map; +import java.util.TreeMap; + +import net.opentsdb.core.Internal.Cell; +import net.opentsdb.utils.DateTime; + +import org.hbase.async.Bytes; +import org.hbase.async.KeyValue; +import org.hbase.async.PutRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.stumbleupon.async.Deferred; + +/** + * A class that deals with serializing/deserializing appended data point columns. + * In busy TSDB installs appends can save on storage at write time and network + * bandwidth as TSD compactions are no longer necessary. Each data point is + * concatenated to a byte array in storage. At query time, the values are ordered + * and de-duped. Optionally the column can be re-written when out of order or + * duplicates are detected. + * NOTE: This will increase CPU usage on your HBase servers as it has to perform + * the atomic read-modify-write operation on the column. + * @since 2.2 + */ +public class AppendDataPoints { + private static final Logger LOG = LoggerFactory.getLogger(AppendDataPoints.class); + + /** The prefix ID of append columns */ + public static final byte APPEND_COLUMN_PREFIX = 0x05; + + /** The full column qualifier for append columns */ + public static final byte[] APPEND_COLUMN_QUALIFIER = new byte[] { + APPEND_COLUMN_PREFIX, 0x00, 0x00}; + + /** A threshold in seconds where we avoid writing repairs */ + public static final int REPAIR_THRESHOLD = 3600; + + /** Filled with the qualifiers in the compacted data points format after parsing */ + private byte[] qualifier; + + /** Filled with the values in the compacted data points format after parsing */ + private byte[] value; + + /** A deferred that is set if a repaired column was sent to storage */ + private Deferred repaired_deferred = null; + + /** + * Default empty ctor + */ + public AppendDataPoints() { + + } + + /** + * Creates a new AppendDataPoints object from a qualifier and value. You can + * then call {@link #getBytes()} to write to TSDB. + * @param qualifier The qualifier with the time offset, type and length flags. + * @param value The value to append + * @throws IllegalArgumentException if the qualifier or value is null or empty + */ + public AppendDataPoints(final byte[] qualifier, final byte[] value) { + if (qualifier == null || qualifier.length < 1) { + throw new IllegalArgumentException("Qualifier cannot be null or empty"); + } + if (value == null || value.length < 1) { + throw new IllegalArgumentException("Value cannot be null or empty"); + } + this.qualifier = qualifier; + this.value = value; + } + + /** + * Concatenates the qualifier and value for appending to a column in the + * backing data store. + * @return A byte array to append to the value of a column. + */ + public byte[] getBytes() { + final byte[] bytes = new byte[qualifier.length + value.length]; + System.arraycopy(this.qualifier, 0, bytes, 0, qualifier.length); + System.arraycopy(value, 0, bytes, qualifier.length, value.length); + return bytes; + } + + /** + * Parses a column from storage, orders and drops newer duplicate data points. + * The parsing will return both a Cell collection for debugging and add + * the cells to concatenated qualifier and value arrays in the compacted data + * point format so that the results can be merged with other non-append + * columns or rows. + *

    + * WARNING: If the "tsd.core.repair_appends" config is set to true then this + * method will issue puts against the database, overwriting the column with + * sorted and de-duplicated data. It will only do this for rows that are at + * least an hour old so as to avoid pounding current rows. + *

    + * TODO (CL) - allow for newer or older data points depending on a config. + * @param tsdb The TSDB to which we belong + * @param kv The key value t parse + * @throws IllegalArgumentException if the given KV is not an append column + * or we were unable to parse the value. + */ + public final Collection parseKeyValue(final TSDB tsdb, final KeyValue kv) { + if (kv.qualifier().length != 3 || kv.qualifier()[0] != APPEND_COLUMN_PREFIX) { + // it's really not an issue if the offset is not 0, maybe in the future + // we'll support appends at different offsets. + throw new IllegalArgumentException("Can not parse cell, it is not " + + " an appended cell. It has a different qualifier " + + Bytes.pretty(kv.qualifier()) + ", row key " + Bytes.pretty(kv.key())); + } + final boolean repair = tsdb.getConfig().repair_appends(); + final long base_time; + try { + base_time = Internal.baseTime(tsdb, kv.key()); + } catch (ArrayIndexOutOfBoundsException oob) { + throw new IllegalDataException("Corrupted value: invalid row key: " + kv, + oob); + } + + int val_idx = 0; + int val_length = 0; + int qual_length = 0; + int last_delta = -1; // Time delta, extracted from the qualifier. + + final Map deltas = new TreeMap(); + boolean has_duplicates = false; + boolean out_of_order = false; + boolean needs_repair = false; + + try { + while (val_idx < kv.value().length) { + byte[] q = Internal.extractQualifier(kv.value(), val_idx); + System.arraycopy(kv.value(), val_idx, q, 0, q.length); + val_idx=val_idx + q.length; + + int vlen = Internal.getValueLengthFromQualifier(q, 0); + byte[] v = new byte[vlen]; + System.arraycopy(kv.value(), val_idx, v, 0, vlen); + val_idx += vlen; + int delta = Internal.getOffsetFromQualifier(q); + + final Cell duplicate = deltas.get(delta); + if (duplicate != null) { + // This is a duplicate cell, skip it + has_duplicates = true; + qual_length -= duplicate.qualifier.length; + val_length -= duplicate.value.length; + } + + qual_length += q.length; + val_length += vlen; + final Cell cell = new Cell(q, v); + deltas.put(delta, cell); + + if (!out_of_order) { + // Data points needs to be sorted if we find at least one out of + // order data + if (delta <= last_delta) { + out_of_order = true; + } + last_delta = delta; + } + } + } catch (ArrayIndexOutOfBoundsException oob) { + throw new IllegalDataException("Corrupted value: couldn't break down" + + " into individual values (consumed " + val_idx + " bytes, but was" + + " expecting to consume " + (kv.value().length) + "): " + kv + + ", cells so far: " + deltas.values(), oob); + } + + if (has_duplicates || out_of_order) { + if ((DateTime.currentTimeMillis() / 1000) - base_time > REPAIR_THRESHOLD) { + needs_repair = true; + } + } + + // Check we consumed all the bytes of the value. + if (val_idx != kv.value().length) { + throw new IllegalDataException("Corrupted value: couldn't break down" + + " into individual values (consumed " + val_idx + " bytes, but was" + + " expecting to consume " + (kv.value().length) + "): " + kv + + ", cells so far: " + deltas.values()); + } + + val_idx = 0; + int qual_idx = 0; + byte[] healed_cell = null; + int healed_index = 0; + + this.value = new byte[val_length]; + this.qualifier = new byte[qual_length]; + + if (repair && needs_repair) { + healed_cell = new byte[val_length+qual_length]; + } + + for (final Cell cell: deltas.values()) { + System.arraycopy(cell.qualifier, 0, this.qualifier, qual_idx, + cell.qualifier.length); + qual_idx += cell.qualifier.length; + System.arraycopy(cell.value, 0, this.value, val_idx, cell.value.length); + val_idx += cell.value.length; + + if (repair && needs_repair) { + System.arraycopy(cell.qualifier, 0, healed_cell, healed_index, + cell.qualifier.length); + healed_index += cell.qualifier.length; + System.arraycopy(cell.value, 0, healed_cell, healed_index, cell.value.length); + healed_index += cell.value.length; + } + } + + if (repair && needs_repair) { + LOG.debug("Repairing appended data column " + kv); + final PutRequest put = RequestBuilder.buildPutRequest(tsdb.getConfig(), tsdb.table, kv.key(), + TSDB.FAMILY(), kv.qualifier(), healed_cell, kv.timestamp()); + repaired_deferred = tsdb.getClient().put(put); + } + + return deltas.values(); + } + + /** @return the sorted qualifier in a compacted data point format after + * {@link #parseKeyValue(TSDB, KeyValue)} has been called */ + public byte[] qualifier() { + return qualifier; + } + + /** @return the sorted value in a compacted data point format after + * {@link #parseKeyValue(TSDB, KeyValue)} has been called */ + public byte[] value() { + return value; + } + + /** @return a deferred to wait on if the call to + * {@link #parseKeyValue(TSDB, KeyValue)} triggered a put to storage. */ + public Deferred repairedDeferred() { + return repaired_deferred; + } + + /** @return whether or not a qualifier of AppendDataPoints */ + public static boolean isAppendDataPoints(byte[] qualifier) { + return qualifier != null && qualifier.length == 3 && qualifier[0] == APPEND_COLUMN_PREFIX; + } +} diff --git a/src/core/BatchedDataPoints.java b/src/core/BatchedDataPoints.java new file mode 100644 index 0000000000..f21a70d23f --- /dev/null +++ b/src/core/BatchedDataPoints.java @@ -0,0 +1,525 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2014 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Map; + +import com.stumbleupon.async.Deferred; + +import org.hbase.async.Bytes; +import org.hbase.async.Bytes.ByteMap; + +import net.opentsdb.meta.Annotation; + +/** + * Receives new data points and stores them in compacted form. No points are + * written until {@code flushNow} is called. This ensures that true batch + * dynamics can be leveraged. This implementation will allow an entire hours + * worth of data to be written in a single transaction to the data table. + */ +final class BatchedDataPoints implements WritableDataPoints { + + /** + * The {@code TSDB} instance we belong to. + */ + private final TSDB tsdb; + + /** + * The row key. Optional salt + 3 bytes for the metric name, 4 bytes for the + * base timestamp, 6 bytes per tag (3 for the name, 3 for the value). + */ + private byte[] row_key; + + /** + * Track the last timestamp written for this series. + */ + private long last_timestamp; + + /** + * Number of data points in this row. + */ + private int size = 0; + + /** + * Storage of the compacted qualifier. + */ + private byte[] batched_qualifier = new byte[Const.MAX_TIMESPAN * 4]; + + /** + * Storage of the compacted value. + */ + private byte[] batched_value = new byte[Const.MAX_TIMESPAN * 8]; + + /** + * Track the index position where the next qualifier gets written. + */ + private int qualifier_index = 0; + + /** + * Track the index position where the next value gets written. + */ + private int value_index = 0; + + /** + * Track the base time for this batch of points. + */ + private long base_time; + + /** + * Constructor. + * + * @param tsdb The TSDB we belong to. + */ + BatchedDataPoints(final TSDB tsdb, final String metric, + final Map tags) { + this.tsdb = tsdb; + setSeries(metric, tags); + } + + /** + * Sets the metric name and tags of this batch. This method only need be + * called if there is a desire to reuse the data structure after the data has + * been flushed. This will reset all cached information in this data structure. + * @throws IllegalArgumentException if the metric name is empty or contains + * illegal characters or if the tags list is empty or one of the elements + * contains illegal characters. + */ + @Override + public void setSeries(final String metric, final Map tags) { + IncomingDataPoints.checkMetricAndTags(metric, tags); + try { + row_key = IncomingDataPoints.rowKeyTemplate(tsdb, metric, tags); + RowKey.prefixKeyWithSalt(row_key); + reset(); + } + catch (RuntimeException e) { + throw e; + } + catch (Exception e) { + throw new RuntimeException("Should never happen", e); + } + } + + /** + * Resets the indices without overwriting the buffers. So the same amount of + * space will remain allocated. + */ + private void reset() { + size = 0; + qualifier_index = 0; + value_index = 0; + base_time = Long.MIN_VALUE; + last_timestamp = Long.MIN_VALUE; + } + + /** + * A copy of the values is created and sent with a put request. A reset is + * initialized which makes this data structure ready to be reused for the same + * metric and tags but for a different hour of data. + * @return {@inheritDoc} + */ + @Override + public Deferred persist() { + final byte[] q = Arrays.copyOfRange(batched_qualifier, 0, qualifier_index); + final byte[] v = Arrays.copyOfRange(batched_value, 0, value_index); + final byte[] r = Arrays.copyOfRange(row_key, 0, row_key.length); + final long base_time = this.base_time; // shadow fixes issue #1436 + System.out.println(Arrays.toString(q) + " " + Arrays.toString(v) + " " + Arrays.toString(r)); + reset(); + return tsdb.put(r, q, v, base_time); + } + + @Override + public void setBufferingTime(short time) { + // does nothing + } + + @Override + public void setBatchImport(boolean batchornot) { + // does nothing + } + + @Override + public Deferred addPoint(final long timestamp, final long value) { + final byte[] v; + if (Byte.MIN_VALUE <= value && value <= Byte.MAX_VALUE) { + v = new byte[] {(byte) value}; + } + else if (Short.MIN_VALUE <= value && value <= Short.MAX_VALUE) { + v = Bytes.fromShort((short) value); + } + else if (Integer.MIN_VALUE <= value && value <= Integer.MAX_VALUE) { + v = Bytes.fromInt((int) value); + } + else { + v = Bytes.fromLong(value); + } + final short flags = (short) (v.length - 1); // Just the length. + return addPointInternal(timestamp, v, flags); + } + + @Override + public Deferred addPoint(final long timestamp, final float value) { + if (Float.isNaN(value) || Float.isInfinite(value)) { + throw new IllegalArgumentException("value is NaN or Infinite: " + value + + " for timestamp=" + timestamp); + } + final short flags = Const.FLAG_FLOAT | 0x3; // A float stored on 4 bytes. + return addPointInternal(timestamp, + Bytes.fromInt(Float.floatToRawIntBits(value)), flags); + } + + /** + * Implements {@link #addPoint} by storing a value with a specific flag. + * + * @param timestamp The timestamp to associate with the value. + * @param value The value to store. + * @param flags Flags to store in the qualifier (size and type of the data point). + */ + private Deferred addPointInternal(final long timestamp, + final byte[] value, final short flags) throws IllegalDataException { + final boolean ms_timestamp = (timestamp & Const.SECOND_MASK) != 0; + + // we only accept unix epoch timestamps in seconds or milliseconds + if (timestamp < 0 || (ms_timestamp && timestamp > 9999999999999L)) { + throw new IllegalArgumentException((timestamp < 0 ? "negative " : "bad") + + " timestamp=" + timestamp + + " when trying to add value=" + Arrays.toString(value) + " to " + this); + } + + // always maintain lastTimestamp in milliseconds + if ((ms_timestamp ? timestamp : timestamp * 1000) <= last_timestamp) { + throw new IllegalArgumentException("New timestamp=" + timestamp + + " is less than or equal to previous=" + last_timestamp + + " when trying to add value=" + Arrays.toString(value) + + " to " + this); + } + last_timestamp = (ms_timestamp ? timestamp : timestamp * 1000); + + long incomingBaseTime; + if (ms_timestamp) { + // drop the ms timestamp to seconds to calculate the base timestamp + incomingBaseTime = + ((timestamp / 1000) - ((timestamp / 1000) % Const.MAX_TIMESPAN)); + } + else { + incomingBaseTime = (timestamp - (timestamp % Const.MAX_TIMESPAN)); + } + + /** + * First time we add a point initialize the rows timestamp. + */ + if (base_time == Long.MIN_VALUE) { + base_time = incomingBaseTime; + Bytes.setInt(row_key, (int) base_time, + tsdb.metrics.width() + Const.SALT_WIDTH()); + } + + if (incomingBaseTime - base_time >= Const.MAX_TIMESPAN) { + throw new IllegalDataException( + "The timestamp is beyond the boundary of this batch of data points"); + } + if (incomingBaseTime < base_time) { + throw new IllegalDataException( + "The timestamp is prior to the boundary of this batch of data points"); + } + + // Java is so stupid with its auto-promotion of int to float. + final byte[] new_qualifier = Internal.buildQualifier(timestamp, flags); + + // compact this data point with the previously compacted data points. + append(new_qualifier, value); + size++; + + /** + * Satisfies the interface. + */ + return Deferred.fromResult((Object) null); + } + + /** + * Checks the size of the qualifier and value arrays to make sure we have + * space. If not then we double the size of the arrays. This way a row + * allocates space for a full hour of second data but if the user requires + * millisecond storage with more than 3600 points, it will expand. + * @param next_qualifier The next qualifier to use for it's length + * @param next_value The next value to use for it's length + */ + private void ensureCapacity(final byte[] next_qualifier, + final byte[] next_value) { + if (qualifier_index + next_qualifier.length >= batched_qualifier.length) { + batched_qualifier = Arrays.copyOf(batched_qualifier, + batched_qualifier.length * 2); + } + if (value_index + next_value.length >= batched_value.length) { + batched_value = Arrays.copyOf(batched_value, batched_value.length * 2); + } + } + + /** + * Appends the value and qualifier to the appropriate arrays + * @param next_qualifier The next qualifier to append + * @param next_value The next value to append + */ + private void append(final byte[] next_qualifier, final byte[] next_value) { + ensureCapacity(next_qualifier, next_value); + + // Now let's simply concatenate all the values together. + System.arraycopy(next_value, 0, batched_value, value_index, next_value.length); + value_index += next_value.length; + + // Now let's concatenate all the qualifiers together. + System.arraycopy(next_qualifier, 0, batched_qualifier, qualifier_index, + next_qualifier.length); + qualifier_index += next_qualifier.length; + } + + @Override + public String metricName() { + try { + return metricNameAsync().joinUninterruptibly(); + } + catch (RuntimeException e) { + throw e; + } + catch (Exception e) { + throw new RuntimeException("Should never be here", e); + } + } + + @Override + public Deferred metricNameAsync() { + if (row_key == null) { + throw new IllegalStateException("Instance was not properly constructed!"); + } + final byte[] id = Arrays.copyOfRange(row_key, Const.SALT_WIDTH(), + tsdb.metrics.width() + Const.SALT_WIDTH()); + return tsdb.metrics.getNameAsync(id); + } + + @Override + public byte[] metricUID() { + return Arrays.copyOfRange(row_key, Const.SALT_WIDTH(), + Const.SALT_WIDTH() + TSDB.metrics_width()); + } + + @Override + public Map getTags() { + try { + return getTagsAsync().joinUninterruptibly(); + } + catch (RuntimeException e) { + throw e; + } + catch (Exception e) { + throw new RuntimeException("Should never be here", e); + } + } + + @Override + public ByteMap getTagUids() { + return Tags.getTagUids(row_key); + } + + @Override + public Deferred> getTagsAsync() { + return Tags.getTagsAsync(tsdb, row_key); + } + + @Override + public List getAggregatedTags() { + return Collections.emptyList(); + } + + @Override + public Deferred> getAggregatedTagsAsync() { + final List empty = Collections.emptyList(); + return Deferred.fromResult(empty); + } + + @Override + public List getAggregatedTagUids() { + return Collections.emptyList(); + } + + @Override + public List getTSUIDs() { + return Collections.emptyList(); + } + + @Override + public List getAnnotations() { + return null; + } + + @Override + public int size() { + return size; + } + + @Override + public int aggregatedSize() { + return 0; + } + + @Override + public SeekableView iterator() { + return new DataPointsIterator(this); + } + + /** + * @throws IndexOutOfBoundsException if {@code i} is out of bounds. + */ + private void checkIndex(final int i) { + if (i > size) { + throw new IndexOutOfBoundsException("index " + i + " > " + size + + " for this=" + this); + } + if (i < 0) { + throw new IndexOutOfBoundsException("negative index " + i + + " for this=" + this); + } + } + + /** + * Computes the proper offset to reach qualifier + * @param i + * @return + */ + private int qualifierOffset(final int i) { + int offset = 0; + for (int j = 0; j < i; j++) { + offset += Internal.getQualifierLength(batched_qualifier, offset); + } + return offset; + } + + @Override + public long timestamp(final int i) { + checkIndex(i); + return Internal.getTimestampFromQualifier(batched_qualifier, base_time, qualifierOffset(i)); + } + + @Override + public boolean isInteger(final int i) { + checkIndex(i); + return isInteger(i, qualifierOffset(i)); + } + + /** + * Tells whether or not the ith value is integer. Uses pre-computed qualifier offset. + * @param i + * @param q_offset qualifier offset + * @return + */ + private boolean isInteger(final int i, final int q_offset) { + final short flags = Internal.getFlagsFromQualifier(batched_qualifier, q_offset); + return (flags & Const.FLAG_FLOAT) == 0x0; + } + + @Override + public long longValue(final int i) { + checkIndex(i); + // compute the prope value and qualifier offsets + int v_offset = 0; + int q_offset = 0; + for (int j = 0; j < i; j++) { + v_offset += Internal.getValueLengthFromQualifier(batched_qualifier, q_offset); + q_offset += Internal.getQualifierLength(batched_qualifier, q_offset); + } + + if (isInteger(i, q_offset)) { + final short flags = Internal.getFlagsFromQualifier(batched_qualifier, q_offset); + return Internal.extractIntegerValue(batched_value, v_offset, (byte)flags); + } + throw new ClassCastException("value #" + i + " is not a long in " + this); + } + + @Override + public double doubleValue(final int i) { + checkIndex(i); + // compute the proper value and qualifier offsets + int v_offset = 0; + int q_offset = 0; + for (int j = 0; j < i; j++) { + v_offset += Internal.getValueLengthFromQualifier(batched_qualifier, q_offset); + q_offset += Internal.getQualifierLength(batched_qualifier, q_offset); + } + + if (!isInteger(i, q_offset)) { + final short flags = Internal.getFlagsFromQualifier(batched_qualifier, q_offset); + return Internal.extractFloatingPointValue(batched_value, v_offset, (byte)flags); + } + throw new ClassCastException("value #" + i + " is not a float in " + this); + } + + /** + * Returns a human readable string representation of the object. + */ + @Override + public String toString() { + // The argument passed to StringBuilder is a pretty good estimate of the + // length of the final string based on the row key and number of elements. + final String metric = metricName(); + final StringBuilder buf = new StringBuilder(80 + metric.length() + + row_key.length * 4 + size * 16); + buf.append("BatchedDataPoints(") + .append(row_key == null ? "" : Arrays.toString(row_key)) + .append(" (metric=") + .append(metric) + .append("), base_time=") + .append(base_time) + .append(" (") + .append(base_time > 0 ? new Date(base_time * 1000) : "no date") + .append("), ["); + int q_offset = 0; + int v_offset = 0; + for (int i = 0; i < size; i++) { + buf.append('+').append(Internal.getOffsetFromQualifier(batched_qualifier, q_offset)); + final short flags = Internal.getFlagsFromQualifier(batched_qualifier, q_offset); + if (isInteger(i, q_offset)) { + buf.append(":long(") + .append(Internal.extractIntegerValue(batched_value, v_offset, (byte)flags)); + } + else { + buf.append(":float(") + .append(Internal.extractFloatingPointValue(batched_value, v_offset, (byte)flags)); + } + buf.append(')'); + if (i != size - 1) { + buf.append(", "); + } + v_offset += Internal.getValueLengthFromQualifier(batched_qualifier, q_offset); + q_offset += Internal.getQualifierLength(batched_qualifier, q_offset); + } + buf.append("])"); + return buf.toString(); + } + + public int getQueryIndex() { + throw new UnsupportedOperationException("Not mapped to a query"); + } + @Override + public boolean isPercentile() { + return false; + } + + @Override + public float getPercentile() { + throw new UnsupportedOperationException("getPercentile not supported"); + } +} diff --git a/src/core/ColumnDatapointIterator.java b/src/core/ColumnDatapointIterator.java index 423f796aa9..8df8a20626 100644 --- a/src/core/ColumnDatapointIterator.java +++ b/src/core/ColumnDatapointIterator.java @@ -12,14 +12,18 @@ // see . package net.opentsdb.core; +import java.nio.ByteBuffer; import java.util.Arrays; + import org.hbase.async.KeyValue; +import net.opentsdb.utils.Pair; + /** * Internal implementation detail for {@link net.opentsdb.core.CompactionQueue}. This * allows iterating over the datapoints in a column without creating objects for each * datapoint. -* +* * @since 2.1 */ final class ColumnDatapointIterator implements Comparable { @@ -116,6 +120,19 @@ public void writeToBuffers(ByteBufferList compQualifier, ByteBufferList compValu compValue.add(value, value_offset, current_val_length); } + public void writeToBuffersFromOffset(ByteBufferList compQualifier, ByteBufferList compValue, Pair offsets, Pair offsetLengths) { + compQualifier.add(qualifier, offsets.getKey(), offsetLengths.getKey()); + compValue.add(value, offsets.getValue(), offsetLengths.getValue()); + } + + public Pair getOffsets() { + return new Pair(qualifier_offset, value_offset); + } + + public Pair getOffsetLengths() { + return new Pair(current_qual_length, current_val_length); + } + /** * @return the length of the qualifier for the current datapoint. */ @@ -135,9 +152,16 @@ public byte[] getCopyOfCurrentValue() { } } + /** + * @return a copy of the Qualifier of the current datapoint, after any fixups. + */ + public byte[] getCopyOfCurrentQualifier() { + return Arrays.copyOfRange(qualifier, qualifier_offset, qualifier_offset + current_qual_length); + } + /** * Advance to the next datapoint. - * + * * @return true if there is at least one more datapoint after advancing */ public boolean advance() { @@ -174,6 +198,18 @@ public int compareTo(ColumnDatapointIterator o) { return c; } + public double getCellValueAsDouble() { + byte[] copy = this.getCopyOfCurrentValue(); + byte[] qual = this.getCopyOfCurrentQualifier(); + ByteBuffer bb = ByteBuffer.wrap(copy); + if (Internal.isFloat(qual)) { + return copy.length == 4 ? bb.getFloat() : bb.getDouble(); + } else { + return ((copy.length == 1) ? bb.get() + : ((copy.length == 2) ? bb.getShort() : ((copy.length == 4) ? bb.getInt() : bb.getLong()))); + } + } + @Override public String toString() { return "q=" + Arrays.toString(qualifier) + " [ofs=" + qualifier_offset + "], v=" diff --git a/src/core/CompactionQueue.java b/src/core/CompactionQueue.java index 06385cfc15..857d60fbd2 100644 --- a/src/core/CompactionQueue.java +++ b/src/core/CompactionQueue.java @@ -27,7 +27,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import org.hbase.async.Bytes; import org.hbase.async.HBaseRpc; import org.hbase.async.KeyValue; @@ -36,6 +35,7 @@ import net.opentsdb.meta.Annotation; import net.opentsdb.stats.StatsCollector; import net.opentsdb.utils.JSON; +import net.opentsdb.utils.Pair; /** * "Queue" of rows to compact. @@ -76,6 +76,18 @@ final class CompactionQueue extends ConcurrentSkipListMap { /** On how many bytes do we encode metrics IDs. */ private final short metric_width; + /** How frequently the compaction thread wakes up to flush stuff. */ + private final int flush_interval; // seconds + + /** Minimum number of rows we'll attempt to compact at once. */ + private final int min_flush_threshold; // rows + + /** Maximum number of rows we'll compact concurrently. */ + private final int max_concurrent_flushes; // rows + + /** If this is X then we'll flush X times faster than we really need. */ + private final int flush_speed; // multiplicative factor + /** * Constructor. * @param tsdb The TSDB we belong to. @@ -84,6 +96,11 @@ public CompactionQueue(final TSDB tsdb) { super(new Cmp(tsdb)); this.tsdb = tsdb; metric_width = tsdb.metrics.width(); + flush_interval = tsdb.config.getInt("tsd.storage.compaction.flush_interval"); + min_flush_threshold = tsdb.config.getInt("tsd.storage.compaction.min_flush_threshold"); + max_concurrent_flushes = tsdb.config.getInt("tsd.storage.compaction.max_concurrent_flushes"); + flush_speed = tsdb.config.getInt("tsd.storage.compaction.flush_speed"); + if (tsdb.config.enable_compactions()) { startCompactionThread(); } @@ -153,8 +170,7 @@ private Deferred> flush(final long cut_off, int maxflushes) { return Deferred.fromResult(new ArrayList(0)); } final ArrayList> ds = - new ArrayList>(Math.min(maxflushes, - MAX_CONCURRENT_FLUSHES)); + new ArrayList>(Math.min(maxflushes, max_concurrent_flushes)); int nflushes = 0; int seed = (int) (System.nanoTime() % 3); for (final byte[] row : this.keySet()) { @@ -164,10 +180,11 @@ private Deferred> flush(final long cut_off, int maxflushes) { if (seed == row.hashCode() % 3) { continue; } - final long base_time = Bytes.getUnsignedInt(row, metric_width); + final long base_time = Bytes.getUnsignedInt(row, + Const.SALT_WIDTH() + metric_width); if (base_time > cut_off) { break; - } else if (nflushes == MAX_CONCURRENT_FLUSHES) { + } else if (nflushes == max_concurrent_flushes) { // We kicked off the compaction of too many rows already, let's wait // until they're done before kicking off more. break; @@ -185,10 +202,10 @@ private Deferred> flush(final long cut_off, int maxflushes) { ds.add(tsdb.get(row).addCallbacks(compactcb, handle_read_error)); } final Deferred> group = Deferred.group(ds); - if (nflushes == MAX_CONCURRENT_FLUSHES && maxflushes > 0) { + if (nflushes == max_concurrent_flushes && maxflushes > 0) { // We're not done yet. Once this group of flushes completes, we need // to kick off more. - tsdb.flush(); // Speed up this batch by telling the client to flush. + tsdb.getClient().flush(); // Speed up this batch by telling the client to flush. final int maxflushez = maxflushes; // Make it final for closure. final class FlushMoreCB implements Callback>, ArrayList> { @@ -218,7 +235,7 @@ public String toString() { private final class CompactCB implements Callback> { @Override public Object call(final ArrayList row) { - return compact(row, null); + return compact(row, null, null, null); } @Override public String toString() { @@ -233,9 +250,10 @@ public String toString() { * @return A compacted version of this row. */ KeyValue compact(final ArrayList row, - List annotations) { + List annotations, + List histograms) { final KeyValue[] compacted = { null }; - compact(row, compacted, annotations); + compact(row, compacted, annotations, histograms); return compacted[0]; } @@ -243,7 +261,7 @@ KeyValue compact(final ArrayList row, * Maintains state for a single compaction; exists to break the steps down into manageable * pieces without having to worry about returning multiple values and passing many parameters * around. - * + * * @since 2.1 */ private class Compaction { @@ -252,6 +270,8 @@ private class Compaction { private final ArrayList row; private final KeyValue[] compacted; private final List annotations; + private final List histograms; + private long compactedKVTimestamp; private final int nkvs; @@ -271,12 +291,18 @@ private class Compaction { // checking if the compacted qualifier already exists. private KeyValue longest; - public Compaction(ArrayList row, KeyValue[] compacted, List annotations) { + // the latest append column. If set then we don't want to re-write the row + // and if we only had a single column with a single value, we return this. + private KeyValue last_append_column; + + public Compaction(ArrayList row, KeyValue[] compacted, List annotations, List histograms) { nkvs = row.size(); this.row = row; this.compacted = compacted; this.annotations = annotations; + this.histograms = histograms; to_delete = new ArrayList(nkvs); + compactedKVTimestamp = Long.MIN_VALUE; } /** @@ -317,6 +343,7 @@ public Deferred compact() { return null; } + compactedKVTimestamp = Long.MIN_VALUE; // go through all the columns, process annotations, and heap = new PriorityQueue(nkvs); int tot_values = buildHeapProcessAnnotations(); @@ -348,7 +375,8 @@ public Deferred compact() { if (compacted != null) { // Caller is interested in the compacted form. compacted[0] = compact; - final long base_time = Bytes.getUnsignedInt(compact.key(), metric_width); + final long base_time = Bytes.getUnsignedInt(compact.key(), + Const.SALT_WIDTH() + metric_width); final long cut_off = System.currentTimeMillis() / 1000 - Const.MAX_TIMESPAN - 1; if (base_time > cut_off) { // If row is too recent... @@ -365,26 +393,31 @@ public Deferred compact() { deleted_cells.addAndGet(to_delete.size()); // We're going to delete this. if (write) { written_cells.incrementAndGet(); - Deferred deferred = tsdb.put(key, compact.qualifier(), compact.value()); + Deferred deferred = tsdb.put(key, compact.qualifier(), compact.value(), compactedKVTimestamp); if (!to_delete.isEmpty()) { deferred = deferred.addCallbacks(new DeleteCompactedCB(to_delete), handle_write_error); } return deferred; - } else { + } else if (last_append_column == null) { // We had nothing to write, because one of the cells is already the // correctly compacted version, so we can go ahead and delete the // individual cells directly. new DeleteCompactedCB(to_delete).call(null); return null; + } else { + return null; } } /** - * Find the first datapoint column in a row. + * Find the first datapoint column in a row. It may be an appended column * * @return the first found datapoint column in the row, or null if none */ private KeyValue findFirstDatapointColumn() { + if (last_append_column != null) { + return last_append_column; + } for (final KeyValue kv : row) { if (isDatapoint(kv)) { return kv; @@ -401,13 +434,37 @@ private KeyValue findFirstDatapointColumn() { */ private int buildHeapProcessAnnotations() { int tot_values = 0; + for (final KeyValue kv : row) { - final byte[] qual = kv.qualifier(); - final int len = qual.length; + byte[] qual = kv.qualifier(); + int len = qual.length; if ((len & 1) != 0) { // process annotations and other extended formats if (qual[0] == Annotation.PREFIX()) { annotations.add(JSON.parseToObject(kv.value(), Annotation.class)); + } else if (qual[0] == HistogramDataPoint.PREFIX) { + try { + HistogramDataPoint histogram = + Internal.decodeHistogramDataPoint(tsdb, kv); + histograms.add(histogram); + } catch (Throwable t) { + LOG.error("Failed to decode histogram data point", t); + } + } else if (qual[0] == AppendDataPoints.APPEND_COLUMN_PREFIX){ + compactedKVTimestamp = Math.max(compactedKVTimestamp, kv.timestamp()); + final AppendDataPoints adp = new AppendDataPoints(); + tot_values += adp.parseKeyValue(tsdb, kv).size(); + last_append_column = new KeyValue(kv.key(), kv.family(), + adp.qualifier(), kv.timestamp(), adp.value()); + if (longest == null || + longest.qualifier().length < last_append_column.qualifier().length) { + longest = last_append_column; + } + final ColumnDatapointIterator col = + new ColumnDatapointIterator(last_append_column); + if (col.hasMoreData()) { + heap.add(col); + } } else { LOG.warn("Ignoring unexpected extended format type " + qual[0]); } @@ -422,6 +479,7 @@ private int buildHeapProcessAnnotations() { longest = kv; } ColumnDatapointIterator col = new ColumnDatapointIterator(kv); + compactedKVTimestamp = Math.max(compactedKVTimestamp, kv.timestamp()); if (col.hasMoreData()) { heap.add(col); } @@ -437,7 +495,59 @@ private int buildHeapProcessAnnotations() { * @param compacted_qual qualifiers for sorted datapoints * @param compacted_val values for sorted datapoints */ - private void mergeDatapoints(ByteBufferList compacted_qual, ByteBufferList compacted_val) { + + private void mergeDatapoints(ByteBufferList compacted_qual, + ByteBufferList compacted_val) { + if (tsdb.getConfig().use_otsdb_timestamp()) { + dtcsMergeDataPoints(compacted_qual, compacted_val); + } else { + defaultMergeDataPoints(compacted_qual, compacted_val); + } + } + + private void dtcsMergeDataPoints(ByteBufferList compacted_qual, ByteBufferList compacted_val) { + // Compare timestamps for two KeyValues at the same time, if they are same compare their values + // Return maximum or minimum value depending upon tsd.storage.use_max_value parameter + // This function is called once for every RowKey, so we only care about comparing offsets, which + // are a part of column qualifier + ColumnDatapointIterator col1 = null; + ColumnDatapointIterator col2 = null; + while (!heap.isEmpty()) { + col1 = heap.remove(); + Pair offsets = col1.getOffsets(); + Pair offsetLengths = col1.getOffsetLengths(); + int ts1 = col1.getTimestampOffsetMs(); + double val1 = col1.getCellValueAsDouble(); + if (col1.advance()) { + heap.add(col1); + } + int ts2 = ts1; + while (ts1 == ts2) { + col2 = heap.peek(); + ts2 = col2 != null ? col2.getTimestampOffsetMs() : ts2; + if (col2 == null || ts1 != ts2) + break; + double val2 = col2.getCellValueAsDouble(); + if ((tsdb.config.use_max_value() && val2 > val1) || (!tsdb.config.use_max_value() && val1 > val2)) { + // Reduce copying of byte arrays by just using col1 variable to reference to either max or min KeyValue + col1 = col2; + val1 = val2; + offsets = col2.getOffsets(); + offsetLengths = col2.getOffsetLengths(); + } + heap.remove(); + if (col2.advance()) { + heap.add(col2); + } + } + col1.writeToBuffersFromOffset(compacted_qual, compacted_val, offsets, offsetLengths); + ms_in_row |= col1.isMilliseconds(); + s_in_row |= !col1.isMilliseconds(); + } + } + + private void defaultMergeDataPoints(ByteBufferList compacted_qual, + ByteBufferList compacted_val) { int prevTs = -1; while (!heap.isEmpty()) { final ColumnDatapointIterator col = heap.remove(); @@ -472,7 +582,6 @@ private void mergeDatapoints(ByteBufferList compacted_qual, ByteBufferList compa } } } - /** * Build the compacted column from the list of byte buffers that were * merged together. @@ -499,17 +608,29 @@ private KeyValue buildCompactedColumn(ByteBufferList compacted_qual, } final KeyValue first = row.get(0); - return new KeyValue(first.key(), first.family(), cq, cv); + if(tsdb.getConfig().getBoolean("tsd.storage.use_otsdb_timestamp")) { + return new KeyValue(first.key(), first.family(), cq, compactedKVTimestamp, cv); + } else { + return new KeyValue(first.key(), first.family(), cq, cv); + } } /** * Make sure we don't delete the row that is the result of the compaction, so we * remove the compacted value from the list of values to delete if it is there. + * Also, if one or more columns were appends then we don't want to mess with + * the row for now. * * @param compact the compacted column * @return true if we need to write the compacted value */ private boolean updateDeletesCheckForWrite(KeyValue compact) { + if (last_append_column != null) { + // TODO appends are involved so we may want to squash dps into the + // append or vice-versa. + return false; + } + // if the longest entry isn't as long as the compacted one, obviously the compacted // one can't have already existed if (longest != null && longest.qualifier().length >= compact.qualifier().length) { @@ -559,8 +680,9 @@ protected static boolean isDatapoint(KeyValue kv) { */ Deferred compact(final ArrayList row, final KeyValue[] compacted, - List annotations) { - return new Compaction(row, compacted, annotations).compact(); + List annotations, + List histograms) { + return new Compaction(row, compacted, annotations, histograms).compact(); } /** @@ -650,21 +772,7 @@ private void startCompactionThread() { thread.start(); } - /** How frequently the compaction thread wakes up flush stuff. */ - // TODO(tsuna): Make configurable? - private static final int FLUSH_INTERVAL = 10; // seconds - /** Minimum number of rows we'll attempt to compact at once. */ - // TODO(tsuna): Make configurable? - private static final int MIN_FLUSH_THRESHOLD = 100; // rows - - /** Maximum number of rows we'll compact concurrently. */ - // TODO(tsuna): Make configurable? - private static final int MAX_CONCURRENT_FLUSHES = 10000; // rows - - /** If this is X then we'll flush X times faster than we really need. */ - // TODO(tsuna): Make configurable? - private static final int FLUSH_SPEED = 2; // multiplicative factor /** * Background thread to trigger periodic compactions. @@ -682,7 +790,7 @@ public void run() { // Flush if we have too many rows to recompact. // Note that in we might not be able to actually // flush anything if the rows aren't old enough. - if (size > MIN_FLUSH_THRESHOLD) { + if (size > min_flush_threshold) { // How much should we flush during this iteration? This scheme is // adaptive and flushes at a rate that is proportional to the size // of the queue, so we flush more aggressively if the queue is big. @@ -701,8 +809,8 @@ public void run() { // FLUSH_SPEED is 2, then instead of taking 1h to flush what we have // for the previous hour, we'll take only 30m. This is desirable so // that we evict old entries from the queue a bit faster. - final int maxflushes = Math.max(MIN_FLUSH_THRESHOLD, - size * FLUSH_INTERVAL * FLUSH_SPEED / Const.MAX_TIMESPAN); + final int maxflushes = Math.max(min_flush_threshold, + size * flush_interval * flush_speed / Const.MAX_TIMESPAN); final long now = System.currentTimeMillis(); flush(now / 1000 - Const.MAX_TIMESPAN - 1, maxflushes); if (LOG.isDebugEnabled()) { @@ -740,7 +848,7 @@ public void run() { return; } try { - Thread.sleep(FLUSH_INTERVAL * 1000); + Thread.sleep(flush_interval * 1000); } catch (InterruptedException e) { LOG.error("Compaction thread interrupted, doing one last flush", e); flush(); @@ -758,16 +866,16 @@ public void run() { */ private static final class Cmp implements Comparator { - /** On how many bytes do we encode metrics IDs. */ - private final short metric_width; + /** The position with which the timestamp of metric starts. */ + private final short timestamp_pos; public Cmp(final TSDB tsdb) { - metric_width = tsdb.metrics.width(); + timestamp_pos = (short) (Const.SALT_WIDTH() + tsdb.metrics.width()); } @Override public int compare(final byte[] a, final byte[] b) { - final int c = Bytes.memcmp(a, b, metric_width, Const.TIMESTAMP_BYTES); + final int c = Bytes.memcmp(a, b, timestamp_pos, Const.TIMESTAMP_BYTES); // If the timestamps are equal, sort according to the entire row key. return c != 0 ? c : Bytes.memcmp(a, b); } diff --git a/src/core/Const.java b/src/core/Const.java index a6cc834e83..472ff5f61e 100644 --- a/src/core/Const.java +++ b/src/core/Const.java @@ -12,6 +12,12 @@ // see . package net.opentsdb.core; +import java.nio.charset.Charset; +import java.util.TimeZone; + +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hashing; + /** Constants used in various places. */ public final class Const { @@ -19,8 +25,38 @@ public final class Const { public static final short TIMESTAMP_BYTES = 4; /** Maximum number of tags allowed per data point. */ - public static final short MAX_NUM_TAGS = 8; - // 8 is an aggressive limit on purpose. Can always be increased later. + private static short MAX_NUM_TAGS = 8; + public static short MAX_NUM_TAGS() { + return MAX_NUM_TAGS; + } + + /** + * -------------- WARNING ---------------- + * Package private method to override the maximum number of tags. + * 8 is an aggressive limit on purpose to avoid performance issues. + * @param tags The number of tags to allow + * @throws IllegalArgumentException if the number of tags is less + * than 1 (OpenTSDB requires at least one tag per metric). + */ + static void setMaxNumTags(final short tags) { + if (tags < 1) { + throw new IllegalArgumentException("tsd.storage.max_tags must be greater than 0"); + } + MAX_NUM_TAGS = tags; + } + + /** The default ASCII character set for encoding tables and qualifiers that + * don't depend on user input that may be encoded with UTF. + * Charset to use with our server-side row-filter. + * We use this one because it preserves every possible byte unchanged. + */ + public static final Charset ASCII_CHARSET = Charset.forName("ISO-8859-1"); + + /** Used for metrics, tags names and tag values */ + public static final Charset UTF8_CHARSET = Charset.forName("UTF8"); + + /** The UTC timezone used for rollup and calendar conversions */ + public static final TimeZone UTC_TZ = TimeZone.getTimeZone("UTC"); /** Number of LSBs in time_deltas reserved for flags. */ public static final short FLAG_BITS = 4; @@ -81,4 +117,71 @@ public final class Const { public static final boolean DONT_CREATE = false; public static final boolean CREATE_IF_NEEDED = true; public static final boolean MUST_BE_WRITEABLE = true; + + /** + * The number of buckets to use for salting. + * WARNING: Changing this after writing data will break TSUID and direct + * queries as the salt calculation will differ. Scanning queries will be OK + * though. + */ + private static int SALT_BUCKETS = 20; + public static int SALT_BUCKETS() { + return SALT_BUCKETS; + } + + /** + * -------------- WARNING ---------------- + * Package private method to override the bucket size. + * ONLY change this value in your configs if you are starting out with a brand + * new install or set of tables. Users wanted this, lets hope they don't + * regret it. + * @param buckets The number of buckets to use. + * @throws IllegalArgumentException if the bucket size is less than 1. You + * *could* have one bucket if you plan to change it later, but *shrug* + */ + static void setSaltBuckets(final int buckets) { + if (buckets < 1) { + throw new IllegalArgumentException("Salt buckets must be greater than 0"); + } + SALT_BUCKETS = buckets; + } + + /** + * Width of the salt in bytes. + * Its width should be proportional to SALT_BUCKETS data type. + * When set to 0, salting is disabled. + * if SALT_WIDTH = 1, the SALT_BUCKETS should be byte + * if SALT_WIDTH = 2, the SALT_BUCKETS can be byte or short + * WARNING: Do NOT change this after you start writing data or you will not + * be able to query for anything. + */ + private static int SALT_WIDTH = 0; + public static int SALT_WIDTH() { + return SALT_WIDTH; + } + + /** + * -------------- WARNING ---------------- + * Package private method to override the salt byte width. + * ONLY change this value in your configs if you are starting out with a brand + * new install or set of tables. Users wanted this, lets hope they don't + * regret it. + * @param buckets The number of bytes of salt to use + * @throws IllegalArgumentException if width < 0 or > 8 + */ + static void setSaltWidth(final int width) { + if (width < 0 || width > 8) { + throw new IllegalArgumentException("Salt width must be between 0 and 8"); + } + SALT_WIDTH = width; + } + + /** + * A global function to use for NON-SECURE hashing of things like queries and + * cache objects. Used for deterministic hashing. + */ + private static HashFunction HASH_FUNCTION = Hashing.murmur3_128(); + public static HashFunction HASH_FUNCTION() { + return HASH_FUNCTION; + } } diff --git a/src/core/DataPoint.java b/src/core/DataPoint.java index cb86c93c1a..7a377a2462 100644 --- a/src/core/DataPoint.java +++ b/src/core/DataPoint.java @@ -53,4 +53,12 @@ public interface DataPoint { */ double toDouble(); + /** + * Represents the number of real values behind this data point when referring + * to a pre-aggregated and/or rolled up value. + * @return The number of real values represented in this data point. Usually + * just 1 for raw values. + */ + long valueCount(); + } diff --git a/src/core/DataPoints.java b/src/core/DataPoints.java index 896499444a..d231f9c1f7 100644 --- a/src/core/DataPoints.java +++ b/src/core/DataPoints.java @@ -1,5 +1,5 @@ // This file is part of OpenTSDB. -// Copyright (C) 2010-2012 The OpenTSDB Authors. +// Copyright (C) 2010-2017 The OpenTSDB Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU Lesser General Public License as published by @@ -15,6 +15,8 @@ import java.util.List; import java.util.Map; +import org.hbase.async.Bytes.ByteMap; + import com.stumbleupon.async.Deferred; import net.opentsdb.meta.Annotation; @@ -36,6 +38,12 @@ public interface DataPoints extends Iterable { * @since 1.2 */ Deferred metricNameAsync(); + + /** + * @return the metric UID + * @since 2.3 + */ + byte[] metricUID(); /** * Returns the tags associated with these data points. @@ -49,6 +57,16 @@ public interface DataPoints extends Iterable { * @since 1.2 */ Deferred> getTagsAsync(); + + /** + * Returns a map of tag pairs as UIDs. + * When used on a span or row, it returns the tag set. When used on a span + * group it will return only the tag pairs that are common across all + * time series in the group. + * @return A potentially empty map of tagk to tagv pairs as UIDs + * @since 2.2 + */ + ByteMap getTagUids(); /** * Returns the tags associated with some but not all of the data points. @@ -81,6 +99,13 @@ public interface DataPoints extends Iterable { */ Deferred> getAggregatedTagsAsync(); + /** + * Returns the tagk UIDs associated with some but not all of the data points. + * @return a non-{@code null} list of tagk UIDs. + * @since 2.3 + */ + List getAggregatedTagUids(); + /** * Returns a list of unique TSUIDs contained in the results * @return an empty list if there were no results, otherwise a list of TSUIDs @@ -185,4 +210,31 @@ public interface DataPoints extends Iterable { */ double doubleValue(int i); + /** + * Return the query index that maps this datapoints to the original TSSubQuery. + * @return index of the query in the TSQuery class + * @throws UnsupportedOperationException if the implementing class can't map + * to a sub query. + * @since 2.2 + */ + int getQueryIndex(); + + /** + * Return whether these data points are the result of the percentile calculation + * on the histogram data points. The client can call {@code getPercentile} to get + * the percentile calculation parameter. + * + * @return true or false + * @since 2.4 + */ + boolean isPercentile(); + + /** + * Return the percentile calculation parameter. This interface and {@code isPercentile} are used + * to convert {@code HistogramDataPoints} to {@code DataPoints} + * + * @return the percentile parameter + * @since 2.4 + */ + float getPercentile(); } diff --git a/src/core/DataPointsIterator.java b/src/core/DataPointsIterator.java index aaf870b113..a766af29f3 100644 --- a/src/core/DataPointsIterator.java +++ b/src/core/DataPointsIterator.java @@ -129,4 +129,10 @@ public String toString() { + ", dp=" + dp + ')'; } + + @Override + public long valueCount() { + return 1; + } + } diff --git a/src/core/Downsampler.java b/src/core/Downsampler.java index bd707d362e..97765a8d70 100644 --- a/src/core/Downsampler.java +++ b/src/core/Downsampler.java @@ -12,22 +12,58 @@ // see . package net.opentsdb.core; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Iterator; +import java.util.List; import java.util.NoSuchElementException; +import net.opentsdb.core.Aggregator.Doubles; +import net.opentsdb.rollup.RollupQuery; +import net.opentsdb.utils.DateTime; /** * Iterator that downsamples data points using an {@link Aggregator}. */ public class Downsampler implements SeekableView, DataPoint { - - /** Function to use for downsampling. */ - private final Aggregator downsampler; + + /** Matches the weekly downsampler as it requires special handling. */ + protected final static int WEEK_UNIT = DateTime.unitsToCalendarType("w"); + protected final static int DAY_UNIT = DateTime.unitsToCalendarType("d"); + protected final static int WEEK_LENGTH = 7; + + /** The downsampling specification when provided */ + protected final DownsamplingSpecification specification; + + /** The start timestamp of the actual query for use with "all" */ + protected final long query_start; + + /** The end timestamp of the actual query for use with "all" */ + protected final long query_end; + + /** The data source */ + protected final SeekableView source; + /** Iterator to iterate the values of the current interval. */ - private final ValuesInInterval values_in_interval; + protected final ValuesInInterval values_in_interval; + /** Last normalized timestamp */ - private long timestamp; + protected long timestamp; + /** Last value as a double */ - private double value; + protected double value; + + /** An optional rollup query. */ + protected RollupQuery rollup_query; + + /** Whether or not to merge all DPs in the source into one vaalue */ + protected final boolean run_all; + + /** The interval to use with a calendar */ + protected final int interval; + + /** The unit to use with a calendar as a Calendar integer */ + protected final int unit; /** * Ctor. @@ -35,25 +71,155 @@ public class Downsampler implements SeekableView, DataPoint { * @param interval_ms The interval in milli seconds wanted between each data * point. * @param downsampler The downsampling function to use. + * @deprecated as of 2.3 */ Downsampler(final SeekableView source, final long interval_ms, final Aggregator downsampler) { - this.values_in_interval = new ValuesInInterval(source, interval_ms); - this.downsampler = downsampler; + this.source = source; + if (downsampler == Aggregators.NONE) { + throw new IllegalArgumentException("cannot use the NONE " + + "aggregator for downsampling"); + } + specification = new DownsamplingSpecification(interval_ms, downsampler, + DownsamplingSpecification.DEFAULT_FILL_POLICY); + values_in_interval = new ValuesInInterval(); + query_start = 0; + query_end = 0; + interval = unit = 0; + run_all = false; + } + + /** + * Ctor. + * @param source The iterator to access the underlying data. + * @param specification The downsampling spec to use + * @param query_start The start timestamp of the actual query for use with "all" + * @param query_end The end timestamp of the actual query for use with "all" + * @since 2.3 + */ + Downsampler(final SeekableView source, + final DownsamplingSpecification specification, + final long query_start, + final long query_end + ) { + this(source, specification, query_start, query_end, null); } + /** + * Ctor. + * @param source The iterator to access the underlying data. + * @param specification The downsampling spec to use + * @param query_start The start timestamp of the actual query for use with "all" + * @param query_end The end timestamp of the actual query for use with "all" + * @param rollup_query An optional rollup query. + * @since 2.4 + */ + Downsampler(final SeekableView source, + final DownsamplingSpecification specification, + final long query_start, + final long query_end, + final RollupQuery rollup_query + ) { + this.source = source; + this.specification = specification; + values_in_interval = new ValuesInInterval(); + this.query_start = query_start; + this.query_end = query_end; + this.rollup_query = rollup_query; + + final String s = specification.getStringInterval(); + if (s != null && s.toLowerCase().contains("all")) { + run_all = true; + interval = unit = 0; + } else if (s != null && specification.useCalendar()) { + if (s.toLowerCase().contains("ms")) { + interval = Integer.parseInt(s.substring(0, s.length() - 2)); + unit = DateTime.unitsToCalendarType(s.substring(s.length() - 2)); + } else { + interval = Integer.parseInt(s.substring(0, s.length() - 1)); + unit = DateTime.unitsToCalendarType(s.substring(s.length() - 1)); + } + run_all = false; + } else { + run_all = false; + interval = unit = 0; + } + } + // ------------------ // // Iterator interface // // ------------------ // + @Override public boolean hasNext() { return values_in_interval.hasNextValue(); } + /** + * @throws NoSuchElementException if no data points remain. + */ + @Override public DataPoint next() { if (hasNext()) { - value = downsampler.runDouble(values_in_interval); + if (rollup_query != null && + (rollup_query.getRollupAgg() == Aggregators.AVG || + rollup_query.getRollupAgg() == Aggregators.DEV)) { + if (rollup_query.getRollupAgg() == Aggregators.AVG) { + if (specification.getFunction() == Aggregators.AVG) { + double sum = 0; + long count = 0; + while (values_in_interval.hasNextValue()) { + count += values_in_interval.nextValueCount(); + sum += values_in_interval.nextDoubleValue(); + } + if (count == 0) { // avoid # / 0 + value = 0; + } else { + value = sum / (double)count; + } + } else { + class Accumulator implements Doubles { + List values = new ArrayList(); + Iterator iterator; + @Override + public boolean hasNextValue() { + return iterator.hasNext(); + } + @Override + public double nextDoubleValue() { + return iterator.next(); + } + } + + final Accumulator accumulator = new Accumulator(); + while (values_in_interval.hasNextValue()) { + long count = values_in_interval.nextValueCount(); + double sum = values_in_interval.nextDoubleValue(); + if (count == 0) { + accumulator.values.add(0D); + } else { + accumulator.values.add(sum / (double) count); + } + } + accumulator.iterator = accumulator.values.iterator(); + value = specification.getFunction().runDouble(accumulator); + } + } else if (rollup_query.getRollupAgg() == Aggregators.DEV) { + throw new UnsupportedOperationException("Standard deviation over " + + "rolled up data is not supported at this time"); + } + } else if (rollup_query != null && + specification.getFunction() == Aggregators.COUNT) { + double count = 0; + while (values_in_interval.hasNextValue()) { + count += values_in_interval.nextDoubleValue(); + } + value = count; + } else { + value = specification.getFunction().runDouble(values_in_interval); + } + timestamp = values_in_interval.getIntervalTimestamp(); values_in_interval.moveToNextInterval(); return this; @@ -61,6 +227,7 @@ public DataPoint next() { throw new NoSuchElementException("no more data points in " + this); } + @Override public void remove() { throw new UnsupportedOperationException(); } @@ -69,53 +236,73 @@ public void remove() { // SeekableView interface // // ---------------------- // + @Override public void seek(final long timestamp) { values_in_interval.seekInterval(timestamp); } - @Override - public String toString() { - final StringBuilder buf = new StringBuilder(); - buf.append("Downsampler: ") - .append("interval_ms=").append(values_in_interval.interval_ms) - .append(", downsampler=").append(downsampler) - .append(", current data=(timestamp=").append(timestamp) - .append(", value=").append(value) - .append("), values_in_interval=").append(values_in_interval); - return buf.toString(); - } + // ------------------- // + // DataPoint interface // + // ------------------- // + @Override public long timestamp() { + if (run_all) { + return query_start; + } return timestamp; } + @Override public boolean isInteger() { return false; } + @Override public long longValue() { throw new ClassCastException("Downsampled values are doubles"); } + @Override public double doubleValue() { return value; } + @Override public double toDouble() { return value; } - + + @Override + public String toString() { + final StringBuilder buf = new StringBuilder(); + buf.append("Downsampler: ") + .append(", downsampler=").append(specification) + .append(", rollupQuery=").append(rollup_query) + .append(", queryStart=").append(query_start) + .append(", queryEnd=").append(query_end) + .append(", runAll=").append(run_all) + .append(", current data=(timestamp=").append(timestamp) + .append(", value=").append(value) + .append("), values_in_interval=").append(values_in_interval); + return buf.toString(); + } + /** Iterates source values for an interval. */ - private static class ValuesInInterval implements Aggregator.Doubles { + protected class ValuesInInterval implements Aggregator.Doubles { - /** The iterator of original source values. */ - private final SeekableView source; - /** The sampling interval in milliseconds. */ - private final long interval_ms; + /** An optional calendar set to the current timestamp for the data point */ + private Calendar previous_calendar; + + /** An optional calendar set to the end of the interval timestamp */ + private Calendar next_calendar; + /** The end of the current interval. */ private long timestamp_end_interval = Long.MIN_VALUE; + /** True if the last value was successfully extracted from the source. */ private boolean has_next_value_from_source = false; + /** The last data point extracted from the source. */ private DataPoint next_dp = null; @@ -124,24 +311,42 @@ private static class ValuesInInterval implements Aggregator.Doubles { /** * Constructor. - * @param source The iterator to access the underlying data. - * @param interval_ms Downsampling interval. */ - ValuesInInterval(final SeekableView source, final long interval_ms) { - this.source = source; - this.interval_ms = interval_ms; - this.timestamp_end_interval = interval_ms; + protected ValuesInInterval() { + if (run_all) { + timestamp_end_interval = query_end; + } else if (!specification.useCalendar()) { + timestamp_end_interval = specification.getInterval(); + } } /** Initializes to iterate intervals. */ - private void initializeIfNotDone() { + protected void initializeIfNotDone() { // NOTE: Delay initialization is required to not access any data point // from the source until a user requests it explicitly to avoid the severe // performance penalty by accessing the unnecessary first data of a span. if (!initialized) { initialized = true; - moveToNextValue(); - resetEndOfInterval(); + if (source.hasNext()) { + moveToNextValue(); + if (!run_all) { + if (specification.useCalendar()) { + previous_calendar = DateTime.previousInterval(next_dp.timestamp(), + interval, unit, specification.getTimezone()); + next_calendar = DateTime.previousInterval(next_dp.timestamp(), + interval, unit, specification.getTimezone()); + if (unit == WEEK_UNIT) { + next_calendar.add(DAY_UNIT, interval * WEEK_LENGTH); + } else { + next_calendar.add(unit, interval); + } + timestamp_end_interval = next_calendar.getTimeInMillis(); + } else { + timestamp_end_interval = alignTimestamp(next_dp.timestamp()) + + specification.getInterval(); + } + } + } } } @@ -149,7 +354,25 @@ private void initializeIfNotDone() { private void moveToNextValue() { if (source.hasNext()) { has_next_value_from_source = true; - next_dp = source.next(); + // filter out dps that don't match start and end for run_alls + if (run_all) { + while (source.hasNext()) { + next_dp = source.next(); + if (next_dp.timestamp() < query_start) { + next_dp = null; + continue; + } + if (next_dp.timestamp() >= query_end) { + has_next_value_from_source = false; + } + break; + } + if (next_dp == null) { + has_next_value_from_source = false; + } + } else { + next_dp = source.next(); + } } else { has_next_value_from_source = false; } @@ -160,10 +383,22 @@ private void moveToNextValue() { * the next value read from source. It is the first value of the next * interval. */ private void resetEndOfInterval() { - if (has_next_value_from_source) { - // Sets the end of the interval of the timestamp. - timestamp_end_interval = alignTimestamp(next_dp.timestamp()) + - interval_ms; + if (has_next_value_from_source && !run_all) { + if (specification.useCalendar()) { + while (next_dp.timestamp() >= timestamp_end_interval) { + if (unit == WEEK_UNIT) { + previous_calendar.add(DAY_UNIT, interval * WEEK_LENGTH); + next_calendar.add(DAY_UNIT, interval * WEEK_LENGTH); + } else { + previous_calendar.add(unit, interval); + next_calendar.add(unit, interval); + } + timestamp_end_interval = next_calendar.getTimeInMillis(); + } + } else { + timestamp_end_interval = alignTimestamp(next_dp.timestamp()) + + specification.getInterval(); + } } } @@ -174,28 +409,50 @@ void moveToNextInterval() { } /** Advances the interval iterator to the given timestamp. */ - void seekInterval(long timestamp) { + void seekInterval(final long timestamp) { // To make sure that the interval of the given timestamp is fully filled, // rounds up the seeking timestamp to the smallest timestamp that is // a multiple of the interval and is greater than or equal to the given // timestamp.. - source.seek(alignTimestamp(timestamp + interval_ms - 1)); + if (run_all) { + source.seek(timestamp); + } else if (specification.useCalendar()) { + final Calendar seek_calendar = DateTime.previousInterval( + timestamp, interval, unit, specification.getTimezone()); + if (timestamp > seek_calendar.getTimeInMillis()) { + if (unit == WEEK_UNIT) { + seek_calendar.add(DAY_UNIT, interval * WEEK_LENGTH); + } else { + seek_calendar.add(unit, interval); + } + } + source.seek(seek_calendar.getTimeInMillis()); + } else { + source.seek(alignTimestamp(timestamp + specification.getInterval() - 1)); + } initialized = false; } /** Returns the representative timestamp of the current interval. */ - private long getIntervalTimestamp() { + protected long getIntervalTimestamp() { // NOTE: It is well-known practice taking the start time of // a downsample interval as a representative timestamp of it. It also // provides the correct context for seek. - return alignTimestamp(timestamp_end_interval - interval_ms); + if (run_all) { + return timestamp_end_interval; + } else if (specification.useCalendar()) { + return previous_calendar.getTimeInMillis(); + } else { + return alignTimestamp(timestamp_end_interval - + specification.getInterval()); + } } /** Returns timestamp aligned by interval. */ - private long alignTimestamp(long timestamp) { - return timestamp - (timestamp % interval_ms); + protected long alignTimestamp(final long timestamp) { + return timestamp - (timestamp % specification.getInterval()); } - + // ---------------------- // // Doubles interface // // ---------------------- // @@ -203,6 +460,9 @@ private long alignTimestamp(long timestamp) { @Override public boolean hasNextValue() { initializeIfNotDone(); + if (run_all) { + return has_next_value_from_source; + } return has_next_value_from_source && next_dp.timestamp() < timestamp_end_interval; } @@ -218,14 +478,28 @@ public double nextDoubleValue() { + timestamp_end_interval); } + // call me BEFORE you call nextDoubleValue + public long nextValueCount() { + if (hasNextValue()) { + if (next_dp != null) { + return next_dp.valueCount(); + } + } + throw new NoSuchElementException("no more values in interval of " + + timestamp_end_interval); + } + @Override public String toString() { final StringBuilder buf = new StringBuilder(); buf.append("ValuesInInterval: ") - .append("interval_ms=").append(interval_ms) .append(", timestamp_end_interval=").append(timestamp_end_interval) .append(", has_next_value_from_source=") - .append(has_next_value_from_source); + .append(has_next_value_from_source) + .append(", previousCalendar=") + .append(previous_calendar == null ? "null" : previous_calendar) + .append(", nextCalendar=") + .append(next_calendar == null ? "null" : next_calendar); if (has_next_value_from_source) { buf.append(", nextValue=(").append(next_dp).append(')'); } @@ -233,4 +507,9 @@ public String toString() { return buf.toString(); } } + + @Override + public long valueCount() { + throw new UnsupportedOperationException(); + } } diff --git a/src/core/DownsamplingSpecification.java b/src/core/DownsamplingSpecification.java new file mode 100644 index 0000000000..0d38404adb --- /dev/null +++ b/src/core/DownsamplingSpecification.java @@ -0,0 +1,267 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import java.util.NoSuchElementException; +import java.util.TimeZone; + +import com.google.common.base.MoreObjects; +import net.opentsdb.utils.DateTime; + +/** + * Representation of a downsampling specification in a TSDB query. + * @since 2.2 + */ +public final class DownsamplingSpecification { + /** Instance of a specification indicating no downsampling requested. */ + public static final DownsamplingSpecification NO_DOWNSAMPLER = + new DownsamplingSpecification(); + + /** Special value representing no downsampling interval given. */ + public static final long NO_INTERVAL = 0L; + + /** Special value representing no downsampling function given. */ + public static final Aggregator NO_FUNCTION = null; + + /** The default fill policy. */ + public static final FillPolicy DEFAULT_FILL_POLICY = FillPolicy.NONE; + + public static final HistogramAggregation NO_HIST_AGG = null; + + // Parsed downsample interval. + private final long interval; + + //The string interval, e.g. 1h, 30d, etc + private final String string_interval; + + // Parsed downsampler function. + private final Aggregator function; + + // Parsed fill policy: whether to interpolate or to fill. + private final FillPolicy fill_policy; + + // Whether or not to use the calendar for intervals + private boolean use_calendar; + + // The user provided timezone for calendar alignment (defaults to UTC) + private TimeZone timezone; + + private final HistogramAggregation hist_agg; + + /** + * A specification indicating no downsampling is requested. + */ + private DownsamplingSpecification() { + interval = NO_INTERVAL; + function = NO_FUNCTION; + fill_policy = DEFAULT_FILL_POLICY; + string_interval = null; + use_calendar = false; + timezone = DateTime.timezones.get(DateTime.UTC_ID); + hist_agg = NO_HIST_AGG; + } + + /** + * Non-stringified, piecewise c-tor. + * @param interval The downsampling interval, in milliseconds. + * @param function The downsampling function. + * @param fill_policy The policy specifying how to deal with missing data. + * @throws IllegalArgumentException if any argument is invalid. + * @deprecated since 2.3 + */ + public DownsamplingSpecification(final long interval, + final Aggregator function, final FillPolicy fill_policy) { + if (null == function) { + throw new IllegalArgumentException("downsampling function cannot be null"); + } + if (interval <= 0L) { + throw new IllegalArgumentException("interval not > 0: " + interval); + } + if (null == fill_policy) { + throw new IllegalArgumentException("fill policy cannot be null"); + } + if (function == Aggregators.NONE) { + throw new IllegalArgumentException("cannot use the NONE " + + "aggregator for downsampling"); + } + + this.interval = interval; + this.function = function; + this.fill_policy = fill_policy; + string_interval = null; + use_calendar = false; + timezone = DateTime.timezones.get(DateTime.UTC_ID); + hist_agg = NO_HIST_AGG; + } + + /** + * C-tor for string representations. + * The argument to this c-tor should have the following format: + * {@code interval-function[-fill_policy]}. + * This ctor supports the "all" flag to downsample to a single value as well + * as units suffixed with 'c' to use the calendar for downsample alignment. + * @param specification String representation of a downsample specifier. + * @throws IllegalArgumentException if the specification is null or invalid. + */ + public DownsamplingSpecification(final String specification) { + if (null == specification) { + throw new IllegalArgumentException("Downsampling specifier cannot be " + + "null"); + } + + final String[] parts = specification.split("-"); + if (parts.length < 2) { + // Too few items. + throw new IllegalArgumentException("Invalid downsampling specifier '" + + specification + "': must provide at least interval and function"); + } else if (parts.length > 3) { + // Too many items. + throw new IllegalArgumentException("Invalid downsampling specifier '" + + specification + "': must consist of interval, function, and optional " + + "fill policy"); + } + + // This porridge is just right. + + // INTERVAL. + // This will throw if interval is invalid. + if (parts[0].contains("all")) { + interval = NO_INTERVAL; + use_calendar = false; + string_interval = parts[0]; + } else if (parts[0].charAt(parts[0].length() - 1) == 'c') { + final String duration = parts[0].substring(0, parts[0].length() - 1); + interval = DateTime.parseDuration(duration); + string_interval = duration; + use_calendar = true; + } else { + interval = DateTime.parseDuration(parts[0]); + use_calendar = false; + string_interval = parts[0]; + } + + if (parts[1].toLowerCase().equals("sum")) { + hist_agg = HistogramAggregation.SUM; + } else { + hist_agg = null; + } + + // FUNCTION. + try { + function = Aggregators.get(parts[1]); + } catch (final NoSuchElementException e) { + throw new IllegalArgumentException("No such downsampling function: " + + parts[1]); + } + if (function == Aggregators.NONE) { + throw new IllegalArgumentException("cannot use the NONE " + + "aggregator for downsampling"); + } + + // FILL POLICY. + if (3 == parts.length) { + // If the user gave us three parts, then the third must be a fill + // policy. + fill_policy = FillPolicy.fromString(parts[2]); + if (null == fill_policy) { + final StringBuilder oss = new StringBuilder(); + oss.append("No such fill policy: '").append(parts[2]) + .append("': must be one of:"); + for (final FillPolicy policy : FillPolicy.values()) { + oss.append(" ").append(policy.getName()); + } + + throw new IllegalArgumentException(oss.toString()); + } + } else { + // Default to linear interpolation. + fill_policy = FillPolicy.NONE; + } + timezone = DateTime.timezones.get(DateTime.UTC_ID); + } + + /** @param use_calendar Whether or not to use the calendar when downsampling + * @since 2.3 */ + public void setUseCalendar(final boolean use_calendar) { + this.use_calendar = use_calendar; + } + + /** @param timezone The timezone to use when downsampling on calendar + * boundaries. + * @since 2.3 */ + public void setTimezone(final TimeZone timezone) { + if (timezone == null) { + throw new IllegalArgumentException("Timezone cannot be null"); + } + this.timezone = timezone; + } + + /** + * Get the downsampling interval, in milliseconds. + * @return the downsampling interval, in milliseconds. + */ + public long getInterval() { + return interval; + } + + /** @return The string interval from the user (without the 'c' if given) + * @since 2.3 */ + public String getStringInterval() { + return string_interval; + } + + /** + * Get the downsampling function. + * @return the downsampling function. + */ + public Aggregator getFunction() { + return function; + } + + /** + * Get the policy specifying how to deal with missing data. + * @return the policy specifying how to deal with missing data. + */ + public FillPolicy getFillPolicy() { + return fill_policy; + } + + /** @return Whether or not to use the calendar when downsampling + * @since 2.3 */ + public boolean useCalendar() { + return use_calendar; + } + + /** @return The timezone to use when downsampling on calendar boundaries. + * @since 2.3 */ + public TimeZone getTimezone() { + return timezone; + } + + public HistogramAggregation getHistogramAggregation() { + return hist_agg; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("interval", getInterval()) + .add("function", getFunction()) + .add("fillPolicy", getFillPolicy()) + .add("stringInterval", string_interval) + .add("useCalendar", useCalendar()) + .add("timeZone", getTimezone() != null ? getTimezone().getID() : null) + .toString(); + } +} + diff --git a/src/core/FillPolicy.java b/src/core/FillPolicy.java new file mode 100644 index 0000000000..09c145f284 --- /dev/null +++ b/src/core/FillPolicy.java @@ -0,0 +1,63 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; + +/** + * Specification of how to deal with missing intervals when downsampling. + * @since 2.2 + */ +public enum FillPolicy { + NONE("none"), + ZERO("zero"), + NOT_A_NUMBER("nan"), + NULL("null"), + SCALAR("scalar"); + + // The user-friendly name of this policy. + private final String name; + + FillPolicy(final String name) { + this.name = name; + } + + /** + * Get this fill policy's user-friendly name. + * @return this fill policy's user-friendly name. + */ + @JsonValue + public String getName() { + return name; + } + + /** + * Get an instance of this enumeration from a user-friendly name. + * @param name The user-friendly name of a fill policy. + * @return an instance of {@link FillPolicy}, or {@code null} if the name + * does not match any instance. + * @throws IllegalArgumentException if the name doesn't match a policy + */ + @JsonCreator + public static FillPolicy fromString(final String name) { + for (final FillPolicy policy : FillPolicy.values()) { + if (policy.name.equalsIgnoreCase(name)) { + return policy; + } + } + + throw new IllegalArgumentException("Unrecognized fill policy: " + name); + } +} + diff --git a/src/core/FillingDownsampler.java b/src/core/FillingDownsampler.java new file mode 100644 index 0000000000..5edf23509b --- /dev/null +++ b/src/core/FillingDownsampler.java @@ -0,0 +1,310 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; + +import net.opentsdb.core.Aggregator.Doubles; +import net.opentsdb.rollup.RollupQuery; +import net.opentsdb.utils.DateTime; + +/** + * A specialized downsampler that returns special values, based on the fill + * policy, for intervals for which no data could be found. The default + * implementation, {@link Downsampler}, simply skips intervals that have no + * data, which causes the {@link AggregationIterator} up the chain to + * interpolate. + * @since 2.2 + */ +public class FillingDownsampler extends Downsampler { + /** Track when the downsampled data should end. */ + protected long end_timestamp; + + /** An optional calendar set to the current timestamp for the data point */ + private final Calendar previous_calendar; + + /** An optional calendar set to the end of the interval timestamp */ + private final Calendar next_calendar; + + /** + * Create a new nulling downsampler. + * @param source The iterator to access the underlying data. + * @param start_time The time in milliseconds at which the data begins. + * @param end_time The time in milliseconds at which the data ends. + * @param interval_ms The interval in milli seconds wanted between each data + * point. + * @param downsampler The downsampling function to use. + * @param fill_policy Policy specifying whether to interpolate or to fill + * missing intervals with special values. + * @throws IllegalArgumentException if fill_policy is interpolation. + * @deprecated as of 2.3 + */ + FillingDownsampler(final SeekableView source, final long start_time, + final long end_time, final long interval_ms, + final Aggregator downsampler, final FillPolicy fill_policy) { + this(source, start_time, end_time, + new DownsamplingSpecification(interval_ms, downsampler, fill_policy) + , 0, 0); + } + + /** + * Create a new filling downsampler. + * @param source The iterator to access the underlying data. + * @param start_time The time in milliseconds at which the data begins. + * @param end_time The time in milliseconds at which the data ends. + * @param specification The downsampling spec to use + * @param query_start The start timestamp of the actual query for use with "all" + * @param query_end The end timestamp of the actual query for use with "all" + * @throws IllegalArgumentException if fill_policy is interpolation. + * @since 2.3 + */ + FillingDownsampler(final SeekableView source, final long start_time, + final long end_time, final DownsamplingSpecification specification, + final long query_start, final long end_start) { + this(source, start_time, end_time, specification, query_start, end_start, + null); + } + + /** + * Create a new filling downsampler. + * @param source The iterator to access the underlying data. + * @param start_time The time in milliseconds at which the data begins. + * @param end_time The time in milliseconds at which the data ends. + * @param specification The downsampling spec to use + * @param query_start The start timestamp of the actual query for use with "all" + * @param query_end The end timestamp of the actual query for use with "all" + * @param rollup_query An optional rollup query. + * @throws IllegalArgumentException if fill_policy is interpolation. + * @since 2.4 + */ + FillingDownsampler(final SeekableView source, final long start_time, + final long end_time, final DownsamplingSpecification specification, + final long query_start, final long end_start, + final RollupQuery rollup_query) { + // Lean on the superclass implementation. + super(source, specification, query_start, end_start, rollup_query); + + // Ensure we aren't given a bogus fill policy. + if (FillPolicy.NONE == specification.getFillPolicy()) { + throw new IllegalArgumentException("Cannot instantiate this class with" + + " linear-interpolation fill policy"); + } + + // Use the values-in-interval object to align the timestamps at which we + // expect data to arrive for the first and last intervals. + if (run_all) { + timestamp = start_time; + end_timestamp = end_time; + previous_calendar = next_calendar = null; + } else if (specification.useCalendar()) { + previous_calendar = DateTime.previousInterval(start_time, interval, unit, + specification.getTimezone()); + if (unit == WEEK_UNIT) { + previous_calendar.add(DAY_UNIT, -(interval * WEEK_LENGTH)); + } else { + previous_calendar.add(unit, -interval); + } + next_calendar = DateTime.previousInterval(start_time, interval, unit, + specification.getTimezone()); + + final Calendar end_calendar = DateTime.previousInterval( + end_time, interval, unit, specification.getTimezone()); + if (end_calendar.getTimeInMillis() == next_calendar.getTimeInMillis()) { + // advance once + if (unit == WEEK_UNIT) { + end_calendar.add(DAY_UNIT, interval * WEEK_LENGTH); + } else { + end_calendar.add(unit, interval); + } + } + timestamp = next_calendar.getTimeInMillis(); + end_timestamp = end_calendar.getTimeInMillis(); + } else { + // Use the values-in-interval object to align the timestamps at which we + // expect data to arrive for the first and last intervals. + timestamp = values_in_interval.alignTimestamp(start_time); + end_timestamp = values_in_interval.alignTimestamp(end_time); + previous_calendar = next_calendar = null; + } + } + + /** + * Please note that when this method returns true, the value yielded by the + * object returned by {@link #next()} might be NaN, which indicates no data + * could be found for the current interval. + * @return true if this iterator has not yet reached the end of the specified + * range of data; otherwise, false. + */ + @Override + public boolean hasNext() { + // No matter the state of the values-in-interval object, if our current + // timestamp hasn't reached the end of the requested overall interval, then + // we still have iterating to do. + if (run_all) { + return values_in_interval.hasNextValue(); + } + return timestamp < end_timestamp; + } + + /** + * Please note that the object returned by this method may return the value + * NaN, which indicates that no data count be found for the interval. This is + * intentional. Future intervals, if any, may still hava data and thus yield + * non-NaN values. + * @return the next data point, which might yield a NaN value. + * @throws NoSuchElementException if no more intervals remain. + */ + @Override + public DataPoint next() { + // Don't proceed if we've already completed iteration. + if (hasNext()) { + // Ensure that the timestamp we request is valid. + values_in_interval.initializeIfNotDone(); + + // Skip any leading data outside the query bounds. + long actual = values_in_interval.hasNextValue() ? + values_in_interval.getIntervalTimestamp() : Long.MAX_VALUE; + + while (!run_all && values_in_interval.hasNextValue() + && actual < timestamp) { + // The actual timestamp precedes our expected, so there's data in the + // values-in-interval object that we wish to ignore. + specification.getFunction().runDouble(values_in_interval); + values_in_interval.moveToNextInterval(); + actual = values_in_interval.getIntervalTimestamp(); + } + + // Check whether the timestamp of the calculation interval matches what + // we expect. + if (run_all || actual == timestamp) { + // The calculated interval timestamp matches what we expect, so we can + // do normal processing. + if (rollup_query != null && + (rollup_query.getRollupAgg() == Aggregators.AVG || + rollup_query.getRollupAgg() == Aggregators.DEV)) { + if (rollup_query.getRollupAgg() == Aggregators.AVG) { + if (specification.getFunction() == Aggregators.AVG) { + double sum = 0; + long count = 0; + while (values_in_interval.hasNextValue()) { + count += values_in_interval.nextValueCount(); + sum += values_in_interval.nextDoubleValue(); + } + if (count == 0) { // avoid # / 0 + value = 0; + } else { + value = sum / (double)count; + } + } else { + class Accumulator implements Doubles { + List values = new ArrayList(); + Iterator iterator; + @Override + public boolean hasNextValue() { + return iterator.hasNext(); + } + @Override + public double nextDoubleValue() { + return iterator.next(); + } + } + + final Accumulator accumulator = new Accumulator(); + while (values_in_interval.hasNextValue()) { + long count = values_in_interval.nextValueCount(); + double sum = values_in_interval.nextDoubleValue(); + if (count == 0) { + accumulator.values.add(0D); + } else { + accumulator.values.add(sum / (double) count); + } + } + accumulator.iterator = accumulator.values.iterator(); + value = specification.getFunction().runDouble(accumulator); + } + } else if (specification.getFunction() == Aggregators.DEV) { + throw new UnsupportedOperationException("Standard deviation over " + + "rolled up data is not supported at this time"); + } + } else if (rollup_query != null && + specification.getFunction() == Aggregators.COUNT) { + double count = 0; + while (values_in_interval.hasNextValue()) { + count += values_in_interval.nextDoubleValue(); + } + value = count; + } else { + value = specification.getFunction().runDouble(values_in_interval); + } + values_in_interval.moveToNextInterval(); + } else { + // Our expected timestamp precedes the actual, so the interval is + // missing. We will use a special value, based on the fill policy, to + // represent this case. + switch (specification.getFillPolicy()) { + case NOT_A_NUMBER: + case NULL: + value = Double.NaN; + break; + + case ZERO: + value = 0.0; + break; + + // TODO - scalar + + default: + throw new RuntimeException("unhandled fill policy"); + } + } + + // Advance the expected timestamp to the next interval. + if (!run_all) { + if (specification.useCalendar()) { + if (unit == WEEK_UNIT) { + previous_calendar.add(DAY_UNIT, interval * WEEK_LENGTH); + next_calendar.add(DAY_UNIT, interval * WEEK_LENGTH); + } else { + previous_calendar.add(unit, interval); + next_calendar.add(unit, interval); + } + timestamp = next_calendar.getTimeInMillis(); + } else { + timestamp += specification.getInterval(); + } + } + + // This object also represents the data. + return this; + } + + // Ideally, the user will not call this method when no data remains, but + // we can't enforce that. + throw new NoSuchElementException("no more data points in " + this); + } + + @Override + public long timestamp() { + if (run_all) { + return query_start; + } else if (specification.useCalendar()) { + return previous_calendar.getTimeInMillis(); + } + return timestamp - specification.getInterval(); + } +} + diff --git a/src/core/GroupCallback.java b/src/core/GroupCallback.java new file mode 100644 index 0000000000..da835dbd63 --- /dev/null +++ b/src/core/GroupCallback.java @@ -0,0 +1,30 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2010-2012 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import com.stumbleupon.async.Callback; + +import java.util.ArrayList; + +class GroupCallback implements Callback> { + /** + * We're only waiting for all callbacks to complete, ignoring their return values. + * + * @param ignored The return values of the individual callbacks - ignored + * @return null + */ + @Override + public Object call(ArrayList ignored) { + return null; + } +} diff --git a/src/core/Histogram.java b/src/core/Histogram.java new file mode 100644 index 0000000000..94bd2d7a5c --- /dev/null +++ b/src/core/Histogram.java @@ -0,0 +1,37 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import java.util.List; +import java.util.Map; + +public interface Histogram { + + public byte[] histogram(final boolean include_id); + + public void fromHistogram(final byte[] raw, final boolean includes_id); + + public double percentile(final double p); + + public List percentiles(List p); + + public Map getHistogram(); + + public Histogram clone(); + + public int getId(); + + void aggregate(Histogram histo, HistogramAggregation func); + + void aggregate(List histos, HistogramAggregation func); +} diff --git a/src/core/HistogramAggregation.java b/src/core/HistogramAggregation.java new file mode 100644 index 0000000000..aa859af7c5 --- /dev/null +++ b/src/core/HistogramAggregation.java @@ -0,0 +1,22 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +/** + * Aggregator functions for histogram data points. + * + * @since 2.4 + */ +public enum HistogramAggregation { + SUM; +} diff --git a/src/core/HistogramAggregationIterator.java b/src/core/HistogramAggregationIterator.java new file mode 100644 index 0000000000..4e1d97acc2 --- /dev/null +++ b/src/core/HistogramAggregationIterator.java @@ -0,0 +1,319 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . + +package net.opentsdb.core; + +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + * This is where the real business of @{link HistogramSpanGroup}. It provides a + * merged and aggregated view of the data points. It will apply the following + * processing: + *
      + *
    • Down sampling + *
    • Aggregation + *
    + * + * The following processing is inapplicable: + *
      + *
    • Interpolation + *
    • Rate Calculation + *
    + * + * @since 2.4 + */ +public class HistogramAggregationIterator implements + HistogramSeekableView, HistogramDataPoint { + private static final Logger LOG = LoggerFactory.getLogger( + HistogramAggregationIterator.class); + + /** Aggregator to use to aggregate histogram data points from different + * HistogramSpans. */ + private final HistogramAggregation aggregation; + + /** + * Where we are in each {@link HistogramSpan} in the group. + */ + private final HistogramSeekableView[] iterators; + + /** + * Start time (UNIX timestamp in seconds or ms) on 32 bits ("unsigned" int). + */ + private final long start_time; + + /** End time (UNIX timestamp in seconds or ms) on 32 bits ("unsigned" int). */ + private final long end_time; + + /** + * The next timestamps for the data points being used + */ + private final long[] timestamps; + + /** + * The next values for the data points being used. + */ + private final HistogramDataPoint[] values; + + /** + * The current value + */ + private HistogramDataPoint value; + + /** + * Cotr. + * + * @param spans The spans that join the aggregation + * @param start_time Any data point strictly before this timestamp will be ignored. + * @param end_time Any data point strictly after this timestamp will be ignored. + * @param aggregation The aggregation will be applied on the spans + * @param downsampler The downsamper will be applied on each span + * @param query_start The start time of the actual query + * @param query_end The end time of the actual query + * @param is_rollup Whether we are handling the rollup data points + * @return + */ + public static HistogramAggregationIterator create(final List spans, + final long start_time, + final long end_time, + final HistogramAggregation aggregation, + final DownsamplingSpecification downsampler, + final long query_start, + final long query_end, + final boolean is_rollup) { + final int size = spans.size(); + final HistogramSeekableView[] iterators = new HistogramSeekableView[size]; + for (int i = 0; i < size; i++) { + HistogramSeekableView it; + if (downsampler == DownsamplingSpecification.NO_DOWNSAMPLER) { + it = spans.get(i).spanIterator(); + } else { + it = spans.get(i).downsampler(start_time, end_time, downsampler, + is_rollup, query_start, query_end); + } + iterators[i] = it; + } + return new HistogramAggregationIterator(iterators, start_time, + end_time, aggregation); + } + + private HistogramAggregationIterator(final HistogramSeekableView[] iterators, + final long start_time, + final long end_time, + final HistogramAggregation aggregation) { + this.iterators = iterators; + this.start_time = start_time; + this.end_time = end_time; + this.aggregation = aggregation; + + timestamps = new long[this.iterators.length]; + values = new HistogramDataPoint[this.iterators.length]; + + // Initialize every Iterator, fetch their first values that fall + // within our time range. + int num_empty_spans = 0; + for (int i = 0; i < this.iterators.length; i++) { + HistogramSeekableView it = iterators[i]; + it.seek(this.start_time); + + final HistogramDataPoint dp; + if (!it.hasNext()) { + ++num_empty_spans; + endReached(i); + continue; + } + + dp = it.next(); + if (dp.timestamp() >= this.start_time) { + putDataPoint(i, dp); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("No DP in range for #%d: %d < %d", i, + dp.timestamp(), start_time)); + } + endReached(i); + continue; + } + } // end for + + if (num_empty_spans > 0) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("%d out of %d spans are empty!", + num_empty_spans, this.iterators.length)); + } + } + } + + /** + * Indicates that an iterator in {@link #iterators} has reached the end. + * + * @param i The index in {@link #iterators} of the iterator. + */ + private void endReached(final int i) { + timestamps[i] = 0; + iterators[i] = null; // We won't use it anymore, so free() it. + } + + /** + * Puts the next data point of an iterator in the internal buffer. + * + * @param i The index in {@link #iterators} of the iterator. + * @param dp The last data point returned by that iterator. + */ + private void putDataPoint(final int i, final HistogramDataPoint dp) { + timestamps[i] = dp.timestamp(); + values[i] = dp.clone(); + } + + @Override + public long timestamp() { + return value.timestamp(); + } + + @Override + public byte[] getRawData(final boolean include_id) { + return value.getRawData(include_id); + } + + @Override + public void resetFromRawData(byte[] raw_data, final boolean includes_id) { + value.resetFromRawData(raw_data, includes_id); + } + + @Override + public double percentile(double p) { + return value.percentile(p); + } + + @Override + public List percentile(List p) { + return value.percentile(p); + } + + @Override + public void aggregate(HistogramDataPoint histo, HistogramAggregation func) { + value.aggregate(histo, func); + } + + @Override + public HistogramDataPoint clone() { + return value.clone(); + } + + @Override + public HistogramDataPoint cloneAndSetTimestamp(long timestamp) { + return value.cloneAndSetTimestamp(timestamp); + } + + @Override + public int getId() { + return value.getId(); + } + + @Override + public boolean hasNext() { + for (int i = 0; i < iterators.length; ++i) { + if (0 != this.timestamps[i] && this.timestamps[i] <= this.end_time) { + return true; + } + } + + return false; + } + + @Override + public HistogramDataPoint next() { + if (!hasNext()) { + throw new NoSuchElementException("no more elements"); + } + + long min_ts = Long.MAX_VALUE; + int fist_min_ts_index = -1; + boolean is_multiple = false; + + // find out the data points with the smallest timestamp + for (int i = 0; i < this.iterators.length; ++i) { + if (0 == this.timestamps[i]) { + continue; + } + + if (this.timestamps[i] > this.end_time) { + continue; + } + + if (this.timestamps[i] < min_ts) { + min_ts = this.timestamps[i]; + fist_min_ts_index = i; + is_multiple = false; + } else if (this.timestamps[i] == min_ts) { + is_multiple = true; + } + } // end for + + if (fist_min_ts_index < 0) { + throw new NoSuchElementException("no more elements"); + } + + // do the aggregation on the data points with the smallest timestamp + this.value = this.values[fist_min_ts_index]; + if (is_multiple) { + for (int i = fist_min_ts_index + 1; i < this.iterators.length; ++i) { + if (this.timestamps[i] == min_ts) { + this.value.aggregate(this.values[i], this.aggregation); + + // move to next data point on this span + moveToNext(i); + } + } // end for + } + + moveToNext(fist_min_ts_index); + return this; + } + + /** + * Makes iterator number {@code i} move forward to the next data point. + * + * @param i The index in {@link #iterators} of the iterator. + */ + private void moveToNext(final int i) { + final HistogramSeekableView it = iterators[i]; + if (it.hasNext()) { + putDataPoint(i, it.next()); + } else { + endReached(i); + } + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + @Override + public void seek(long timestamp) { + for (final HistogramSeekableView it : iterators) { + it.seek(timestamp); + } + } + + @Override + public Map getHistogramBucketsIfHas() { + return this.value.getHistogramBucketsIfHas(); + } +} diff --git a/src/core/HistogramAggregator.java b/src/core/HistogramAggregator.java new file mode 100644 index 0000000000..4f53512d30 --- /dev/null +++ b/src/core/HistogramAggregator.java @@ -0,0 +1,27 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +/** + * Aggregator for histogram data points. + * + * @since 2.4 + */ +public class HistogramAggregator { + + public interface Histograms { + boolean hasNextValue(); + + HistogramDataPoint nextHistogramValue(); + } +} diff --git a/src/core/HistogramBucketDataPointsAdaptor.java b/src/core/HistogramBucketDataPointsAdaptor.java new file mode 100644 index 0000000000..f25fb6b5f0 --- /dev/null +++ b/src/core/HistogramBucketDataPointsAdaptor.java @@ -0,0 +1,275 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import java.util.List; +import java.util.Map; + +import org.hbase.async.Bytes.ByteMap; + +import com.stumbleupon.async.Callback; +import com.stumbleupon.async.Deferred; +import net.opentsdb.meta.Annotation; + +/** + * A class for converting histograms back into DataPoints so that we can re-use + * the existing TSDB serialization when displaying percentiles. + * + * @since 2.4 + */ +public class HistogramBucketDataPointsAdaptor implements DataPoints { + private final HistogramDataPoints hist_data_points; + private final HistogramDataPoint.HistogramBucket bucket; + + HistogramBucketDataPointsAdaptor(final HistogramDataPoints hists, + final HistogramDataPoint.HistogramBucket bucket) { + this.hist_data_points = hists; + this.bucket = bucket; + } + + @Override + public String metricName() { + return (this.hist_data_points.metricName() + metricNamePostfix()); + } + + @Override + public Deferred metricNameAsync() { + return this.hist_data_points.metricNameAsync() + .addCallback(new Callback() { + public String call(final String name) { + return name + metricNamePostfix(); + } + }); + } + + @Override + public byte[] metricUID() { + return this.hist_data_points.metricUID(); + } + + @Override + public Map getTags() { + return this.hist_data_points.getTags(); + } + + @Override + public Deferred> getTagsAsync() { + return this.hist_data_points.getTagsAsync(); + } + + @Override + public ByteMap getTagUids() { + return this.hist_data_points.getTagUids(); + } + + @Override + public List getAggregatedTags() { + return this.hist_data_points.getAggregatedTags(); + } + + @Override + public Deferred> getAggregatedTagsAsync() { + return this.hist_data_points.getAggregatedTagsAsync(); + } + + @Override + public List getAggregatedTagUids() { + return this.hist_data_points.getAggregatedTagUids(); + } + + @Override + public List getTSUIDs() { + return this.hist_data_points.getTSUIDs(); + } + + @Override + public List getAnnotations() { + return this.hist_data_points.getAnnotations(); + } + + @Override + public int size() { + return this.hist_data_points.size(); + } + + @Override + public int aggregatedSize() { + return this.hist_data_points.aggregatedSize(); + } + + @Override + public SeekableView iterator() { + return internalIterator(); + } + + private HistogramDataPoint getHistogramDataPoint(int i) { + if (i < 0) { + throw new IndexOutOfBoundsException("negative index: " + i); + } + final int saved_i = i; + final HistogramSeekableView it = this.hist_data_points.iterator(); + HistogramDataPoint dp = null; + while (it.hasNext() && i >= 0) { + dp = it.next(); + i--; + } + if (i != -1 || dp == null) { + throw new IndexOutOfBoundsException("index " + saved_i + + " too large (it's >= " + size() + ") for " + this); + } + return dp; + } + + @Override + public long timestamp(int i) { + return getHistogramDataPoint(i).timestamp(); + } + + @Override + public boolean isInteger(int i) { + return true; + } + + @Override + public long longValue(int i) { + HistogramDataPoint hdp = this.getHistogramDataPoint(i); + + try { + Map buckets = + hdp.getHistogramBucketsIfHas(); + if (null != buckets && buckets.containsKey(bucket)) { + return buckets.get(bucket).longValue(); + } + } catch (UnsupportedOperationException e) { + // Just ignore + } + + return 0; + } + + @Override + public double doubleValue(int i) { + return this.longValue(i); + } + + @Override + public int getQueryIndex() { + return this.hist_data_points.getQueryIndex(); + } + + @Override + public boolean isPercentile() { + return false; + } + + @Override + public float getPercentile() { + return 0; + } + + private String metricNamePostfix() { + if (this.bucket.bucketType() == + HistogramDataPoint.HistogramBucket.BucketType.UNDERFLOW) { + return "_UNDERFLOW"; + } else if (this.bucket.bucketType() == + HistogramDataPoint.HistogramBucket.BucketType.OVERFLOW) { + return "_OVERFLOW"; + } else { + StringBuilder sb = new StringBuilder(); + sb.append("_").append(this.bucket.getLowerBound()).append("_") + .append(this.bucket.getUpperBound()); + return sb.toString(); + } + } + + private Iterator internalIterator() { + return new Iterator(); + } + + ///////////////////////////////////////////////////////////////////////////// + // internal iterator + ///////////////////////////////////////////////////////////////////////////// + final class Iterator implements SeekableView, DataPoint { + final private HistogramSeekableView source; + private long value; + private long timestamp; + + public Iterator() { + this.source = hist_data_points.iterator(); + } + + @Override + public boolean hasNext() { + return this.source.hasNext(); + } + + @Override + public DataPoint next() { + HistogramDataPoint hdp = this.source.next(); + + this.value = 0; + try { + Map buckets = + hdp.getHistogramBucketsIfHas(); + if (null != buckets && buckets.containsKey(bucket)) { + this.value = buckets.get(bucket).longValue(); + } + } catch (UnsupportedOperationException e) { + // Just ignore + } + + this.timestamp = hdp.timestamp(); + return this; + } + + @Override + public void remove() { + throw new UnsupportedOperationException("remove is not supported here"); + } + + @Override + public void seek(long timestamp) { + this.source.seek(timestamp); + } + + @Override + public long timestamp() { + return this.timestamp; + } + + @Override + public boolean isInteger() { + return true; + } + + @Override + public long longValue() { + return this.value; + } + + @Override + public double doubleValue() { + throw new ClassCastException("value #" + " is not a long in " + this); + } + + @Override + public double toDouble() { + return this.value; + } + + @Override + public long valueCount() { + return 0; + } + } +} diff --git a/src/core/HistogramCodecManager.java b/src/core/HistogramCodecManager.java new file mode 100644 index 0000000000..8376e4c1b4 --- /dev/null +++ b/src/core/HistogramCodecManager.java @@ -0,0 +1,207 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import java.io.File; +import java.io.IOException; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.base.Strings; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.common.io.Files; + +import net.opentsdb.utils.JSON; +import net.opentsdb.utils.PluginLoader; + +/** + *

    + * Manages the histogram codecs loaded by default or as plugins. Codecs are + * defined in the 'tsd.core.histograms.config' property as either direct, + * escaped JSON or a file with the ".json" extension. Each codec config entry + * must include the full class name and a unique ID for the type of histogram + * or digest stored. + * WARNING: After writing data with a given ID, you cannot change the + * ID and read that data any more. + *

    + * This class is thread safe + * + * @since 2.4 + */ +public class HistogramCodecManager { + private static final Logger LOG = LoggerFactory.getLogger( + HistogramCodecManager.class); + + /** The map of IDs to decoders. */ + private final Map codecs; + + /** The map of classes to decoder IDs. */ + private final Map, Integer> codecs_ids; + + /** + * Default ctor that loads the codec map. It will parse the + * 'tsd.core.histograms.config' parameter. If it ends with .json then we'll + * try to load a file of that name, otherwise we'll just parse it as raw JSON. + * For each map in the JSON we search the classpath then loaded plugins. + * + * @param tsdb A non-null TSDB to load the config from. + * @throws IllegalArgumentException if the config was null or empty or if the + * JSON was malformed. + * @throws RuntimeException if the file couldn't be opened. + * @throws IllegalStateException if no classes/plugins of the type could be + * found OR if one was found but couldn't be instantiated. + */ + public HistogramCodecManager(final TSDB tsdb) { + final String config = tsdb.getConfig().getString("tsd.core.histograms.config"); + if (Strings.isNullOrEmpty(config)) { + throw new IllegalArgumentException("Missing configuration " + + "'tsd.core.histograms.config'"); + } + + final TypeReference> type_ref = + new TypeReference>() {}; + final Map mappings; + if (config.endsWith(".json")) { + final String json; + try { + json = Files.toString(new File(config), Const.UTF8_CHARSET); + } catch (IOException e) { + throw new RuntimeException("Unable to open plugin config file: " + + config, e); + } + mappings = JSON.parseToObject(json, type_ref); + } else { + mappings = JSON.parseToObject(config, type_ref); + } + + codecs = Maps.newHashMap(); + codecs_ids = Maps.newHashMap(); + + if (mappings.isEmpty()) { + LOG.warn("No histograms configured. Histogram writes and reads will " + + "throw exceptions."); + return; + } + + final Set ids = Sets.newHashSet(); + for (final Entry mapping : mappings.entrySet()) { + if (mapping.getValue() < 0 || mapping.getValue() > 255) { + throw new IllegalArgumentException("ID for codec '" + mapping.getKey() + + "' must be from 0 to 255."); + } + if (ids.contains(mapping.getValue())) { + throw new IllegalArgumentException("Duplicate ID found for codec '" + + mapping.getKey() + "': " + mapping.getValue()); + } + ids.add(mapping.getValue()); + + HistogramDataPointCodec codec = null; + try { + final Class clazz = Class.forName(mapping.getKey()); + codec = (HistogramDataPointCodec) clazz.newInstance(); + } catch (ClassNotFoundException e) { + codec = PluginLoader + .loadSpecificPlugin(mapping.getKey(), HistogramDataPointCodec.class); + } catch (InstantiationException e) { + throw new IllegalStateException("Found decoder '" + mapping.getKey() + + "' on the class path but failed to instantiate it.", e); + } catch (IllegalAccessException e) { + throw new IllegalStateException("Found decoder '" + mapping.getKey() + + "' on the class path but we did not have permission to access it.", e); + } + + if (codec == null) { + throw new IllegalStateException("Unable to find a decoder named '" + + mapping.getKey() + "'"); + } else { + codec.setId(mapping.getValue()); + codecs.put(mapping.getValue(), codec); + codecs_ids.put(codec.getClass(), mapping.getValue()); + LOG.info("Successfully loaded decoder '" + mapping.getKey() + + "' with ID " + mapping.getValue()); + } + } + } + + /** + * Return the instance of the given codec. + * @param id The numeric ID of the codec (the first byte in storage). + * @return The instance of the given codec + * @throws IllegalArgumentException if no codec was found for the given ID. + */ + public HistogramDataPointCodec getCodec(final int id) { + final HistogramDataPointCodec codec = codecs.get(id); + if (codec == null) { + throw new IllegalArgumentException("No codec found mapped to ID " + id); + } + return codec; + } + + /** + * Return the ID of the given codec. + * @param clazz The non-null class to search for. + * @return The ID of the codec. + * @throws IllegalArgumentException if the class was null or no ID was assigned + * to the class. + */ + public int getCodec(final Class clazz) { + if (clazz == null) { + throw new IllegalArgumentException("Clazz cannot be null."); + } + final Integer id = codecs_ids.get(clazz); + if (id == null) { + throw new IllegalArgumentException("No codec ID assigned to class " + + clazz); + } + return id; + } + + /** + * Finds the proper codec and calls it's encode method to create a byte array + * with the given ID as the first byte. + * @param id The ID of the histogram type to search for. + * @param data_point The non-null data point to encode. + * @param include_id Whether or not to include the ID prefix when encoding. + * @return A non-null and non-empty byte array if the codec was found. + * @throws IllegalArgumentException if no codec was found for the given ID or + * the histogram may have been of the wrong type or failed encoding. + */ + public byte[] encode(final int id, + final Histogram data_point, + final boolean include_id) { + final HistogramDataPointCodec codec = getCodec(id); + return codec.encode(data_point, include_id); + } + + /** + * Finds the proper codec and calls it's decode method to return the histogram + * data point for queries or validation. + * @param id The ID of the histogram type to search for. + * @param raw_data The non-null and non-empty byte array to parse. Should NOT + * include the first byte of the ID in the data. + * @param includes_id Whether or not the data includes the ID prefix. + * @return A non-null data point if decoding was successful. + */ + public Histogram decode(final int id, + final byte[] raw_data, + final boolean includes_id) { + final HistogramDataPointCodec codec = getCodec(id); + return codec.decode(raw_data, includes_id); + } +} diff --git a/src/core/HistogramDataPoint.java b/src/core/HistogramDataPoint.java new file mode 100644 index 0000000000..01d32cd20f --- /dev/null +++ b/src/core/HistogramDataPoint.java @@ -0,0 +1,192 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import java.util.List; +import java.util.Map; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.KryoSerializable; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; + +/** + * Represents a single histogram data point, e.g. all of the buckets for a + * measurement at a point in time. + * + * @since 2.4 + */ +public interface HistogramDataPoint extends Cloneable { + byte PREFIX = 0x6; + + /** + * Returns the timestamp (in milliseconds) associated with this data point. + * @return A strictly positive, 32 bit integer. + */ + long timestamp(); + + /** + * Get the encoded value of this histogram. + * NOTE: implementation should store the serialize information + * in the byte array so latter it can decide how to deserialize it back + * @return The encoded value os this histogram data point + */ + byte[] getRawData(final boolean include_id); + + /** + * Decode the raw data and reset the current histogram data point to the + * decoded value + * @param raw_data The encoded value of the histogram data point + */ + void resetFromRawData(final byte[] raw_data, final boolean includes_id); + + int getId(); + + /** + * Calculate percentile of this histogram data point + * @param p the distribution threshold + * @return The percentile value + */ + double percentile(final double p); + + /** + * Calculate percentile values of this histogram data point + * @param p the distribution threshold list + * @return A list of the percentile values + */ + List percentile(final List p); + + void aggregate(HistogramDataPoint histo, HistogramAggregation func); + + /** + * Create and return a copy of this object + * + * @return A deep copy object {@link HistogramDataPoint} + */ + HistogramDataPoint clone(); + + HistogramDataPoint cloneAndSetTimestamp(final long timestamp); + + /////////////////////////////////////////////////////////////////////////// + // A nested class to present the bucket information + /////////////////////////////////////////////////////////////////////////// + public class HistogramBucket implements KryoSerializable, Comparable { + public enum BucketType { + UNDERFLOW, REGULAR, OVERFLOW + } + + private final BucketType type; + private float lower_bound; + private float upper_bound; + + public HistogramBucket() { + this.type = BucketType.REGULAR; + this.lower_bound = 0; + this.upper_bound = 0; + } + + public HistogramBucket(final BucketType type, final float lower_bound, + final float uper_bound) { + this.type = type; + this.lower_bound = lower_bound; + this.upper_bound = uper_bound; + } + + public BucketType bucketType() { + return this.type; + } + + public float getLowerBound() { + return this.lower_bound; + } + + public float getUpperBound() { + return this.upper_bound; + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } + + if (that == null || getClass() != that.getClass()) { + return false; + } + + HistogramBucket bk = (HistogramBucket)that; + if (bucketType() != bk.bucketType()) { + return false; + } + + if ((BucketType.UNDERFLOW == bucketType() && + BucketType.UNDERFLOW == bk.bucketType()) || + (BucketType.OVERFLOW == bucketType() && + BucketType.OVERFLOW == bk.bucketType())) { + return true; + } + + if (Float.compare(getLowerBound(), bk.getLowerBound()) != 0) { + return false; + } + + return (Float.compare(getUpperBound(), bk.getUpperBound()) == 0); + } + + @Override + public int compareTo(HistogramBucket that) { + if (this.equals(that)) { + return 0; + } else if (BucketType.UNDERFLOW == type) { + return -1; + } else if (BucketType.REGULAR == type) { + int lower_bound_compare = Float.compare(getLowerBound(), + that.getLowerBound()); + if (lower_bound_compare != 0) { + return lower_bound_compare; + } else { + return Float.compare(getUpperBound(), that.getUpperBound()); + } + } else if (BucketType.OVERFLOW == type) { + return +1; + } + + return 0; + } + + public void write(Kryo kryo, Output output) { + output.writeFloat(lower_bound); + output.writeFloat(upper_bound); + } + + public void read(Kryo kryo, Input input) { + lower_bound = input.readFloat(); + upper_bound = input.readFloat(); + } + + @Override + public String toString() { + if (this.getUpperBound() != Float.NaN) { + return this.getLowerBound() + "-" + this.getUpperBound(); + } else { + return this.getLowerBound() + "-"; + } + } + } + + /** + * Get buckets from this histogram data point + * @return + */ + Map getHistogramBucketsIfHas(); +} diff --git a/src/core/HistogramDataPointCodec.java b/src/core/HistogramDataPointCodec.java new file mode 100644 index 0000000000..7ac01c0cd2 --- /dev/null +++ b/src/core/HistogramDataPointCodec.java @@ -0,0 +1,58 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +/** + * Responsible for encoding or decoding {@code HistogramDataPoint}s to and from + * byte arrays. + * + * NOTE: Implementation of this plugin should be thread safe. + * @see HistogramCodecManager + * + * @since 2.4 + */ +public abstract class HistogramDataPointCodec { + + /** The ID of this codec in the Histogram Manager. */ + protected int id; + + /** + * Default empty ctor, required for plugin and class instantiation. + * WARNING Any overrides with arguments will be ignored. + */ + public HistogramDataPointCodec() { + + } + + public int getId() { + return id; + } + + public void setId(final int id) { + this.id = id; + } + + /** + * Creates {@code HistogramDataPoint} from raw data and timestamp. Note that + * the data point identifier is separate. + * @param raw_data The encoded byte array of the histogram data + * @param includes_id Whether or not to include the id prefix. + * @return The decoded histogram data point instance + */ + public abstract Histogram decode(final byte[] raw_data, + final boolean includes_id); + + + public abstract byte[] encode(final Histogram data_point, + final boolean include_id); +} diff --git a/src/core/HistogramDataPoints.java b/src/core/HistogramDataPoints.java new file mode 100644 index 0000000000..96d0dd5238 --- /dev/null +++ b/src/core/HistogramDataPoints.java @@ -0,0 +1,178 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import com.stumbleupon.async.Deferred; +import net.opentsdb.meta.Annotation; +import org.hbase.async.Bytes; + +import java.util.List; +import java.util.Map; + +/** + * A clone of the DataPoints interface except for handling histograms. + * + * @since 2.4 + */ +public interface HistogramDataPoints extends Iterable { + + /** + * Returns the name of the series. + * @return The name of the metric as a string. + */ + String metricName(); + + /** + * Returns the name of the series. + * @return The name of the metric in a deferred (may contain an exception). + */ + Deferred metricNameAsync(); + + /** + * @return the metric UID + * @return The metric UID as an array of bytes. + */ + byte[] metricUID(); + + /** + * Returns the tags associated with these data points. + * @return A non-{@code null} map of tag names (keys), tag values (values). + */ + Map getTags(); + + /** + * Returns the tags associated with these data points. + * @return A non-{@code null} map of tag names (keys), tag values (values). + */ + Deferred> getTagsAsync(); + + /** + * Returns a map of tag pairs as UIDs. + * When used on a span or row, it returns the tag set. When used on a span + * group it will return only the tag pairs that are common across all + * time series in the group. + * @return A potentially empty map of tagk to tagv pairs as UIDs + */ + Bytes.ByteMap getTagUids(); + + /** + * Returns the tags associated with some but not all of the data points. + *

    + * When this instance represents the aggregation of multiple time series + * (same metric but different tags), {@link #getTags} returns the tags that + * are common to all data points (intersection set) whereas this method + * returns all the tags names that are not common to all data points (union + * set minus the intersection set, also called the symmetric difference). + *

    + * If this instance does not represent an aggregation of multiple time + * series, the list returned is empty. + * @return A non-{@code null} list of tag names. + */ + List getAggregatedTags(); + + /** + * Returns the tags associated with some but not all of the data points. + *

    + * When this instance represents the aggregation of multiple time series + * (same metric but different tags), {@link #getTags} returns the tags that + * are common to all data points (intersection set) whereas this method + * returns all the tags names that are not common to all data points (union + * set minus the intersection set, also called the symmetric difference). + *

    + * If this instance does not represent an aggregation of multiple time + * series, the list returned is empty. + * @return A non-{@code null} list of tag names. + */ + Deferred> getAggregatedTagsAsync(); + + /** + * Returns the tagk UIDs associated with some but not all of the data points. + * @return a non-{@code null} list of tagk UIDs. + */ + List getAggregatedTagUids(); + + /** + * Returns a list of unique TSUIDs contained in the results + * @return an empty list if there were no results, otherwise a list of TSUIDs + */ + public List getTSUIDs(); + + /** + * Compiles the annotations for each span into a new array list + * @return Null if none of the spans had any annotations, a list if one or + * more were found + */ + public List getAnnotations(); + + /** + * Returns a warning about the query, i.e if it terminated prematurely + * @return A null if no warning, otherwise a string to return to the user + */ + public String getWarning(); + + /** + * Returns the number of histogram data points. + *

    + * This method must be implemented in {@code O(1)} or {@code O(n)} + * where n = {@link #aggregatedSize} > 0. + * @return A positive integer. + */ + int size(); + + /** + * Returns the number of data points aggregated in this instance. + *

    + * When this instance represents the aggregation of multiple time series + * (same metric but different tags), {@link #size} returns the number of data + * points after aggregation, whereas this method returns the number of data + * points before aggregation. + *

    + * If this instance does not represent an aggregation of multiple time + * series, then 0 is returned. + * @return A positive integer. + */ + int aggregatedSize(); + + /** + * Returns a zero-copy view to go through {@code size()} data points. + *

    + * The iterator returned must return each {@link DataPoint} in {@code O(1)}. + * The {@link DataPoint} returned must not be stored and gets + * invalidated as soon as {@code next} is called on the iterator. If you + * want to store individual data points, you need to copy the timestamp + * and value out of each {@link DataPoint} into your own data structures. + * @return An iterator over the data points. + */ + HistogramSeekableView iterator(); + + /** + * Returns the timestamp associated with the {@code i}th data point. + * The first data point has index 0. + *

    + * This method must be implemented in + * O({@link #aggregatedSize}) or better. + *

    + * It is guaranteed that 

    timestamp(i) < timestamp(i+1)
    + * @param i The index to fetch a timestamp for + * @return A strictly positive integer. + * @throws IndexOutOfBoundsException if {@code i} is not in the range + * [0, {@link #size} - 1] + */ + long timestamp(int i); + + /** + * Return the query index that maps this datapoints to the original subquery + * @return index of the query in the TSQuery class + */ + int getQueryIndex(); +} diff --git a/src/core/HistogramDataPointsToDataPointsAdaptor.java b/src/core/HistogramDataPointsToDataPointsAdaptor.java new file mode 100644 index 0000000000..3b1cc0658f --- /dev/null +++ b/src/core/HistogramDataPointsToDataPointsAdaptor.java @@ -0,0 +1,239 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import java.util.List; +import java.util.Map; + +import org.hbase.async.Bytes.ByteMap; + +import com.stumbleupon.async.Callback; +import com.stumbleupon.async.Deferred; + +import net.opentsdb.meta.Annotation; + +/** + * Converts histogram DataPoints to DataPoints for using the existing metric + * serialization code when querying percentiles. + * + * @since 2.4 + */ +public class HistogramDataPointsToDataPointsAdaptor implements DataPoints { + final private HistogramDataPoints hist_data_points; + final private float percentile; + + public HistogramDataPointsToDataPointsAdaptor(final HistogramDataPoints hdps, + final float percentile) { + this.hist_data_points = hdps; + this.percentile = percentile; + } + + @Override + public String metricName() { + return this.hist_data_points.metricName() + "_pct_" + + Float.toString(this.percentile); + } + + @Override + public Deferred metricNameAsync() { + return this.hist_data_points.metricNameAsync() + .addCallback(new Callback() { + public String call(final String name) { + return name + "_pct_" + Float.toString(percentile); + } + }); + } + + @Override + public byte[] metricUID() { + return this.hist_data_points.metricUID(); + } + + @Override + public Map getTags() { + return this.hist_data_points.getTags(); + } + + @Override + public Deferred> getTagsAsync() { + return this.hist_data_points.getTagsAsync(); + } + + @Override + public ByteMap getTagUids() { + return this.hist_data_points.getTagUids(); + } + + @Override + public List getAggregatedTags() { + return this.hist_data_points.getAggregatedTags(); + } + + @Override + public Deferred> getAggregatedTagsAsync() { + return this.hist_data_points.getAggregatedTagsAsync(); + } + + @Override + public List getAggregatedTagUids() { + return this.hist_data_points.getAggregatedTagUids(); + } + + @Override + public List getTSUIDs() { + return this.hist_data_points.getTSUIDs(); + } + + @Override + public List getAnnotations() { + return this.hist_data_points.getAnnotations(); + } + + @Override + public int size() { + return this.hist_data_points.size(); + } + + @Override + public int aggregatedSize() { + return this.hist_data_points.aggregatedSize(); + } + + @Override + public SeekableView iterator() { + return internalIterator(); + } + + private HistogramDataPoint getHistogramDataPoint(int i) { + if (i < 0) { + throw new IndexOutOfBoundsException("negative index: " + i); + } + final int saved_i = i; + final HistogramSeekableView it = this.hist_data_points.iterator(); + HistogramDataPoint dp = null; + while (it.hasNext() && i >= 0) { + dp = it.next(); + i--; + } + if (i != -1 || dp == null) { + throw new IndexOutOfBoundsException("index " + saved_i + + " too large (it's >= " + size() + ") for " + this); + } + return dp; + } + + @Override + public long timestamp(int i) { + return getHistogramDataPoint(i).timestamp(); + } + + @Override + public boolean isInteger(int i) { + return false; + } + + @Override + public long longValue(int i) { + throw new ClassCastException("value #" + i + " is not a long in " + this); + } + + @Override + public double doubleValue(int i) { + return getHistogramDataPoint(i).percentile(this.percentile); + } + + @Override + public int getQueryIndex() { + return this.hist_data_points.getQueryIndex(); + } + + @Override + public boolean isPercentile() { + return true; + } + + @Override + public float getPercentile() { + return this.percentile; + } + + private Iterator internalIterator() { + return new Iterator(); + } + + //////////////////////////////////////////////////////////////////////////// + // internal iterator + //////////////////////////////////////////////////////////////////////////// + final class Iterator implements SeekableView, DataPoint { + final private HistogramSeekableView source; + private double value; + private long timestamp; + + public Iterator() { + this.source = hist_data_points.iterator(); + } + + @Override + public boolean hasNext() { + return this.source.hasNext(); + } + + @Override + public DataPoint next() { + HistogramDataPoint hdp = this.source.next(); + this.value = hdp.percentile(percentile); + this.timestamp = hdp.timestamp(); + return this; + } + + @Override + public void remove() { + throw new UnsupportedOperationException("remove is not supported here"); + } + + @Override + public void seek(long timestamp) { + this.source.seek(timestamp); + } + + @Override + public long timestamp() { + return this.timestamp; + } + + @Override + public boolean isInteger() { + return false; + } + + @Override + public long longValue() { + throw new ClassCastException("value #" + " is not a long in " + this); + } + + @Override + public double doubleValue() { + return this.value; + } + + @Override + public double toDouble() { + return this.value; + } + + @Override + public long valueCount() { + return 0; + } + } +} diff --git a/src/core/HistogramDownsampler.java b/src/core/HistogramDownsampler.java new file mode 100644 index 0000000000..6545f6ec54 --- /dev/null +++ b/src/core/HistogramDownsampler.java @@ -0,0 +1,403 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import net.opentsdb.utils.DateTime; + +import java.util.Calendar; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; + +/** + * Iterator that downsamples histogram data points using an + * {@link HistogramAggregation}. + * + * @since 2.4 + */ +public class HistogramDownsampler implements HistogramSeekableView, HistogramDataPoint { + + /** Matches the weekly downsampler as it requires special handling. */ + protected final static int WEEK_UNIT = DateTime.unitsToCalendarType("w"); + protected final static int DAY_UNIT = DateTime.unitsToCalendarType("d"); + protected final static int WEEK_LENGTH = 7; + + protected final HistogramSeekableView source; + /** Iterator to iterate the values of the current interval. */ + protected final HistogramDownsampler.ValuesInInterval values_in_interval; + /** The downsampling specification when provided */ + protected final DownsamplingSpecification specification; + /** The start timestamp of the actual query for use with "all" */ + protected final long query_start; + /** The end timestamp of the actual query for use with "all" */ + protected final long query_end; + /** Last normalized timestamp */ + protected long timestamp; + /** Last value */ + protected HistogramDataPoint value; + /** The interval to use with a calendar */ + protected final int interval; + /** The unit to use with a calendar as a Calendar integer */ + protected final int unit; + /** Whether or not to merge all DPs in the source into one value */ + protected final boolean run_all; + + /** + * Ctor. + * + * @param source The iterator to access the underlying data. + * @param specification The downsampling spec to use + * @param query_start The start timestamp of the actual query for use with "all" + * @param query_end The end timestamp of the actual query for use with "all" + * @since 2.3 + */ + HistogramDownsampler(final HistogramSeekableView source, + final DownsamplingSpecification specification, + final long query_start, + final long query_end) { + this.source = source; + this.specification = specification; + this.values_in_interval = new ValuesInInterval(); + this.query_start = query_start; + this.query_end = query_end; + + final String s = specification.getStringInterval(); + if (s != null && s.toLowerCase().contains("all")) { + run_all = true; + interval = 0; + unit = 0; + } else if (s != null && specification.useCalendar()) { + if (s.toLowerCase().contains("ms")) { + interval = Integer.parseInt(s.substring(0, s.length() - 2)); + unit = DateTime.unitsToCalendarType(s.substring(s.length() - 2)); + } else { + interval = Integer.parseInt(s.substring(0, s.length() - 1)); + unit = DateTime.unitsToCalendarType(s.substring(s.length() - 1)); + } + run_all = false; + } else { + run_all = false; + interval = 0; + unit = 0; + } + } + + @Override + public long timestamp() { + if (run_all) { + return query_start; + } + return timestamp; + } + + @Override + public byte[] getRawData(final boolean include_id) { + return value.getRawData(include_id); + } + + @Override + public void resetFromRawData(byte[] raw_data, final boolean includes_id) { + throw new UnsupportedOperationException(); + } + + @Override + public double percentile(double p) { + return value.percentile(p); + } + + @Override + public List percentile(List p) { + return value.percentile(p); + } + + @Override + public void aggregate(HistogramDataPoint histo, HistogramAggregation func) { + value.aggregate(histo, func); + } + + @Override + public boolean hasNext() { + return values_in_interval.hasNextValue(); + } + + @Override + public HistogramDataPoint next() { + if (hasNext()) { + value = values_in_interval.nextHistogramValue(); + while (values_in_interval.hasNextValue()) { + // this call will change the data in @{code value} + value.aggregate(values_in_interval.nextHistogramValue(), + specification.getHistogramAggregation()); + } + timestamp = values_in_interval.getIntervalTimestamp(); + + values_in_interval.moveToNextInterval(); + return this; + } + + throw new NoSuchElementException("no more data points in " + this); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + @Override + public void seek(long timestamp) { + values_in_interval.seekInterval(timestamp); + } + + @Override + public String toString() { + final StringBuilder buf = new StringBuilder(); + buf.append("HistogramDownsampler: ") + .append(", downsampler=") + .append(specification) + .append(", query_start=") + .append(query_start) + .append(", current data=(timestamp=") + .append(timestamp) + .append(", value=") + .append(value) + .append("), values_in_interval=") + .append(values_in_interval); + return buf.toString(); + } + + @Override + public HistogramDataPoint clone() { + // make sure give the right timestamp here, because value has a timestamp + // from the underlaid data point, not the aligned the downsampled timestamp + return this.value.cloneAndSetTimestamp(timestamp); + } + + @Override + public HistogramDataPoint cloneAndSetTimestamp(final long timestamp) { + return this.value.cloneAndSetTimestamp(timestamp); + } + + @Override + public int getId() { + return value.getId(); + } + + class ValuesInInterval implements HistogramAggregator.Histograms { + + /** An optional calendar set to the current timestamp for the data point */ + private Calendar previous_calendar; + /** An optional calendar set to the end of the interval timestamp */ + private Calendar next_calendar; + /** The end of the current interval. */ + private long timestamp_end_interval = Long.MIN_VALUE; + /** True if the last value was successfully extracted from the source. */ + private boolean has_next_value_from_source = false; + /** The last data point extracted from the source. */ + private HistogramDataPoint next_dp = null; + /** True if it is initialized for iterating intervals. */ + private boolean initialized = false; + + protected ValuesInInterval() { + if (run_all) { + timestamp_end_interval = query_end; + } else if (!specification.useCalendar()) { + timestamp_end_interval = specification.getInterval(); + } + } + + /** Initializes to iterate intervals. */ + protected void initializeIfNotDone() { + // NOTE: Delay initialization is required to not access any data point + // from the source until a user requests it explicitly to avoid the severe + // performance penalty by accessing the unnecessary first data of a span. + if (!initialized) { + initialized = true; + if (source.hasNext()) { + moveToNextValue(); + if (!run_all) { + if (specification.useCalendar()) { + previous_calendar = + DateTime.previousInterval(next_dp.timestamp(), interval, unit, + specification.getTimezone()); + next_calendar = + DateTime.previousInterval(next_dp.timestamp(), interval, unit, + specification.getTimezone()); + if (unit == WEEK_UNIT) { + next_calendar.add(DAY_UNIT, interval * WEEK_LENGTH); + } else { + next_calendar.add(unit, interval); + } + timestamp_end_interval = next_calendar.getTimeInMillis(); + } else { + timestamp_end_interval = alignTimestamp(next_dp.timestamp()) + + specification.getInterval(); + } + } + } + } + } + + /** Extracts the next value from the source. */ + private void moveToNextValue() { + if (source.hasNext()) { + has_next_value_from_source = true; + // filter out dps that don't match start and end for run_alls + if (run_all) { + while (source.hasNext()) { + next_dp = source.next(); + if (next_dp.timestamp() < query_start) { + next_dp = null; + continue; + } + if (next_dp.timestamp() >= query_end) { + has_next_value_from_source = false; + } + break; + } + if (next_dp == null) { + has_next_value_from_source = false; + } + } else { + next_dp = source.next(); + } + } else { + has_next_value_from_source = false; + } + } + + /** + * Resets the current interval with the interval of the timestamp of the + * next value read from source. It is the first value of the next interval. + */ + private void resetEndOfInterval() { + if (has_next_value_from_source && !run_all) { + if (specification.useCalendar()) { + while (next_dp.timestamp() >= timestamp_end_interval) { + if (unit == WEEK_UNIT) { + previous_calendar.add(DAY_UNIT, interval * WEEK_LENGTH); + next_calendar.add(DAY_UNIT, interval * WEEK_LENGTH); + } else { + previous_calendar.add(unit, interval); + next_calendar.add(unit, interval); + } + timestamp_end_interval = next_calendar.getTimeInMillis(); + } + } else { + timestamp_end_interval = alignTimestamp(next_dp.timestamp()) + + specification.getInterval(); + } + } + } + + /** Moves to the next available interval. */ + void moveToNextInterval() { + initializeIfNotDone(); + resetEndOfInterval(); + } + + /** Advances the interval iterator to the given timestamp. */ + void seekInterval(final long timestamp) { + // To make sure that the interval of the given timestamp is fully filled, + // rounds up the seeking timestamp to the smallest timestamp that is + // a multiple of the interval and is greater than or equal to the given + // timestamp.. + if (run_all) { + source.seek(timestamp); + } else if (specification.useCalendar()) { + final Calendar seek_calendar = DateTime.previousInterval(timestamp, + interval, unit, specification.getTimezone()); + if (timestamp > seek_calendar.getTimeInMillis()) { + if (unit == WEEK_UNIT) { + seek_calendar.add(DAY_UNIT, interval * WEEK_LENGTH); + } else { + seek_calendar.add(unit, interval); + } + } + source.seek(seek_calendar.getTimeInMillis()); + } else { + source.seek(alignTimestamp(timestamp + specification.getInterval() - 1)); + } + initialized = false; + } + + /** Returns the representative timestamp of the current interval. */ + protected long getIntervalTimestamp() { + // NOTE: It is well-known practice taking the start time of + // a downsample interval as a representative timestamp of it. It also + // provides the correct context for seek. + if (run_all) { + return timestamp_end_interval; + } else if (specification.useCalendar()) { + return previous_calendar.getTimeInMillis(); + } else { + return alignTimestamp(timestamp_end_interval - specification.getInterval()); + } + } + + /** Returns timestamp aligned by interval. */ + protected long alignTimestamp(final long timestamp) { + return timestamp - (timestamp % specification.getInterval()); + } + + @Override + public boolean hasNextValue() { + initializeIfNotDone(); + if (run_all) { + return has_next_value_from_source; + } + return has_next_value_from_source && next_dp.timestamp() < + timestamp_end_interval; + } + + @Override + public HistogramDataPoint nextHistogramValue() { + if (hasNextValue()) { + if (next_dp != null) { + HistogramDataPoint value = null; + // we have to clone the object, else when moveToNextValue in the + // next step will also change the @{code next_dp} and @{code value} + // here + value = next_dp.clone(); + moveToNextValue(); + return value; + } + } + throw new NoSuchElementException("no more values in interval of " + + timestamp_end_interval); + } + + @Override + public String toString() { + final StringBuilder buf = new StringBuilder(); + buf.append("ValuesInInterval{") + .append(", timestamp_end_interval=") + .append(timestamp_end_interval) + .append(", unit=") + .append(unit) + .append(", interval=") + .append(interval) + .append(", has_next_value_from_source=") + .append(has_next_value_from_source); + if (has_next_value_from_source) { + buf.append(", nextValue=(").append(next_dp).append(')'); + } + buf.append(", source=").append(source).append("}"); + return buf.toString(); + } + } + + @Override + public Map getHistogramBucketsIfHas() { + return this.value.getHistogramBucketsIfHas(); + } +} diff --git a/src/core/HistogramPojo.java b/src/core/HistogramPojo.java new file mode 100644 index 0000000000..cc1ee78910 --- /dev/null +++ b/src/core/HistogramPojo.java @@ -0,0 +1,153 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import javax.xml.bind.DatatypeConverter; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.base.Strings; + +public class HistogramPojo extends IncomingDataPoint { + private static final Logger LOG = LoggerFactory.getLogger(HistogramPojo.class); + + private int id; + + private Map buckets; + + private long underflow; + + private long overflow; + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + @Override + public boolean validate(final List> details) { + if (this.getMetric() == null || this.getMetric().isEmpty()) { + if (details != null) { + details.add(getHttpDetails("Metric name was empty")); + } + LOG.warn("Metric name was empty: " + this); + return false; + } + + if (this.getTimestamp() <= 0) { + if (details != null) { + details.add(getHttpDetails("Invalid timestamp")); + } + LOG.warn("Invalid timestamp: " + this); + return false; + } + + if (this.getTags() == null || this.getTags().size() < 1) { + if (details != null) { + details.add(getHttpDetails("Missing tags")); + } + LOG.warn("Missing tags: " + this); + return false; + } + + if (Strings.isNullOrEmpty(value)) { + if (buckets == null || buckets.isEmpty()) { + if (details != null) { + details.add(getHttpDetails("Histogram buckets cannot be null or empty " + + "if 'value' is empty.")); + } + LOG.warn("Histogram buckets cannot be null or empty if 'value' is empty."); + return false; + } + } else { + // ID is required for binary histos. + if (id < 0 || id > 255) { + if (details != null) { + details.add(getHttpDetails("Invalid type. Must be from 0 to 255.")); + } + LOG.warn("Invalid type. Must be from 0 to 255."); + return false; + } + } + return true; + } + + public SimpleHistogram toSimpleHistogram(final TSDB tsdb) { + if (buckets == null || buckets.isEmpty()) { + throw new IllegalArgumentException("Buckets cannot be empty when " + + "creating a simple histogram."); + } + + final SimpleHistogram shdp = new SimpleHistogram( + tsdb.histogramManager().getCodec(SimpleHistogramDecoder.class)); + shdp.setOverflow(overflow); + shdp.setUnderflow(underflow); + + for (final Entry bucket : buckets.entrySet()) { + final String[] bounds = Tags.splitString(bucket.getKey(), ','); + if (bounds.length != 2) { + throw new IllegalArgumentException("Unable to parse bucket bounds: " + + bucket.getKey()); + } + shdp.addBucket(Float.parseFloat(bounds[0]), Float.parseFloat(bounds[1]), + bucket.getValue()); + } + return shdp; + } + + public Map getBuckets() { + return buckets; + } + + public long getUnderflow() { + return underflow; + } + + public long getOverflow() { + return overflow; + } + + @JsonIgnore + public byte[] getBytes() { + return base64StringToBytes(value); + } + + public void setBuckets(final Map buckets) { + this.buckets = buckets; + } + + public void setUnderflow(final long underflow) { + this.underflow = underflow; + } + + public void setOverflow(final long overflow) { + this.overflow = overflow; + } + + public static String bytesToBase64String(final byte[] raw) { + return DatatypeConverter.printBase64Binary(raw); + } + + public static byte[] base64StringToBytes(final String encoded) { + return DatatypeConverter.parseBase64Binary(encoded); + } +} diff --git a/src/core/HistogramRowSeq.java b/src/core/HistogramRowSeq.java new file mode 100644 index 0000000000..8f808073eb --- /dev/null +++ b/src/core/HistogramRowSeq.java @@ -0,0 +1,395 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . + +package net.opentsdb.core; + +import com.stumbleupon.async.Deferred; + +import net.opentsdb.meta.Annotation; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; + +import org.hbase.async.Bytes; + +/** + * Represents a read-only sequence of continuous HBase rows. + *

    + * This class stores in memory the data of one or more continuous HBase rows for + * a given time series. To consolidate memory, the data points are stored in two + * byte arrays: one for the time offsets/flags and another for the values. + * Access is granted via pointers. + * + * @since 3.0 + */ +public class HistogramRowSeq implements iHistogramRowSeq { + + /** The {@link TSDB} instance we belong to. */ + private final TSDB tsdb; + + /** First row key. */ + protected byte[] key; + + protected List rowSeq; + + public HistogramRowSeq(final TSDB tsdb) { + this.tsdb = tsdb; + } + + @Override + public void setRow(final byte[] key, final List row) { + if (this.key != null) { + throw new IllegalStateException("setRow was already called on " + this); + } + + this.key = key; + this.rowSeq = row; + } + + @Override + public void addRow(final List row) { + if (null == this.key) { + throw new IllegalStateException("setRow was never called on " + this); + } + + int index_local = 0; + int index_remote = 0; + List combinedRows = + new ArrayList(this.rowSeq.size() + row.size()); + while (index_local < this.rowSeq.size() && index_remote < row.size()) { + HistogramDataPoint hdp_local = this.rowSeq.get(index_local); + HistogramDataPoint hdp_remote = row.get(index_remote); + + long sort = hdp_remote.timestamp() - hdp_local.timestamp(); + if (0 == sort) { + // duplicate histogram data point with the same timestamp, pick the local one + combinedRows.add(hdp_local); + ++index_local; + ++index_remote; + } else if (sort > 0) { + // the remote one has a bigger timestamp, pick the local one + combinedRows.add(hdp_local); + ++index_local; + } else { + // the local one has a bigger timestamp, pick the remote one + combinedRows.add(hdp_remote); + ++index_remote; + } + } // end while + + if (index_local < this.rowSeq.size()) { + // add the left elements of local to the combined list + combinedRows.addAll(this.rowSeq.subList(index_local, this.rowSeq.size())); + } else if (index_remote < row.size()) { + // add the left elements of remote to the combined list + combinedRows.addAll(row.subList(index_remote, row.size())); + } + + this.rowSeq = combinedRows; + } + + @Override + public byte[] key() { + return key; + } + + @Override + public long baseTime() { + return Internal.baseTime(key); + } + + @Override + public Iterator internalIterator() { + return new Iterator(); + } + + @Override + public String metricName() { + try { + return metricNameAsync().join(); + } catch (InterruptedException iex) { + throw new RuntimeException("Interrupted the metric name call", iex); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Should never be here", e); + } + } + + @Override + public Deferred metricNameAsync() { + if (key == null) { + throw new IllegalStateException("the row key is null!"); + } + return RowKey.metricNameAsync(tsdb, key); + } + + @Override + public byte[] metricUID() { + return Arrays.copyOfRange(key, Const.SALT_WIDTH(), Const.SALT_WIDTH() + + TSDB.metrics_width()); + } + + @Override + public Map getTags() { + try { + return getTagsAsync().join(); + } catch (InterruptedException iex) { + throw new RuntimeException("Interrupted the tags call", iex); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Should never be here", e); + } + } + + @Override + public Deferred> getTagsAsync() { + return Tags.getTagsAsync(tsdb, key); + } + + @Override + public Bytes.ByteMap getTagUids() { + return Tags.getTagUids(key); + } + + @Override + public List getAggregatedTags() { + return Collections.emptyList(); + } + + @Override + public Deferred> getAggregatedTagsAsync() { + final List empty = Collections.emptyList(); + return Deferred.fromResult(empty); + } + + @Override + public List getAggregatedTagUids() { + return Collections.emptyList(); + } + + @Override + public List getTSUIDs() { + return Collections.emptyList(); + } + + @Override + public List getAnnotations() { + return Collections.emptyList(); + } + + @Override + public String getWarning() { + return null; + } + + @Override + public int size() { + return rowSeq.size(); + } + + @Override + public int aggregatedSize() { + return 0; + } + + @Override + public HistogramSeekableView iterator() { + return internalIterator(); + } + + @Override + public long timestamp(int i) { + checkIndex(i); + + return rowSeq.get(i).timestamp(); + } + + @Override + public int getQueryIndex() { + return 0; + } + + /** + * @throws IndexOutOfBoundsException + * if {@code i} is out of bounds. + */ + private void checkIndex(final int i) { + if (i >= size()) { + throw new IndexOutOfBoundsException("index " + i + " >= " + size() + + " for this=" + this); + } + if (i < 0) { + throw new IndexOutOfBoundsException("negative index " + i + + " for this=" + this); + } + } + + /** Returns a human readable string representation of the object. */ + @Override + public String toString() { + final String metric = metricName(); + final int sz = size(); + + // The argument passed to StringBuilder is an estimate = number of data points * 125 + final StringBuilder buf = new StringBuilder(sz * 125); + final long base_time = baseTime(); + buf.append("RowSeq(") + .append(key == null ? "" : Arrays.toString(key)) + .append(" (metric=") + .append(metric) + .append("), base_time=") + .append(base_time) + .append(" (") + .append(base_time > 0 ? new Date(base_time * 1000) : "no date") + .append(")"); + + for (short i = 0; i < sz; ++i) { + buf.append('+').append(rowSeq.get(i).timestamp()); + buf.append(":histogram(").append(Arrays.toString(rowSeq.get(i).getRawData(true))); + buf.append(')'); + if (i != sz -1) { + buf.append(", "); + } + } // end for + + buf.append(")"); + return buf.toString(); + } + + /** + * Used to compare two RowSeq objects when sorting a {@link Span}. Compares on + * the {@code RowSeq#baseTime()} + * + * @since 2.0 + */ + public static final class HistogramRowSeqComparator implements + Comparator { + public int compare(final iHistogramRowSeq a, final iHistogramRowSeq b) { + if (null == a || null == b) { + if (a == b) { + return 0; + } else { + return (null == a) ? -1 : 1; + } + } + if (a.baseTime() == b.baseTime()) { + return 0; + } + return a.baseTime() < b.baseTime() ? -1 : 1; + } + } + + final class Iterator implements iHistogramRowSeq.Iterator { + private int next_index; + + Iterator() { + } + + @Override + public long timestamp() { + return getCurrent().timestamp(); + } + + @Override + public byte[] getRawData(final boolean include_id) { + return getCurrent().getRawData(include_id); + } + + @Override + public void resetFromRawData(final byte[] raw_data, final boolean includes_id) { + getCurrent().resetFromRawData(raw_data, includes_id); + } + + @Override + public double percentile(double p) { + return getCurrent().percentile(p); + } + + @Override + public List percentile(List p) { + return getCurrent().percentile(p); + } + + @Override + public void aggregate(HistogramDataPoint histo, HistogramAggregation func) { + getCurrent().aggregate(histo, func); + } + + @Override + public boolean hasNext() { + return next_index < rowSeq.size(); + } + + @Override + public HistogramDataPoint next() { + if (!hasNext()) { + throw new NoSuchElementException("no more elements"); + } + + ++next_index; + return this; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + @Override + public void seek(long timestamp) { + if ((timestamp & Const.MILLISECOND_MASK) != 0) { // negative or not 48 bits + throw new IllegalArgumentException("invalid timestamp: " + timestamp); + } + + // TODO: this can be optimized to O(nlogn) + next_index = 0; + while (next_index < rowSeq.size() && + rowSeq.get(next_index).timestamp() < timestamp) { + ++next_index; + } + } + + @Override + public HistogramDataPoint clone() { + return getCurrent().clone(); + } + + @Override + public int getId() { + return getCurrent().getId(); + } + + @Override + public HistogramDataPoint cloneAndSetTimestamp(final long timestamp) { + return getCurrent().cloneAndSetTimestamp(timestamp); + } + + private HistogramDataPoint getCurrent() { + assert next_index > 0 : "not initialized: " + this; + return rowSeq.get(next_index - 1); + } + + @Override + public Map getHistogramBucketsIfHas() { + return getCurrent().getHistogramBucketsIfHas(); + } + } +} diff --git a/src/core/HistogramSeekableView.java b/src/core/HistogramSeekableView.java new file mode 100644 index 0000000000..debc03b444 --- /dev/null +++ b/src/core/HistogramSeekableView.java @@ -0,0 +1,57 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * A clone of the DataPoints seekable view but for histograms. + * + * @since 2.4 + */ +public interface HistogramSeekableView extends Iterator { + + /** + * Returns {@code true} if this view has more elements. + */ + boolean hasNext(); + + /** + * Returns a view on the next data point. + * No new object gets created, the referenced returned is always the same + * and must not be stored since its internal data structure will change the + * next time {@code next()} is called. + * @throws NoSuchElementException if there were no more elements to iterate + * on (in which case {@link #hasNext} would have returned {@code false}. + */ + HistogramDataPoint next(); + + /** + * Unsupported operation. + * @throws UnsupportedOperationException always. + */ + void remove(); + + /** + * Advances the iterator to the given point in time. + *

    + * This allows the iterator to skip all the data points that are strictly + * before the given timestamp. + * @param timestamp A strictly positive 32 bit UNIX timestamp (in seconds). + * @throws IllegalArgumentException if the timestamp is zero, or negative, + * or doesn't fit on 32 bits (think "unsigned int" -- yay Java!). + */ + void seek(long timestamp); + +} diff --git a/src/core/HistogramSpan.java b/src/core/HistogramSpan.java new file mode 100644 index 0000000000..d5c00d5423 --- /dev/null +++ b/src/core/HistogramSpan.java @@ -0,0 +1,585 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import com.stumbleupon.async.Deferred; +import net.opentsdb.meta.Annotation; +import net.opentsdb.uid.NoSuchUniqueId; +import net.opentsdb.uid.UniqueId; +import org.hbase.async.Bytes; +import org.hbase.async.KeyValue; + +import java.util.*; + +/** + * Represents a read-only sequence of continuous histogram data points. + *

    + * This class stores a continuous sequence of {@link HistogramRowSeq}s in memory. + * + * @since 2.4 + */ +public class HistogramSpan implements HistogramDataPoints { + + /** + * The {@link TSDB} instance we belong to. + */ + protected final TSDB tsdb; + + /** + * All the rows in this span. + */ + protected List rows = new ArrayList(); + + /** + * A list of annotations for this span. We can't lazily initialize since we + * have to pass a collection to the compaction queue + */ + private List annotations = new ArrayList(0); + + /** + * Whether or not the rows have been sorted. This should be toggled by the + * first call to an iterator method + */ + private boolean sorted; + + /** + * Stores a warning about a prematurely terminated query + */ + private String warning; + + /** + * Default constructor. + * + * @param tsdb The TSDB to which we belong + */ + protected HistogramSpan(final TSDB tsdb) { + this.tsdb = tsdb; + } + + public String getWarning() { + return warning; + } + + /** + * Store a warning about the query + * + * @param warning The warning to store + */ + public void setWarning(final String warning) { + this.warning = warning; + } + + /** + * @throws IllegalStateException if the span doesn't have any rows + */ + private void checkNotEmpty() { + if (rows.size() == 0) { + throw new IllegalStateException("empty Span"); + } + } + + /** + * @return the name of the metric associated with the rows in this span + * @throws IllegalStateException if the span was empty + * @throws NoSuchUniqueId if the row key UID did not exist + */ + public String metricName() { + try { + return metricNameAsync().join(); + } catch (InterruptedException iex) { + throw new RuntimeException("Interrupted the metric name call", iex); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Should never be here", e); + } + } + + public Deferred metricNameAsync() { + checkNotEmpty(); + return rows.get(0).metricNameAsync(); + } + + public byte[] metricUID() { + return rows.get(0).metricUID(); + } + + /** + * @return the list of tag pairs for the rows in this span + * @throws IllegalStateException if the span was empty + * @throws NoSuchUniqueId if the any of the tagk/v UIDs did not exist + */ + public Map getTags() { + try { + return getTagsAsync().join(); + } catch (InterruptedException iex) { + throw new RuntimeException("Interrupted the tags call", iex); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Should never be here", e); + } + } + + public Deferred> getTagsAsync() { + checkNotEmpty(); + return rows.get(0).getTagsAsync(); + } + + @Override + public Bytes.ByteMap getTagUids() { + checkNotEmpty(); + return rows.get(0).getTagUids(); + } + + /** + * @return an empty list since aggregated tags cannot exist on a single span + */ + public List getAggregatedTags() { + return Collections.emptyList(); + } + + public Deferred> getAggregatedTagsAsync() { + final List empty = Collections.emptyList(); + return Deferred.fromResult(empty); + } + + @Override + public List getAggregatedTagUids() { + return Collections.emptyList(); + } + + /** + * @return the number of data points in this span, O(n) Unfortunately we must + * walk the entire array for every row as there may be a mix of second + * and millisecond timestamps + */ + public int size() { + int size = 0; + for (final iHistogramRowSeq row : rows) { + size += row.size(); + } + return size; + } + + /** + * @return 0 since aggregation cannot happen at the span level + */ + public int aggregatedSize() { + return 0; + } + + public List getTSUIDs() { + if (rows.size() < 1) { + return null; + } + final byte[] tsuid = UniqueId.getTSUIDFromKey(rows.get(0).key(), + TSDB.metrics_width(), Const.TIMESTAMP_BYTES); + final List tsuids = new ArrayList(1); + tsuids.add(UniqueId.uidToString(tsuid)); + return tsuids; + } + + /** + * @return a list of annotations associated with this span. May be empty + */ + public List getAnnotations() { + return annotations; + } + + /** + * Adds a row of histogram data points to the span, merging with an existing + * RowSeq or creating a new one if necessary. + *

    + * Take the following rows as an example. Salt A has a row T0 at the base time T0, + * a row T2 at the base time T2...; Salt B has a row T1 at the base time T1, + * a row T3 at the base time T3... + * + * Since the salt is generated basing the following hash code: + * 

    +   * {@code 
+   *  int modulo = (new String(Arrays.copyOfRange(rowKey,  Const.SALT_WIDTH(),
+   *  rowKey.length))).hashCode() % Const.SALT_BUCKETS();
+   *  byte[] salt = Internal.getSalt(modulo);
+   * }
+   *
    + * This hash scheme guarantees that all the data points with the same base time will + * has the same salt. + * 
    +   * |---A---|       |---B---|
+   * |  T0   |       |   T1  |
+   * |-------|       |-------|
+   * |  T2   |       |   T3  |
+   * |-------|       |-------|
+   * |  T4   |       |   T5  |
+   * |-------|       |-------|
+   *
    + * + * This method expects the caller adds the rows from the same salt in the + * timestamp order. When the caller adds the rows from salt A, the result + * rows will be as below: + * 
    +   * |-------|       
+   * |  T0   |      
+   * |-------|      
+   * |  T2   |      
+   * |-------|       
+   * |  T4   |       
+   * |-------|
+   *
    + * + * After the caller adds the rows from salt B, the result rows will be as below: + * 
    +   * |-------|       
+   * |  T0   |      
+   * |-------|      
+   * |  T2   |      
+   * |-------|       
+   * |  T4   |       
+   * |-------|
+   * |  T1   |
+   * |-------|
+   * |  T3   |
+   * |-------|
+   * |  T5   |
+   * |-------|
+   *
    + * When the caller iterates the data points in the Span, the Span will firstly + * sort the rows. Then the final result rows will be as below: + * 
    +   * |-------|       
+   * |  T0   |      
+   * |-------|      
+   * |  T1   |      
+   * |-------|       
+   * |  T2   |       
+   * |-------|
+   * |  T3   |
+   * |-------|
+   * |  T4   |
+   * |-------|
+   * |  T5   |
+   * |-------|
+   *
    + *

    + * @param key The row key of the row that want to add in the span + * @param data_points histogram data points in the row + * @throws IllegalArgumentException if the argument and this span are for + * two different time series. + */ + protected void addRow(final byte[] key, + final List data_points) { + if (null == key || null == data_points) { + throw new NullPointerException("row key and histogram data points " + + "can't be null"); + } + + long last_ts = 0; + if (rows.size() != 0) { + // Verify that we have the same metric id and tags. + final iHistogramRowSeq last = rows.get(rows.size() - 1); + final short metric_width = tsdb.metrics.width(); + final short tags_offset = (short) (Const.SALT_WIDTH() + metric_width + + Const.TIMESTAMP_BYTES); + final short tags_bytes = (short) (key.length - tags_offset); + String error = null; + if (key.length != last.key().length) { + error = "row key length mismatch"; + } else if (Bytes.memcmp(key, last.key(), Const.SALT_WIDTH(), metric_width) != 0) { + error = "metric ID mismatch"; + } else if (Bytes.memcmp(key, last.key(), tags_offset, tags_bytes) != 0) { + error = "tags mismatch"; + } + if (error != null) { + throw new IllegalArgumentException(error + ". " + + "This Span's last row key is " + Arrays.toString(last.key()) + + " whereas the row key being added is " + Arrays.toString(key) + + " and metric_width=" + metric_width); + } + last_ts = last.timestamp(last.size() - 1); // O(n) + } + + final iHistogramRowSeq rowseq = createRowSequence(tsdb); + rowseq.setRow(key, data_points); + sorted = false; + if (last_ts >= rowseq.timestamp(0)) { + // scan to see if we need to merge into an existing row + for (final iHistogramRowSeq rs : rows) { + if ((rs.key().length == key.length) + && (Bytes.memcmp(rs.key(), key, Const.SALT_WIDTH(), + (rs.key().length - Const.SALT_WIDTH())) == 0)) { + rs.addRow(data_points); + return; + } + } + } + + rows.add(rowseq); + } + + /** + * Package private helper to access the last timestamp in an HBase row. + * + * @param metric_width The number of bytes on which metric IDs are stored. + * @param row A compacted HBase row. + * @return A strictly positive timestamp in seconds or ms. + * @throws IllegalArgumentException if {@code row} doesn't contain any cell. + */ + static long lastTimestampInRow(final short metric_width, final KeyValue row) { + final long base_time = Internal.baseTime(row.key()); + final byte[] qual = row.qualifier(); + return Internal.getTimeStampFromNonDP(base_time, qual); + } + + /** + * @return an iterator to run over the list of data points + */ + public HistogramSeekableView iterator() { + checkRowOrder(); + return spanIterator(); + } + + /** + * Finds the index of the row of the ith data point and the offset in the row. + * + * @param i The index of the data point to find. + * @return two ints packed in a long. The first int is the index of the row in + * {@code rows} and the second is offset in that {@link RowSeq} + * instance. + */ + private long getIdxOffsetFor(final int i) { + checkRowOrder(); + int idx = 0; + int offset = 0; + for (final iHistogramRowSeq row : rows) { + final int sz = row.size(); + if (offset + sz > i) { + break; + } + offset += sz; + idx++; + } + return ((long) idx << 32) | (i - offset); + } + + /** + * Returns the timestamp for a data point at index {@code i} if it exists. + * Note: To get to a timestamp this method must walk the entire byte + * array, i.e. O(n) so call this sparingly. Use the iterator instead. + * + * @param i A 0 based index incremented per the number of data points in the + * span. + * @return A Unix epoch timestamp in milliseconds + * @throws IndexOutOfBoundsException + * if the index would be out of bounds + */ + public long timestamp(final int i) { + checkRowOrder(); + final long idxoffset = getIdxOffsetFor(i); + final int idx = (int) (idxoffset >>> 32); + final int offset = (int) (idxoffset & 0x00000000FFFFFFFF); + return rows.get(idx).timestamp(offset); + } + + /** + * Returns a human readable string representation of the object. + */ + @Override + public String toString() { + final StringBuilder buf = new StringBuilder(); + buf.append("HistogramSpan(").append(rows.size()).append(" rows, ["); + for (int i = 0; i < rows.size(); i++) { + if (i != 0) { + buf.append(", "); + } + buf.append(rows.get(i).toString()); + } + buf.append("])"); + return buf.toString(); + } + + /** + * Finds the index of the row in which the given timestamp should be. + * + * @param timestamp A strictly positive 32-bit integer. + * @return A strictly positive index in the {@code rows} array. + */ + private int seekRow(final long timestamp) { + checkRowOrder(); + int row_index = 0; + iHistogramRowSeq row = null; + final int nrows = rows.size(); + for (int i = 0; i < nrows; i++) { + row = rows.get(i); + final int sz = row.size(); + if (sz < 1) { + row_index++; + } else if (row.timestamp(sz - 1) < timestamp) { + row_index++; // The last DP in this row is before 'timestamp'. + } else { + break; + } + } + if (row_index == nrows) { // If this timestamp was too large for the + --row_index; // last row, return the last row. + } + return row_index; + } + + /** + * Checks the sorted flag and sorts the rows if necessary. Should be called by + * any iteration method. Since 2.0 + */ + private void checkRowOrder() { + if (!sorted) { + Collections.sort(rows, new HistogramRowSeq.HistogramRowSeqComparator()); + sorted = true; + } + } + + /** + * Package private iterator method to access it as a Span.Iterator. + */ + HistogramSpan.Iterator spanIterator() { + checkRowOrder(); + return new HistogramSpan.Iterator(); + } + + /** + * Iterator for {@link HistogramSpan}s. + */ + final class Iterator implements HistogramSeekableView { + + /** + * Index of the {@link HistogramRowSeq} we're currently at, in {@code rows}. + */ + private int row_index; + + /** + * Iterator on the current row. + */ + private iHistogramRowSeq.Iterator current_row; + + Iterator() { + current_row = rows.get(0).internalIterator(); + } + + // ------------------ // + // Iterator interface // + // ------------------ // + + @Override + public boolean hasNext() { + if (current_row.hasNext()) { + return true; + } + // handle situations where a row in the middle may be empty due to some + // kind of logic kicking out data points + while (row_index < rows.size() - 1) { + row_index++; + current_row = rows.get(row_index).internalIterator(); + if (current_row.hasNext()) { + return true; + } + } + return false; + } + + @Override + public HistogramDataPoint next() { + if (current_row.hasNext()) { + return current_row.next(); + } + // handle situations where a row in the middle may be empty due to some + // kind of logic kicking out data points + while (row_index < rows.size() - 1) { + row_index++; + current_row = rows.get(row_index).internalIterator(); + if (current_row.hasNext()) { + return current_row.next(); + } + } + throw new NoSuchElementException("no more elements"); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + // ---------------------- // + // SeekableView interface // + // ---------------------- // + + @Override + public void seek(final long timestamp) { + int row_index = seekRow(timestamp); + if (row_index != this.row_index) { + this.row_index = row_index; + current_row = rows.get(row_index).internalIterator(); + } + current_row.seek(timestamp); + } + + @Override + public String toString() { + return "HistogramSpan.Iterator(row_index=" + row_index + + ", current_row=" + current_row + ", span=" + + HistogramSpan.this + ')'; + } + + } + + /** + * + * @param start_time The time in milliseconds at which the data begins. + * @param end_time The time in milliseconds at which the data ends. + * @param downsampler The downsampling specification to use + * @param is_rollup Whether or not the query is handling rollup data + * @param query_start Start of the actual query + * @param query_end End of the actual query + * @return A new downsampler. + * @since 2.3 + */ + HistogramDownsampler downsampler(final long start_time, + final long end_time, + final DownsamplingSpecification downsampler, + final boolean is_rollup, + final long query_start, + final long query_end) { + // ignore the fill policy + return new HistogramDownsampler(spanIterator(), downsampler, query_start, + query_end); + } + + /** + * Return the query index that maps this datapoints to the original subquery + * + * @return index of the query in the TSQuery class + */ + public int getQueryIndex() { + throw new UnsupportedOperationException("Span.java: getQueryIndex not " + + "supported"); + } + + /** + * RowSeq abstract factory API implementation + * + * @param tsdb The TSDB to which we belong + * @return RowSeq object which stores read-only sequence of continuous HBase + * rows + */ + protected iHistogramRowSeq createRowSequence(TSDB tsdb) { + return new HistogramRowSeq(tsdb); + } +} diff --git a/src/core/HistogramSpanGroup.java b/src/core/HistogramSpanGroup.java new file mode 100644 index 0000000000..496a6f2485 --- /dev/null +++ b/src/core/HistogramSpanGroup.java @@ -0,0 +1,529 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.hbase.async.Bytes; +import org.hbase.async.Bytes.ByteMap; + +import com.stumbleupon.async.Callback; +import com.stumbleupon.async.Deferred; + +import net.opentsdb.meta.Annotation; +import net.opentsdb.utils.ByteSet; + +/** + * Clone of the regular SpanGroup but handles histogram data points. + * + * @since 2.4 + */ +final class HistogramSpanGroup implements HistogramDataPoints { + + /** Annotations */ + private final ArrayList annotations; + + /** Start time (UNIX timestamp in seconds or ms) on 32 bits ("unsigned" int). */ + private final long start_time; + + /** End time (UNIX timestamp in seconds or ms) on 32 bits ("unsigned" int). */ + private final long end_time; + + /** + * The tags of this group as names and UIDs + * This is the intersection set between the tags of all the Spans + * in this group. + * @see #computeTags + */ + private Map tags; + private ByteMap tag_uids; + + /** + * The names of the tags that aren't shared by every single data point. + * This is the symmetric difference between the tags of all the Spans + * in this group. + * @see #computeTags + */ + private List aggregated_tags; + private Set aggregated_tag_uids; + + /** Spans in this group. They must all be for the same metric. */ + private final ArrayList spans = new ArrayList(); + + /** Aggregation to use to aggregate data points from different Spans. */ + private final HistogramAggregation aggregation; + + /** + * Downsampling spec to use, if any (can be {@code null}). + */ + private final DownsamplingSpecification downsampler; + + /** Start timestamp of the query for filtering */ + private final long query_start; + + /** End timestamp of the query for filtering */ + private final long query_end; + + /** index of the query in the TSQuery class */ + private int query_index; + + /** The TSDB to which we belong, used for resolution */ + private final TSDB tsdb; + + /** If non-null, a list of tags we're grouping on in the query for determining + * whether or not a tag should be moved to the agg tags list */ + private final ByteSet query_tags; + + /** whether we are handling rollup data points*/ + private final boolean is_rollup; + + /** + * Ctor. + * @param tsdb The TSDB we belong to. + * @param start_time Any data point strictly before this timestamp will be ignored. + * @param end_time Any data point strictly after this timestamp will be ignored. + * @param spans A sequence of initial {@link HistogramSpan} to add to this group. + * Ignored if {@code null}. Additional spans can be added with {@link #add}. + * @param aggregation The aggregation function to use. + * @param downsampler The downsampling specification to use + * @param query_start Start of the actual query + * @param query_end End of the actual query + * @param query_index Index of the sub query + */ + HistogramSpanGroup(final TSDB tsdb, + final long start_time, + final long end_time, + final Iterable spans, + final HistogramAggregation aggregation, + final DownsamplingSpecification downsampler, + final long query_start, + final long query_end, + final int query_index, + final boolean is_rollup, + final ByteSet query_tags) { + annotations = new ArrayList(); + this.start_time = (start_time & Const.SECOND_MASK) == 0 ? + start_time * 1000 : start_time; + this.end_time = (end_time & Const.SECOND_MASK) == 0 ? + end_time * 1000 : end_time; + if (spans != null) { + for (final HistogramSpan span : spans) { + add(span); + } + } + + this.aggregation = aggregation; + this.downsampler = downsampler; + this.query_start = query_start; + this.query_end = query_end; + this.query_index = query_index; + this.tsdb = tsdb; + this.is_rollup = is_rollup; + this.query_tags = query_tags; + } + + public String getWarning() { + // only one of these should be set so grab the first we find. It *should* + // be the first one in the list if this was sorted properly. + for (final HistogramSpan span : spans) { + if (span.getWarning() != null) { + return span.getWarning(); + } + } + return null; + } + + /** + * Adds a span to this group, provided that it's in the right time range. + * Must not be called once {@link #getTags} or + * {@link #getAggregatedTags} has been called on this instance. + * @param span The span to add to this group. If none of the data points + * fall within our time range, this method will silently ignore that span. + */ + void add(final HistogramSpan span) { + if (tags != null) { + throw new AssertionError("The set of tags has already been computed" + + ", you can't add more Spans to " + this); + } + + // normalize timestamps to milliseconds for proper comparison + final long start = (start_time & Const.SECOND_MASK) == 0 ? + start_time * 1000 : start_time; + final long end = (end_time & Const.SECOND_MASK) == 0 ? + end_time * 1000 : end_time; + + if (span.size() == 0) { + // copy annotations that are in the time range + for (Annotation annot : span.getAnnotations()) { + long annot_start = annot.getStartTime(); + if ((annot_start & Const.SECOND_MASK) == 0) { + annot_start *= 1000; + } + long annot_end = annot.getStartTime(); + if ((annot_end & Const.SECOND_MASK) == 0) { + annot_end *= 1000; + } + if (annot_end >= start && annot_start <= end) { + annotations.add(annot); + } + } + } else { + long first_dp = span.timestamp(0); + if ((first_dp & Const.SECOND_MASK) == 0) { + first_dp *= 1000; + } + // The following call to timestamp() will throw an + // IndexOutOfBoundsException if size == 0, which is OK since it would + // be a programming error. + long last_dp = span.timestamp(span.size() - 1); + if ((last_dp & Const.SECOND_MASK) == 0) { + last_dp *= 1000; + } + if (first_dp <= end && last_dp >= start) { + this.spans.add(span); + annotations.addAll(span.getAnnotations()); + } + } + } + + public String metricName() { + try { + return metricNameAsync().join(); + } catch (InterruptedException iex) { + throw new RuntimeException("Interrupted the metric name call", iex); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Should never be here", e); + } + } + + public Deferred metricNameAsync() { + return spans.isEmpty() ? Deferred.fromResult("") : + spans.get(0).metricNameAsync(); + } + + public byte[] metricUID() { + return spans.isEmpty() ? new byte[] {} : spans.get(0).metricUID(); + } + + public Map getTags() { + try { + return getTagsAsync().join(); + } catch (InterruptedException iex) { + throw new RuntimeException("Interrupted the tags call", iex); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Should never be here", e); + } + } + + public Deferred> getTagsAsync() { + if (tags != null) { + return Deferred.fromResult(tags); + } + + if (spans.isEmpty()) { + tags = new HashMap(0); + return Deferred.fromResult(tags); + } + + if (tag_uids == null) { + computeTags(); + } + + return resolveTags(tag_uids); + } + + @Override + public ByteMap getTagUids() { + if (tag_uids == null) { + computeTags(); + } + return tag_uids; + } + + public List getAggregatedTags() { + try { + return getAggregatedTagsAsync().join(); + } catch (InterruptedException iex) { + throw new RuntimeException("Interrupted the aggregated tags call", iex); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Should never be here", e); + } + } + + public Deferred> getAggregatedTagsAsync() { + if (aggregated_tags != null) { + return Deferred.fromResult(aggregated_tags); + } + + if (spans.isEmpty()) { + aggregated_tags = new ArrayList(0); + return Deferred.fromResult(aggregated_tags); + } + + if (aggregated_tag_uids == null) { + computeTags(); + } + + return resolveAggTags(aggregated_tag_uids); + } + + @Override + public List getAggregatedTagUids() { + if (aggregated_tag_uids != null) { + return new ArrayList(aggregated_tag_uids); + } + + if (spans.isEmpty()) { + return Collections.emptyList(); + } + + if (aggregated_tag_uids == null) { + computeTags(); + } + return new ArrayList(aggregated_tag_uids); + } + + public List getTSUIDs() { + List tsuids = new ArrayList(spans.size()); + for (HistogramSpan hsp : spans) { + tsuids.addAll(hsp.getTSUIDs()); + } + return tsuids; + } + + /** + * Compiles the annotations for each span into a new array list + * @return Null if none of the spans had any annotations, a list if one or + * more were found + */ + public List getAnnotations() { + return annotations.isEmpty() ? null : annotations; + } + + public int size() { + // TODO(tsuna): There is a way of doing this way more efficiently by + // inspecting the Spans and counting only data points that fall in + // our time range. + final HistogramSeekableView it = iterator(); + int size = 0; + while (it.hasNext()) { + it.next(); + size++; + } + return size; + } + + public int aggregatedSize() { + int size = 0; + for (final HistogramSpan span : spans) { + size += span.size(); + } + return size; + } + + public HistogramSeekableView iterator() { + return HistogramAggregationIterator.create(spans, start_time, + end_time, aggregation, downsampler, query_start, query_end, is_rollup); + } + + /** + * Finds the {@code i}th data point of this group in {@code O(n)}. + * Where {@code n} is the number of data points in this group. + */ + private HistogramDataPoint getDataPoint(int i) { + if (i < 0) { + throw new IndexOutOfBoundsException("negative index: " + i); + } + final int saved_i = i; + final HistogramSeekableView it = iterator(); + HistogramDataPoint dp = null; + while (it.hasNext() && i >= 0) { + dp = it.next(); + i--; + } + if (i != -1 || dp == null) { + throw new IndexOutOfBoundsException("index " + saved_i + + " too large (it's >= " + size() + ") for " + this); + } + return dp; + } + + public long timestamp(final int i) { + return getDataPoint(i).timestamp(); + } + + public String toString() { + return "HistogramSpanGroup(" + toStringSharedAttributes() + + ", spans=" + spans + + ')'; + } + + private String toStringSharedAttributes() { + return "start_time=" + start_time + + ", end_time=" + end_time + + ", tags=" + tags + + ", aggregated_tags=" + aggregated_tags + + ", aggregator=" + aggregation + + ", downsampler=" + downsampler + + ')'; + } + + /** + * Return the query index that maps this datapoints to the original subquery + * @return index of the query in the TSQuery class + */ + public int getQueryIndex() { + return this.query_index; + } + + /** + * Computes the intersection set + symmetric difference of tags in all spans. + * This method loads the UID aggregated list and tag pair maps with byte arrays + * but does not actually resolve the UIDs to strings. + * On the first run, it will initialize the UID collections (which may be empty) + * and subsequent calls will skip processing. + */ + private void computeTags() { + if (tag_uids != null && aggregated_tag_uids != null) { + return; + } + if (spans.isEmpty()) { + tag_uids = new ByteMap(); + aggregated_tag_uids = new HashSet(); + return; + } + + // local tag uids + final ByteMap tag_set = new ByteMap(); + + // value is always null, we just want the set of unique keys + final ByteMap discards = new ByteMap(); + final Iterator it = spans.iterator(); + while (it.hasNext()) { + final HistogramSpan span = it.next(); + final ByteMap uids = span.getTagUids(); + + for (final Map.Entry tag_pair : uids.entrySet()) { + // we already know it's an aggregated tag + if (discards.containsKey(tag_pair.getKey())) { + continue; + } else if (query_tags != null && !query_tags.contains(tag_pair.getKey())) { + discards.put(tag_pair.getKey(), null); + continue; + } + + final byte[] tag_value = tag_set.get(tag_pair.getKey()); + if (tag_value == null) { + tag_set.put(tag_pair.getKey(), tag_pair.getValue()); + } else if (Bytes.memcmp(tag_value, tag_pair.getValue()) != 0) { + // bump to aggregated tags + discards.put(tag_pair.getKey(), null); + tag_set.remove(tag_pair.getKey()); + } + } + } + + aggregated_tag_uids = discards.keySet(); + tag_uids = tag_set; + } + + /** + * Resolves the set of tag keys to their string names. + * @param tagks The set of unique tag names + * @return a deferred to wait on for all of the tag keys to be resolved. The + * result should be null. + */ + private Deferred> resolveAggTags(final Set tagks) { + if (aggregated_tags != null) { + return Deferred.fromResult(null); + } + aggregated_tags = new ArrayList(tagks.size()); + + final List> names = + new ArrayList>(tagks.size()); + for (final byte[] tagk : tagks) { + names.add(tsdb.tag_names.getNameAsync(tagk)); + } + + /** Adds the names to the aggregated_tags list */ + final class ResolveCB implements Callback, ArrayList> { + @Override + public List call(final ArrayList names) throws Exception { + for (final String name : names) { + aggregated_tags.add(name); + } + return aggregated_tags; + } + } + + return Deferred.group(names).addCallback(new ResolveCB()); + } + + /** + * Resolves the tags to their names, loading them into {@link tags} after + * initializing that map. + * @param tag_uids The tag UIDs + * @return A defeferred to wait on for resolution to complete, the result + * should be null. + */ + private Deferred> resolveTags(final ByteMap tag_uids) { + if (tags != null) { + return Deferred.fromResult(null); + } + tags = new HashMap(tag_uids.size()); + + final List> deferreds = + new ArrayList>(tag_uids.size()); + + /** Dumps the pairs into the map in the correct order */ + final class PairCB implements Callback> { + @Override + public Object call(final ArrayList pair) throws Exception { + tags.put(pair.get(0), pair.get(1)); + return null; + } + } + + /** Callback executed once all of the pairs are resolved and stored in the map */ + final class GroupCB implements Callback, ArrayList> { + @Override + public Map call(final ArrayList group) + throws Exception { + return tags; + } + } + + for (Map.Entry tag_pair : tag_uids.entrySet()) { + final List> resolve_pair = + new ArrayList>(2); + resolve_pair.add(tsdb.tag_names.getNameAsync(tag_pair.getKey())); + resolve_pair.add(tsdb.tag_values.getNameAsync(tag_pair.getValue())); + deferreds.add(Deferred.groupInOrder(resolve_pair).addCallback(new PairCB())); + } + + return Deferred.group(deferreds).addCallback(new GroupCB()); + } +} diff --git a/src/core/IncomingDataPoint.java b/src/core/IncomingDataPoint.java index 0a7c70970d..df25306b99 100644 --- a/src/core/IncomingDataPoint.java +++ b/src/core/IncomingDataPoint.java @@ -1,5 +1,5 @@ // This file is part of OpenTSDB. -// Copyright (C) 2013 The OpenTSDB Authors. +// Copyright (C) 2013-2015 The OpenTSDB Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU Lesser General Public License as published by @@ -13,11 +13,16 @@ package net.opentsdb.core; import java.util.HashMap; +import java.util.List; import java.util.Map; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Bridging class that stores a normalized data point parsed from the "put" * RPC methods and gets it ready for storage. Also has some helper methods that @@ -32,22 +37,25 @@ * @since 2.0 */ @JsonInclude(Include.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) public class IncomingDataPoint { + private static final Logger LOG = LoggerFactory.getLogger(IncomingDataPoint.class); + /** The incoming metric name */ - private String metric; + protected String metric; /** The incoming timestamp in Unix epoch seconds or milliseconds */ - private long timestamp; + protected long timestamp; /** The incoming value as a string, we'll parse it to float or int later */ - private String value; + protected String value; /** A hash map of tag name/values */ - private HashMap tags; + protected Map tags; /** TSUID for the data point */ - private String tsuid; - + protected String tsuid; + /** * Empty constructor necessary for some de/serializers */ @@ -65,7 +73,7 @@ public IncomingDataPoint() { public IncomingDataPoint(final String metric, final long timestamp, final String value, - final HashMap tags) { + final Map tags) { this.metric = metric; this.timestamp = timestamp; this.value = value; @@ -119,7 +127,7 @@ public final String getValue() { } /** @return the tags */ - public final HashMap getTags() { + public final Map getTags() { return tags; } @@ -128,6 +136,11 @@ public final String getTSUID() { return tsuid; } + /** @param moretags the hashmap of kv pair to add */ + public final void addTags(HashMap moretags) { + this.tags.putAll(moretags); + } + /** @param metric the metric to set */ public final void setMetric(String metric) { this.metric = metric; @@ -152,4 +165,60 @@ public final void setTags(HashMap tags) { public final void setTSUID(String tsuid) { this.tsuid = tsuid; } + + /** + * Pre-validation of the various fields to make sure they're valid + * @param details a map to hold detailed error message. If null then + * the errors will be logged + * @return true if data point is valid, otherwise false + */ + public boolean validate(final List> details) { + if (this.getMetric() == null || this.getMetric().isEmpty()) { + if (details != null) { + details.add(getHttpDetails("Metric name was empty")); + } + LOG.warn("Metric name was empty: " + this); + return false; + } + + //TODO add blacklisted metric validatin here too + + if (this.getTimestamp() <= 0) { + if (details != null) { + details.add(getHttpDetails("Invalid timestamp")); + } + LOG.warn("Invalid timestamp: " + this); + return false; + } + + if (this.getValue() == null || this.getValue().isEmpty()) { + if (details != null) { + details.add(getHttpDetails("Empty value")); + } + LOG.warn("Empty value: " + this); + return false; + } + + if (this.getTags() == null || this.getTags().size() < 1) { + if (details != null) { + details.add(getHttpDetails("Missing tags")); + } + LOG.warn("Missing tags: " + this); + return false; + } + return true; + } + + /** + * Creates a map with an error message and this data point to return + * to the HTTP put data point RPC handler + * @param message The message to log + * @return A map to append to the HTTP response + */ + protected final Map getHttpDetails(final String message) { + final Map map = new HashMap(); + map.put("error", message); + map.put("datapoint", this); + return map; + } } diff --git a/src/core/IncomingDataPoints.java b/src/core/IncomingDataPoints.java index fc0b0a4125..108e641a6c 100644 --- a/src/core/IncomingDataPoints.java +++ b/src/core/IncomingDataPoints.java @@ -18,12 +18,15 @@ import java.util.Date; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; import com.stumbleupon.async.Callback; import com.stumbleupon.async.Deferred; +import org.hbase.async.AppendRequest; import org.hbase.async.Bytes; import org.hbase.async.PutRequest; +import org.hbase.async.Bytes.ByteMap; import net.opentsdb.meta.Annotation; import net.opentsdb.stats.Histogram; @@ -33,38 +36,44 @@ */ final class IncomingDataPoints implements WritableDataPoints { - /** For how long to buffer edits when doing batch imports (in ms). */ + /** For how long to buffer edits when doing batch imports (in ms). */ private static final short DEFAULT_BATCH_IMPORT_BUFFER_INTERVAL = 5000; /** - * Keep track of the latency (in ms) we perceive sending edits to HBase. - * We want buckets up to 16s, with 2 ms interval between each bucket up to - * 100 ms after we which we switch to exponential buckets. + * Keep track of the latency (in ms) we perceive sending edits to HBase. We + * want buckets up to 16s, with 2 ms interval between each bucket up to 100 ms + * after we which we switch to exponential buckets. */ static final Histogram putlatency = new Histogram(16000, (short) 2, 100); + /** + * Keep track of the number of UIDs that came back null with auto_metric disabled. + */ + static final AtomicLong auto_metric_rejection_count = new AtomicLong(); + /** The {@code TSDB} instance we belong to. */ private final TSDB tsdb; + + /** Whether or not to allow out of order data. */ + private final boolean allow_out_of_order_data; /** - * The row key. - * 3 bytes for the metric name, 4 bytes for the base timestamp, 6 bytes per - * tag (3 for the name, 3 for the value). + * The row key. Optional salt + 3 bytes for the metric name, 4 bytes for + * the base timestamp, 6 bytes per tag (3 for the name, 3 for the value). */ private byte[] row; /** - * Qualifiers for individual data points. - * The last Const.FLAG_BITS bits are used to store flags (the type of the - * data point - integer or floating point - and the size of the data point - * in bytes). The remaining MSBs store a delta in seconds from the base - * timestamp stored in the row key. + * Qualifiers for individual data points. The last Const.FLAG_BITS bits are + * used to store flags (the type of the data point - integer or floating point + * - and the size of the data point in bytes). The remaining MSBs store a + * delta in seconds from the base timestamp stored in the row key. */ private short[] qualifiers; /** Each value in the row. */ private long[] values; - + /** Track the last timestamp written for this series */ private long last_ts; @@ -73,242 +82,253 @@ final class IncomingDataPoints implements WritableDataPoints { /** Are we doing a batch import? */ private boolean batch_import; + + /** The metric for this time series */ + private String metric; + + /** Copy of the tags given us by the caller */ + private Map tags; /** * Constructor. - * @param tsdb The TSDB we belong to. + * + * @param tsdb + * The TSDB we belong to. */ IncomingDataPoints(final TSDB tsdb) { this.tsdb = tsdb; - // the qualifiers and values were meant for pre-compacting the rows. We - // could implement this later, but for now we don't need to track the values - // as they'll just consume space during an import - //this.qualifiers = new short[3]; - //this.values = new long[3]; + allow_out_of_order_data = tsdb.getConfig() + .getBoolean("tsd.core.bulk.allow_out_of_order_timestamps"); } /** * Validates the given metric and tags. - * @throws IllegalArgumentException if any of the arguments aren't valid. + * + * @throws IllegalArgumentException + * if any of the arguments aren't valid. */ - static void checkMetricAndTags(final String metric, final Map tags) { + static void checkMetricAndTags(final String metric, + final Map tags) { if (tags.size() <= 0) { throw new IllegalArgumentException("Need at least one tag (metric=" + metric + ", tags=" + tags + ')'); - } else if (tags.size() > Const.MAX_NUM_TAGS) { + } else if (tags.size() > Const.MAX_NUM_TAGS()) { throw new IllegalArgumentException("Too many tags: " + tags.size() - + " maximum allowed: " + Const.MAX_NUM_TAGS + ", tags: " + tags); + + " maximum allowed: " + Const.MAX_NUM_TAGS() + ", tags: " + tags); } Tags.validateString("metric name", metric); for (final Map.Entry tag : tags.entrySet()) { - Tags.validateString("tag name", tag.getKey()); - Tags.validateString("tag value", tag.getValue()); + Tags.validateString("tag name with value [" + tag.getValue() + "]", + tag.getKey()); + Tags.validateString("tag value with key [" + tag.getKey() + "]", + tag.getValue()); } } /** - * Returns a partially initialized row key for this metric and these tags. - * The only thing left to fill in is the base timestamp. - */ - static byte[] rowKeyTemplate(final TSDB tsdb, - final String metric, - final Map tags) { + * Returns a partially initialized row key for this metric and these tags. The + * only thing left to fill in is the base timestamp. + */ + static byte[] rowKeyTemplate(final TSDB tsdb, final String metric, + final Map tags) { final short metric_width = tsdb.metrics.width(); final short tag_name_width = tsdb.tag_names.width(); final short tag_value_width = tsdb.tag_values.width(); final short num_tags = (short) tags.size(); - int row_size = (metric_width + Const.TIMESTAMP_BYTES - + tag_name_width * num_tags - + tag_value_width * num_tags); + int row_size = (Const.SALT_WIDTH() + metric_width + Const.TIMESTAMP_BYTES + + tag_name_width * num_tags + tag_value_width * num_tags); final byte[] row = new byte[row_size]; - short pos = 0; + short pos = (short) Const.SALT_WIDTH(); + + byte[] metric_id = (tsdb.config.auto_metric() ? tsdb.metrics.getOrCreateId(metric) + : tsdb.metrics.getId(metric)); - copyInRowKey(row, pos, (tsdb.config.auto_metric() ? - tsdb.metrics.getOrCreateId(metric) : tsdb.metrics.getId(metric))); + if(!tsdb.config.auto_metric() && metric_id == null) { + auto_metric_rejection_count.incrementAndGet(); + } + + copyInRowKey(row, pos, metric_id); pos += metric_width; pos += Const.TIMESTAMP_BYTES; - for(final byte[] tag : Tags.resolveOrCreateAll(tsdb, tags)) { + for (final byte[] tag : Tags.resolveOrCreateAll(tsdb, tags)) { copyInRowKey(row, pos, tag); pos += tag.length; } return row; } - - /** - * Returns a partially initialized row key for this metric and these tags. - * The only thing left to fill in is the base timestamp. - * @since 2.0 - */ - static Deferred rowKeyTemplateAsync(final TSDB tsdb, - final String metric, - final Map tags) { - final short metric_width = tsdb.metrics.width(); - final short tag_name_width = tsdb.tag_names.width(); - final short tag_value_width = tsdb.tag_values.width(); - final short num_tags = (short) tags.size(); - - int row_size = (metric_width + Const.TIMESTAMP_BYTES - + tag_name_width * num_tags - + tag_value_width * num_tags); - final byte[] row = new byte[row_size]; - - // Lookup or create the metric ID. - final Deferred metric_id; - if (tsdb.config.auto_metric()) { - metric_id = tsdb.metrics.getOrCreateIdAsync(metric); - } else { - metric_id = tsdb.metrics.getIdAsync(metric); - } - - // Copy the metric ID at the beginning of the row key. - class CopyMetricInRowKeyCB implements Callback { - public byte[] call(final byte[] metricid) { - copyInRowKey(row, (short) 0, metricid); - return row; - } - } - - // Copy the tag IDs in the row key. - class CopyTagsInRowKeyCB - implements Callback, ArrayList> { - public Deferred call(final ArrayList tags) { - short pos = metric_width; - pos += Const.TIMESTAMP_BYTES; - for (final byte[] tag : tags) { - copyInRowKey(row, pos, tag); - pos += tag.length; - } - // Once we've resolved all the tags, schedule the copy of the metric - // ID and return the row key we produced. - return metric_id.addCallback(new CopyMetricInRowKeyCB()); - } - } - - // Kick off the resolution of all tags. - return Tags.resolveOrCreateAllAsync(tsdb, tags) - .addCallbackDeferring(new CopyTagsInRowKeyCB()); - } public void setSeries(final String metric, final Map tags) { checkMetricAndTags(metric, tags); try { row = rowKeyTemplate(tsdb, metric, tags); + RowKey.prefixKeyWithSalt(row); } catch (RuntimeException e) { throw e; } catch (Exception e) { throw new RuntimeException("Should never happen", e); } + this.metric = metric; + this.tags = tags; size = 0; } /** * Copies the specified byte array at the specified offset in the row key. - * @param row The row key into which to copy the bytes. - * @param offset The offset in the row key to start writing at. - * @param bytes The bytes to copy. + * + * @param row + * The row key into which to copy the bytes. + * @param offset + * The offset in the row key to start writing at. + * @param bytes + * The bytes to copy. */ - private static void copyInRowKey(final byte[] row, final short offset, final byte[] bytes) { + private static void copyInRowKey(final byte[] row, final short offset, + final byte[] bytes) { System.arraycopy(bytes, 0, row, offset, bytes.length); } /** * Updates the base time in the row key. - * @param timestamp The timestamp from which to derive the new base time. + * + * @param timestamp + * The timestamp from which to derive the new base time. * @return The updated base time. */ private long updateBaseTime(final long timestamp) { // We force the starting timestamp to be on a MAX_TIMESPAN boundary - // so that all TSDs create rows with the same base time. Otherwise + // so that all TSDs create rows with the same base time. Otherwise // we'd need to coordinate TSDs to avoid creating rows that cover // overlapping time periods. final long base_time = timestamp - (timestamp % Const.MAX_TIMESPAN); - // Clone the row key since we're going to change it. We must clone it + // Clone the row key since we're going to change it. We must clone it // because the HBase client may still hold a reference to it in its // internal datastructures. row = Arrays.copyOf(row, row.length); - Bytes.setInt(row, (int) base_time, tsdb.metrics.width()); + Bytes.setInt(row, (int) base_time, Const.SALT_WIDTH() + tsdb.metrics.width()); + RowKey.prefixKeyWithSalt(row); // in case the timestamp will be involved in + // salting later tsdb.scheduleForCompaction(row, (int) base_time); return base_time; } /** * Implements {@link #addPoint} by storing a value with a specific flag. - * @param timestamp The timestamp to associate with the value. - * @param value The value to store. - * @param flags Flags to store in the qualifier (size and type of the data - * point). + * + * @param timestamp + * The timestamp to associate with the value. + * @param value + * The value to store. + * @param flags + * Flags to store in the qualifier (size and type of the data point). * @return A deferred object that indicates the completion of the request. */ - private Deferred addPointInternal(final long timestamp, final byte[] value, - final short flags) { + private Deferred addPointInternal(final long timestamp, + final byte[] value, final short flags) { if (row == null) { throw new IllegalStateException("setSeries() never called!"); } final boolean ms_timestamp = (timestamp & Const.SECOND_MASK) != 0; - + // we only accept unix epoch timestamps in seconds or milliseconds if (timestamp < 0 || (ms_timestamp && timestamp > 9999999999999L)) { throw new IllegalArgumentException((timestamp < 0 ? "negative " : "bad") - + " timestamp=" + timestamp - + " when trying to add value=" + Arrays.toString(value) + " to " + this); + + " timestamp=" + timestamp + " when trying to add value=" + + Arrays.toString(value) + " to " + this); } // always maintain last_ts in milliseconds if ((ms_timestamp ? timestamp : timestamp * 1000) <= last_ts) { - throw new IllegalArgumentException("New timestamp=" + timestamp - + " is less than or equal to previous=" + last_ts - + " when trying to add value=" + Arrays.toString(value) - + " to " + this); + if (allow_out_of_order_data) { + // as we don't want to perform any funky calculations to find out if + // we're still in the same time range, just pass it off to the regular + // TSDB add function. + return tsdb.addPointInternal(metric, timestamp, value, tags, flags); + } else { + throw new IllegalArgumentException("New timestamp=" + timestamp + + " is less than or equal to previous=" + last_ts + + " when trying to add value=" + Arrays.toString(value) + " to " + + this); + } } - last_ts = (ms_timestamp ? timestamp : timestamp * 1000); - long base_time = baseTime(); - long incoming_base_time; - if (ms_timestamp) { - // drop the ms timestamp to seconds to calculate the base timestamp - incoming_base_time = ((timestamp / 1000) - - ((timestamp / 1000) % Const.MAX_TIMESPAN)); - } else { - incoming_base_time = (timestamp - (timestamp % Const.MAX_TIMESPAN)); + /** Callback executed for chaining filter calls to see if the value + * should be written or not. */ + final class WriteCB implements Callback, Boolean> { + @Override + public Deferred call(final Boolean allowed) throws Exception { + if (!allowed) { + return Deferred.fromResult(null); + } + + + last_ts = (ms_timestamp ? timestamp : timestamp * 1000); + + long base_time = baseTime(); + long incoming_base_time; + if (ms_timestamp) { + // drop the ms timestamp to seconds to calculate the base timestamp + incoming_base_time = ((timestamp / 1000) - ((timestamp / 1000) % Const.MAX_TIMESPAN)); + } else { + incoming_base_time = (timestamp - (timestamp % Const.MAX_TIMESPAN)); + } + + if (incoming_base_time - base_time >= Const.MAX_TIMESPAN) { + // Need to start a new row as we've exceeded Const.MAX_TIMESPAN. + base_time = updateBaseTime((ms_timestamp ? timestamp / 1000 : timestamp)); + } + + // Java is so stupid with its auto-promotion of int to float. + final byte[] qualifier = Internal.buildQualifier(timestamp, flags); + + // TODO(tsuna): The following timing is rather useless. First of all, + // the histogram never resets, so it tends to converge to a certain + // distribution and never changes. What we really want is a moving + // histogram so we can see how the latency distribution varies over time. + // The other problem is that the Histogram class isn't thread-safe and + // here we access it from a callback that runs in an unknown thread, so + // we might miss some increments. So let's comment this out until we + // have a proper thread-safe moving histogram. + // final long start_put = System.nanoTime(); + // final Callback cb = new Callback() { + // public Object call(final Object arg) { + // putlatency.add((int) ((System.nanoTime() - start_put) / 1000000)); + // return arg; + // } + // public String toString() { + // return "time put request"; + // } + // }; + + // TODO(tsuna): Add an errback to handle some error cases here. + if (tsdb.getConfig().enable_appends()) { + final AppendDataPoints kv = new AppendDataPoints(qualifier, value); + final AppendRequest point = new AppendRequest(tsdb.table, row, TSDB.FAMILY, + AppendDataPoints.APPEND_COLUMN_QUALIFIER, kv.getBytes()); + point.setDurable(!batch_import); + return tsdb.client.append(point);/* .addBoth(cb) */ + } else { + final PutRequest point = RequestBuilder.buildPutRequest(tsdb.getConfig(), tsdb.table, row, TSDB.FAMILY, + qualifier, value, timestamp); + point.setDurable(!batch_import); + return tsdb.client.put(point)/* .addBoth(cb) */; + } + } + @Override + public String toString() { + return "IncomingDataPoints.addPointInternal Write Callback"; + } } - if (incoming_base_time - base_time >= Const.MAX_TIMESPAN) { - // Need to start a new row as we've exceeded Const.MAX_TIMESPAN. - base_time = updateBaseTime((ms_timestamp ? timestamp / 1000: timestamp)); + if (tsdb.getTSfilter() != null && tsdb.getTSfilter().filterDataPoints()) { + return tsdb.getTSfilter().allowDataPoint(metric, timestamp, value, tags, flags) + .addCallbackDeferring(new WriteCB()); } - - // Java is so stupid with its auto-promotion of int to float. - final byte[] qualifier = Internal.buildQualifier(timestamp, flags); - - final PutRequest point = new PutRequest(tsdb.table, row, TSDB.FAMILY, - qualifier, value); - // TODO(tsuna): The following timing is rather useless. First of all, - // the histogram never resets, so it tends to converge to a certain - // distribution and never changes. What we really want is a moving - // histogram so we can see how the latency distribution varies over time. - // The other problem is that the Histogram class isn't thread-safe and - // here we access it from a callback that runs in an unknown thread, so - // we might miss some increments. So let's comment this out until we - // have a proper thread-safe moving histogram. - //final long start_put = System.nanoTime(); - //final Callback cb = new Callback() { - // public Object call(final Object arg) { - // putlatency.add((int) ((System.nanoTime() - start_put) / 1000000)); - // return arg; - // } - // public String toString() { - // return "time put request"; - // } - //}; - - // TODO(tsuna): Add an errback to handle some error cases here. - point.setDurable(!batch_import); - return tsdb.client.put(point)/*.addBoth(cb)*/; + return Deferred.fromResult(true).addCallbackDeferring(new WriteCB()); } private void grow() { @@ -323,7 +343,7 @@ private void grow() { /** Extracts the base timestamp from the row key. */ private long baseTime() { - return Bytes.getUnsignedInt(row, tsdb.metrics.width()); + return Bytes.getUnsignedInt(row, Const.SALT_WIDTH() + tsdb.metrics.width()); } public Deferred addPoint(final long timestamp, final long value) { @@ -337,19 +357,18 @@ public Deferred addPoint(final long timestamp, final long value) { } else { v = Bytes.fromLong(value); } - final short flags = (short) (v.length - 1); // Just the length. + final short flags = (short) (v.length - 1); // Just the length. return addPointInternal(timestamp, v, flags); } public Deferred addPoint(final long timestamp, final float value) { if (Float.isNaN(value) || Float.isInfinite(value)) { throw new IllegalArgumentException("value is NaN or Infinite: " + value - + " for timestamp=" + timestamp); + + " for timestamp=" + timestamp); } - final short flags = Const.FLAG_FLOAT | 0x3; // A float stored on 4 bytes. + final short flags = Const.FLAG_FLOAT | 0x3; // A float stored on 4 bytes. return addPointInternal(timestamp, - Bytes.fromInt(Float.floatToRawIntBits(value)), - flags); + Bytes.fromInt(Float.floatToRawIntBits(value)), flags); } public void setBufferingTime(final short time) { @@ -389,15 +408,23 @@ public String metricName() { throw new RuntimeException("Should never be here", e); } } - + public Deferred metricNameAsync() { if (row == null) { - throw new IllegalStateException("setSeries never called before!"); + throw new IllegalStateException( + "The row key was null, setSeries was not called."); } - final byte[] id = Arrays.copyOfRange(row, 0, tsdb.metrics.width()); + final byte[] id = Arrays.copyOfRange( + row, Const.SALT_WIDTH(), tsdb.metrics.width() + Const.SALT_WIDTH()); return tsdb.metrics.getNameAsync(id); } + @Override + public byte[] metricUID() { + return Arrays.copyOfRange(row, Const.SALT_WIDTH(), + Const.SALT_WIDTH() + TSDB.metrics_width()); + } + public Map getTags() { try { return getTagsAsync().joinUninterruptibly(); @@ -408,6 +435,11 @@ public Map getTags() { } } + @Override + public ByteMap getTagUids() { + return Tags.getTagUids(row); + } + public Deferred> getTagsAsync() { return Tags.getTagsAsync(tsdb, row); } @@ -415,20 +447,25 @@ public Deferred> getTagsAsync() { public List getAggregatedTags() { return Collections.emptyList(); } - + public Deferred> getAggregatedTagsAsync() { final List empty = Collections.emptyList(); return Deferred.fromResult(empty); } - public List getTSUIDs() { + @Override + public List getAggregatedTagUids() { return Collections.emptyList(); } + public List getTSUIDs() { + return Collections.emptyList(); + } + public List getAnnotations() { return null; } - + public int size() { return size; } @@ -441,15 +478,18 @@ public SeekableView iterator() { return new DataPointsIterator(this); } - /** @throws IndexOutOfBoundsException if {@code i} is out of bounds. */ + /** + * @throws IndexOutOfBoundsException + * if {@code i} is out of bounds. + */ private void checkIndex(final int i) { if (i > size) { throw new IndexOutOfBoundsException("index " + i + " > " + size + " for this=" + this); } if (i < 0) { - throw new IndexOutOfBoundsException("negative index " + i - + " for this=" + this); + throw new IndexOutOfBoundsException("negative index " + i + " for this=" + + this); } } @@ -489,17 +529,14 @@ public String toString() { // length of the final string based on the row key and number of elements. final String metric = metricName(); final StringBuilder buf = new StringBuilder(80 + metric.length() - + row.length * 4 + size * 16); + + row.length * 4 + size * 16); final long base_time = baseTime(); buf.append("IncomingDataPoints(") - .append(row == null ? "" : Arrays.toString(row)) - .append(" (metric=") - .append(metric) - .append("), base_time=") - .append(base_time) - .append(" (") - .append(base_time > 0 ? new Date(base_time * 1000) : "no date") - .append("), ["); + .append(row == null ? "" : Arrays.toString(row)) + .append(" (metric=").append(metric).append("), base_time=") + .append(base_time).append(" (") + .append(base_time > 0 ? new Date(base_time * 1000) : "no date") + .append("), ["); for (short i = 0; i < size; i++) { buf.append('+').append(delta(qualifiers[i])); if (isInteger(i)) { @@ -516,4 +553,22 @@ public String toString() { return buf.toString(); } + @Override + public Deferred persist() { + return Deferred.fromResult((Object) null); + } + + public int getQueryIndex() { + throw new UnsupportedOperationException("Not mapped to a query"); + } + + @Override + public boolean isPercentile() { + return false; + } + + @Override + public float getPercentile() { + throw new UnsupportedOperationException("getPercentile not supported"); + } } diff --git a/src/core/Internal.java b/src/core/Internal.java index 89961f045f..146a272c78 100644 --- a/src/core/Internal.java +++ b/src/core/Internal.java @@ -30,8 +30,10 @@ /** * This class is not part of the public API. - * 

    - * ,____________________________,
+ */
+
+/**-
+ *  ,____________________________,
  * | This class is reserved for |
  * | OpenTSDB's internal usage! |
  * `----------------------------'
@@ -52,8 +54,10 @@
  *                ///-._ _ _ _ _ _ _}^ - - - - ~                     ~-- ,.-~
  *                                                                   /.-~
  *              You've been warned by the dragon!
- *

    - * This class is reserved for OpenTSDB's own internal usage only. If you use + * + */ + +/** This class is reserved for OpenTSDB's own internal usage only. If you use * anything from this package outside of OpenTSDB, a dragon will spontaneously * appear and eat you. You've been warned. *

    @@ -86,6 +90,22 @@ public static Scanner getScanner(final Query query) { return ((TsdbQuery) query).getScanner(); } + /** Returns a set of scanners, one for each bucket if salted, or one scanner + * if salting is disabled. + * @see TsdbQuery#getScanner() */ + public static List getScanners(final Query query) { + final List scanners = new ArrayList( + Const.SALT_WIDTH() > 0 ? Const.SALT_BUCKETS() : 1); + if (Const.SALT_WIDTH() > 0) { + for (int i = 0; i < Const.SALT_BUCKETS(); i++) { + scanners.add(((TsdbQuery) query).getScanner(i)); + } + } else { + scanners.add(((TsdbQuery) query).getScanner()); + } + return scanners; + } + /** @see RowKey#metricName */ public static String metricName(final TSDB tsdb, final byte[] id) { return RowKey.metricName(tsdb, id); @@ -93,9 +113,41 @@ public static String metricName(final TSDB tsdb, final byte[] id) { /** Extracts the timestamp from a row key. */ public static long baseTime(final TSDB tsdb, final byte[] row) { - return Bytes.getUnsignedInt(row, tsdb.metrics.width()); + return Bytes.getUnsignedInt(row, Const.SALT_WIDTH() + TSDB.metrics_width()); } - + + /** @return the time normalized to an hour boundary in epoch seconds */ + public static long baseTime(final long timestamp) { + if ((timestamp & Const.SECOND_MASK) != 0) { + // drop the ms timestamp to seconds to calculate the base timestamp + return ((timestamp / 1000) - + ((timestamp / 1000) % Const.MAX_TIMESPAN)); + } else { + return (timestamp - (timestamp % Const.MAX_TIMESPAN)); + } + } + + /** + * Extracts the timestamp from a row key. + * @param row The row to parse the timestamp from. + * @return The timestamp in Unix Epoch seconds. + * @since 2.4 + */ + public static long baseTime(final byte[] row) { + return Bytes.getUnsignedInt(row, Const.SALT_WIDTH() + TSDB.metrics_width()); + } + + /** + * Sets the time in a raw data table row key + * @param row The row to modify + * @param base_time The base time to store + * @since 2.3 + */ + public static void setBaseTime(final byte[] row, int base_time) { + Bytes.setInt(row, base_time, Const.SALT_WIDTH() + + TSDB.metrics_width()); + } + /** @see Tags#getTags */ public static Map getTags(final TSDB tsdb, final byte[] row) { return Tags.getTags(tsdb, row); @@ -189,30 +241,51 @@ public static ArrayList extractDataPoints(final ArrayList row, final byte[] qual = kv.qualifier(); final int len = qual.length; final byte[] val = kv.value(); - - if (len % 2 != 0) { - // skip a non data point column - continue; - } else if (len == 2) { // Single-value cell. - // Maybe we need to fix the flags in the qualifier. - final byte[] actual_val = fixFloatingPointValue(qual[1], val); - final byte q = fixQualifierFlags(qual[1], actual_val.length); - final byte[] actual_qual; - - if (q != qual[1]) { // We need to fix the qualifier. - actual_qual = new byte[] { qual[0], q }; // So make a copy. - } else { - actual_qual = qual; // Otherwise use the one we already have. + + // when enable_appends set to true, should get qualifier and value from the HBase Column Value + if (kv.qualifier()[0] == AppendDataPoints.APPEND_COLUMN_PREFIX) { + int idx = 0; + int q_length = 0; + int v_length = 0; + while (idx < kv.value().length) { + q_length = Internal.getQualifierLength(kv.value(), idx); + v_length = Internal.getValueLengthFromQualifier(kv.value(), idx); + final byte[] q = new byte[q_length]; + final byte[] v = new byte[v_length]; + System.arraycopy(kv.value(),idx,q,0,q_length); + System.arraycopy(kv.value(),idx + q_length,v, 0, v_length); + idx += q_length + v_length; + + final Cell cell = new Cell(q, v); + cells.add(cell); } - - final Cell cell = new Cell(actual_qual, actual_val); - cells.add(cell); - continue; - } else if (len == 4 && inMilliseconds(qual[0])) { - // since ms support is new, there's nothing to fix - final Cell cell = new Cell(qual, val); - cells.add(cell); continue; + } else { + + if (len % 2 != 0) { + // skip a non data point column + continue; + } else if (len == 2) { // Single-value cell. + // Maybe we need to fix the flags in the qualifier. + final byte[] actual_val = fixFloatingPointValue(qual[1], val); + final byte q = fixQualifierFlags(qual[1], actual_val.length); + final byte[] actual_qual; + + if (q != qual[1]) { // We need to fix the qualifier. + actual_qual = new byte[]{qual[0], q}; // So make a copy. + } else { + actual_qual = qual; // Otherwise use the one we already have. + } + + final Cell cell = new Cell(actual_qual, actual_val); + cells.add(cell); + continue; + } else if (len == 4 && inMilliseconds(qual[0])) { + // since ms support is new, there's nothing to fix + final Cell cell = new Cell(qual, val); + cells.add(cell); + continue; + } } // Now break it down into Cells. @@ -768,7 +841,7 @@ public static byte[] extractQualifier(final byte[] qualifier, * the timestamp is in seconds, this returns a 2 byte qualifier. If it's in * milliseconds, returns a 4 byte qualifier * @param timestamp A Unix epoch timestamp in seconds or milliseconds - * @param flags Flags to set on the qualifier (length &| float) + * @param flags Flags to set on the qualifier (length &| float) * @return A 2 or 4 byte qualifier for storage in column or compacted column * @since 2.0 */ @@ -844,7 +917,7 @@ public static void createAndSetTSUIDFilter(final Scanner scanner, buf.append("(?s)" // Ensure we use the DOTALL flag. + "^.{") // ... start by skipping the metric ID and timestamp. - .append(TSDB.metrics_width() + Const.TIMESTAMP_BYTES) + .append(Const.SALT_WIDTH() + metric_width + Const.TIMESTAMP_BYTES) .append("}("); for (final byte[] tags : uids) { @@ -859,4 +932,175 @@ public static void createAndSetTSUIDFilter(final Scanner scanner, buf.append("$"); scanner.setKeyRegexp(buf.toString(), Charset.forName("ISO-8859-1")); } + + /** + * Simple helper to calculate the max value for any width of long from 0 to 8 + * bytes. + * @param width The width of the byte array we're comparing + * @return The maximum unsigned integer value on {@code width} bytes. Note: + * If you ask for 8 bytes, it will return the max signed value. This is due + * to Java lacking unsigned integers... *sigh*. + * @since 2.2 + */ + public static long getMaxUnsignedValueOnBytes(final int width) { + if (width < 0 || width > 8) { + throw new IllegalArgumentException("Width must be from 1 to 8 bytes: " + + width); + } + if (width < 8) { + return ((long) 1 << width * Byte.SIZE) - 1; + } else { + return Long.MAX_VALUE; + } + } + + /** + * Encodes a long on 1, 2, 4 or 8 bytes + * @param value The value to encode + * @return A byte array containing the encoded value + * @since 2.4 + */ + public static byte[] vleEncodeLong(final long value) { + if (Byte.MIN_VALUE <= value && value <= Byte.MAX_VALUE) { + return new byte[] { (byte) value }; + } else if (Short.MIN_VALUE <= value && value <= Short.MAX_VALUE) { + return Bytes.fromShort((short) value); + } else if (Integer.MIN_VALUE <= value && value <= Integer.MAX_VALUE) { + return Bytes.fromInt((int) value); + } else { + return Bytes.fromLong(value); + } + } + + /** + * Returns true if the given TSUID matches the row key (accounting for salt and + * timestamp). + * @param tsuid A non-null TSUID array + * @param row_key A non-null row key array + * @return True if the TSUID matches the row key, false if not. + * @throws IllegalArgumentException if the arguments are invalid. + */ + public static boolean rowKeyMatchsTSUID(final byte[] tsuid, + final byte[] row_key) { + if (tsuid == null || row_key == null) { + throw new IllegalArgumentException("Neither tsuid or row key can be null"); + } + if (row_key.length <= tsuid.length) { + throw new IllegalArgumentException("Row key cannot be the same or " + + "shorter than tsuid"); + } + // check on the metric part + int index_tsuid = 0; + int index_row_key = 0; + for (index_tsuid = 0, index_row_key = Const.SALT_WIDTH(); index_tsuid < TSDB + .metrics_width(); ++index_tsuid, ++index_row_key) { + if (tsuid[index_tsuid] != row_key[index_row_key]) { + return false; + } + } + + // check on the tagk tagv part + for (index_tsuid = TSDB.metrics_width(), index_row_key = Const.SALT_WIDTH() + + TSDB.metrics_width() + Const.TIMESTAMP_BYTES; + index_tsuid < tsuid.length; ++index_tsuid, + ++index_row_key) { + if (tsuid[index_tsuid] != row_key[index_row_key]) { + return false; + } + } // end for + + return true; + } + + /** + * Calculates and returns the column qualifier. The qualifier is the offset + * of the {@code #timestamp} from the row key's base time stamp in seconds + * with a prefix of {@code #PREFIX}. Thus if the offset is 0 and the prefix is + * 1 and the timestamp is in seconds, the qualifier would be [1, 0, 0]. + * Millisecond timestamps will have a 5 byte qualifier. + * @param timestamp The base timestamp. + * @param prefix The prefix to set at the start of the array. + * @return The column qualifier as a byte array + * @throws IllegalArgumentException if the start_time has not been set + * @since 2.4 + */ + public static byte[] getQualifier(final long timestamp, final byte prefix) { + if (timestamp < 1) { + throw new IllegalArgumentException("The start timestamp has not been set"); + } + + final long base_time; + final byte[] qualifier; + if ((timestamp & Const.SECOND_MASK) != 0) { + // drop the ms timestamp to seconds to calculate the base timestamp + base_time = ((timestamp / 1000) - + ((timestamp / 1000) % Const.MAX_TIMESPAN)); + qualifier = new byte[5]; + final int offset = (int) (timestamp - (base_time * 1000)); + System.arraycopy(Bytes.fromInt(offset), 0, qualifier, 1, 4); + } else { + base_time = (timestamp - (timestamp % Const.MAX_TIMESPAN)); + qualifier = new byte[3]; + final short offset = (short) (timestamp - base_time); + System.arraycopy(Bytes.fromShort(offset), 0, qualifier, 1, 2); + } + qualifier[0] = prefix; + return qualifier; + } + + /** + * Get timestamp from base time and quantifier for non datapoints. The returned time + * will always be in ms. + * @param base_time the base time of the point + * @param quantifier the quantifier of the point, it is expected to be either length of + * 3 or length of 5 (the first byte represents the type of the point) + * @return The timestamp in ms + */ + public static long getTimeStampFromNonDP(final long base_time, byte[] quantifier) { + long ret = base_time; + if (quantifier.length == 3) { + ret += quantifier[1] << 8 | (quantifier[2] & 0xFF); + ret *= 1000; + } else if (quantifier.length == 5) { + ret *= 1000; + ret += (quantifier[1] & 0xFF) << 24 | (quantifier[2] & 0xFF) << 16 + | (quantifier[3] & 0xFF) << 8 | quantifier[4] & 0xFF; + } else { + throw new IllegalArgumentException("Quantifier is not valid: " + Bytes.pretty(quantifier)); + } + + return ret; + + } + + /** + * Decode the histogram point from the given key value + * @param kv the key value that contains a histogram + * @return the decoded {@code HistogramDataPoint} + */ + public static HistogramDataPoint decodeHistogramDataPoint(final TSDB tsdb, + final KeyValue kv) { + long timestamp = Internal.baseTime(kv.key()); + return decodeHistogramDataPoint(tsdb, timestamp, kv.qualifier(), kv.value()); + } + + /** + * Decode the histogram point from the given key and values + * @param tsdb The TSDB to use when fetching the decoder manager. + * @param base_time the base time of the histogram + * @param qualifier the qualifier used to store the histogram + * @param value the encoded value of the histogram + * @return the decoded {@code HistogramDataPoint} + */ + public static HistogramDataPoint decodeHistogramDataPoint(final TSDB tsdb, + final long base_time, + final byte[] qualifier, + final byte[] value) { + final HistogramDataPointCodec decoder = + tsdb.histogramManager().getCodec((int) value[0]); + long timestamp = getTimeStampFromNonDP(base_time, qualifier); + final Histogram histogram = decoder.decode(value, true); + return new SimpleHistogramDataPointAdapter(histogram, timestamp); + } + } diff --git a/src/core/MultiGetQuery.java b/src/core/MultiGetQuery.java new file mode 100644 index 0000000000..830a68b5e7 --- /dev/null +++ b/src/core/MultiGetQuery.java @@ -0,0 +1,1296 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.AbstractMap.SimpleEntry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import org.hbase.async.Bytes.ByteMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Maps; +import com.stumbleupon.async.Callback; +import com.stumbleupon.async.Deferred; + +import org.hbase.async.Bytes; +import org.hbase.async.GetRequest; +import org.hbase.async.GetResultOrException; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.hbase.async.KeyValue; + +import net.opentsdb.meta.Annotation; +import net.opentsdb.rollup.RollupInterval; +import net.opentsdb.rollup.RollupQuery; +import net.opentsdb.rollup.RollupSpan; +import net.opentsdb.rollup.RollupUtils; +import net.opentsdb.stats.QueryStats; +import net.opentsdb.stats.QueryStats.QueryStat; +import net.opentsdb.uid.UniqueId; +import net.opentsdb.utils.ByteSet; +import net.opentsdb.utils.DateTime; +import net.opentsdb.utils.JSON; + +/** + * Class that handles fetching TSDB data from storage using GetRequests instead + * of scanning for the data. This is only applicable if the query is using the + * "explicit_tags" feature and specified literal filters for exact matching. + * It also works best when used for data that has high cardinality but the query + * is fetching a small subset of that data. + * + * @since 2.4 + */ +public class MultiGetQuery { + private static final Logger LOG = LoggerFactory.getLogger(MultiGetQuery.class); + private final TSDB tsdb; + private final byte[] metric; + private final List> tags; + private final long start_row_time; // in sec + private final long end_row_time; // in sec + private final byte[] table_to_fetch; + private final TreeMap spans; + private final TreeMap histogramSpans; + private final RollupQuery rollup_query; + private final QueryStats query_stats; + private final int query_index; + + private int multi_get_wait_cnt; + + private int multi_get_num_get_requests; + + private final Map> kvsmap = + new ConcurrentHashMap>(); + + private final Map> annotMap = Collections + .synchronizedMap(new TreeMap>(new RowKey.SaltCmp())); + + private final Map>>> histMap = Maps.newConcurrentMap(); + + private final Deferred> results = + new Deferred>(); + + private final Deferred> histogramResults = + new Deferred>(); + + private final ArrayList> multi_get_tasks; + private final ArrayList multi_get_indexs; + + private long prepare_multi_get_start_time; + + private long prepare_multi_get_end_time; + + // the timestamp of starting fetching data + private long fetch_start_time; + + // the number of data point fetched + private AtomicLong number_pre_filter_data_point; + + private AtomicLong num_post_filter_data_points; + + // the byte size of the data fetched + private AtomicLong byte_size_fetched; + + // the finished multi get number + private AtomicInteger finished_multi_get_cnt; + + private AtomicInteger multi_get_seq_id; + + private final int concurrency_multi_get; + + private final int batch_size; + + /** Whether or not to fetch all possible salts for the rows in case the + * salting has changed during the TSD's run. */ + private final boolean get_all_salts; + + /** A holder for storing the first exception thrown by a scanner if something + * goes pear shaped. Make sure to synchronize on this object when checking + * for null or assigning from a scanner's callback. */ + private volatile Exception exception; + + private long max_bytes; + + private boolean multiget_no_meta; + + private AtomicLong number_byte_fetched; + + private final boolean is_rollup; + private final int rollup_agg_id; + private final int rollup_count_id; + + public MultiGetQuery(final TSDB tsdb, + final TsdbQuery query, + final byte[] metric, + final List> tags, + final long start_row_time, + final long end_row_time, + final byte[] table_to_fetch, + final TreeMap spans, + final TreeMap histogramSpans, + final long timeout, + final RollupQuery rollup_query, + final QueryStats query_stats, + final int query_index, + final long max_bytes, + final boolean override_count_limit, + final boolean multiget_no_meta) { + this.tsdb = tsdb; + this.metric = metric; + this.tags = tags; + this.start_row_time = start_row_time; + this.end_row_time = end_row_time; + this.table_to_fetch = table_to_fetch; + this.spans = spans; + this.histogramSpans = histogramSpans; + this.rollup_query = rollup_query; + this.query_stats = query_stats; + this.query_index = query_index; + this.multiget_no_meta = multiget_no_meta; + + if (tags == null) { + throw new IllegalArgumentException("Tags list cannot be null or empty"); + } + if (tags.isEmpty()) { + query.setNoResults(true); + } + if (end_row_time <= start_row_time) { + throw new IllegalArgumentException("Start time cannot be later or " + + "equal to the end time"); + } + + concurrency_multi_get = tsdb.config + .getInt("tsd.query.multi_get.concurrent"); + batch_size = tsdb.config.getInt("tsd.query.multi_get.batch_size"); + get_all_salts = tsdb.config.getBoolean("tsd.query.multi_get.get_all_salts"); + multi_get_tasks = new ArrayList>(concurrency_multi_get); + multi_get_indexs = new ArrayList(concurrency_multi_get); + for (int i = 0; i < concurrency_multi_get; ++i) { + multi_get_tasks.add(new ArrayList()); + multi_get_indexs.add(new AtomicInteger(-1)); + } + + number_pre_filter_data_point = new AtomicLong(0); + num_post_filter_data_points = new AtomicLong(0); + byte_size_fetched = new AtomicLong(0); + finished_multi_get_cnt = new AtomicInteger(0); + multi_get_seq_id = new AtomicInteger(-1); + number_byte_fetched = new AtomicLong(0); + this.max_bytes = max_bytes; + if (rollup_query != null && RollupQuery.isValidQuery(rollup_query)) { + is_rollup = true; + if (rollup_query.getRollupAgg() == Aggregators.AVG) { + rollup_agg_id = tsdb.getRollupConfig().getIdForAggregator("sum"); + rollup_count_id = tsdb.getRollupConfig().getIdForAggregator("count"); + } else { + rollup_agg_id = tsdb.getRollupConfig().getIdForAggregator( + rollup_query.getRollupAgg().toString()); + rollup_count_id = -1; + } + } else { + is_rollup = false; + rollup_agg_id = rollup_count_id = -1; + } + } + + /** + * Helper container class to store a set of TSUIDs and GetRequests in the same + * object. + */ + final static class MultiGetTask { + private final Set tsuids; + private final List gets; + + /** + * Default Ctor + * @param tsuids Non-null set of TSUIDs. May be empty. + * @param gets Non-null list of GetRequests. May be empty. + */ + public MultiGetTask(final Set tsuids, final List gets) { + this.tsuids = tsuids; + this.gets = gets; + } + + public Set getTSUIDs() { + return tsuids; + } + + public List getGets() { + return gets; + } + } + + ////////////////////////////////////////////////////////////////////////////////////////// + //// Call back to handle result + ///////////////////////////////////////////////////////////////////////////////////////// + final class MulGetCB implements Callback> { + private final int concurrency_index; + private final Set tsuids; + private final List gets; + private final int seq_id; + + private List keyValues = new ArrayList(); + private final Map> annotations = + new ConcurrentHashMap>(); + + // use list here because we want to keep the rows in the scan order - + // timestamp order. + // i don't want to define an additional class to store the information of + // the row key and the histogram data points in the row, then use {@link SimpleEntry} + private List>> histograms = + new ArrayList>>(); + + /////////////////////////////////////////////////////////////////////////// + // nanosecond times - trace response metrics // + /////////////////////////////////////////////////////////////////////////// + + // the time to start the multi get request + private long mul_get_start_time = -1; + + // cumulation of time waiting on HBase + private long mul_get_time = 0; + + // cumulation of time resolving uid + private long mul_get_uid_resolved_time = 0; + + // how many uids is resolved + private long mul_get_uids_resolved = 0; + + // cumulation of time compacting + private long mul_get_compaction_time = 0; + + // how many data points after filtering + private long mul_get_dps_post_filter = 0; + + // how many rows after filtering + private long mul_get_rows_post_filter = 0; + + // how many rows fetched from hbase + private long mul_get_number_row_fetched = 0; + + // how many cells fetched from hbase + private long mul_get_number_column_fetched = 0; + + // how many bytes fetched from hbase + private long mul_get_number_byte_fetched = 0; + + /** The exception thrown by this get request set */ + private Exception get_exception; + + public MulGetCB(final int concurrency_index, final Set tsuids, + final List gets) { + this.concurrency_index = concurrency_index; + this.tsuids = tsuids; + this.gets = gets; + + if (query_stats != null) { + seq_id = multi_get_seq_id.incrementAndGet(); + StringBuilder sb = new StringBuilder(); + sb.append("Mulget_").append(concurrency_index).append("_").append(seq_id); + query_stats.addScannerId(query_index, seq_id, sb.toString()); + } else { + seq_id = 0; + } + } + + /** Error callback that will capture an exception from AsyncHBase and store + * it so we can bubble it up to the caller. + */ + class ErrorCb implements Callback { + @Override + public Object call(final Exception e) throws Exception { + LOG.error("Multi get threw an exception: " + this, e); + MulGetCB.this.get_exception = e; + close(false); + return null; + } + } + + public Object fetch() { + + mul_get_start_time = DateTime.nanoTime(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Trying to fetch data for concurrency index: " + + concurrency_index + "; with " + gets.size() + " gets"); + } + return tsdb.client.get(gets) + .addCallback(this) + .addErrback(new ErrorCb()); + } + + /** + * Iterate through each row of the multi get results, parses out data + * points (and optional meta data). + * @return null if no rows were found, otherwise the TreeMap with spans + */ + @Override + public Object call(final List results) throws Exception { + mul_get_time = (DateTime.nanoTime() - mul_get_start_time); + + try { + for (final GetResultOrException result : results) { + // handle an exception + if (result.getException() != null) { + get_exception = result.getException(); + handleException(get_exception); + close(false); + } + if (null != result.getCells()) { + final ArrayList row = result.getCells(); + if (row.isEmpty()) { + continue; + } + + number_pre_filter_data_point.addAndGet(row.size()); + ++mul_get_number_row_fetched; + mul_get_number_column_fetched += row.size(); + + final byte[] key = row.get(0).key(); + final byte[] tsuid_key = UniqueId.getTSUIDFromKey(key, + TSDB.metrics_width(), Const.TIMESTAMP_BYTES); + + if (!tsuids.contains(tsuid_key)) { + LOG.error("Multi getter fetched the wrong row " + result + + " when fetching metric: " + Bytes.pretty(metric)); + continue; + } + process(key, row); + } else { + // TODO we don't get cells for some get requests. This could be an + // error or the database just didn't have data. Gotta look into it. + } + } // end for + } catch (Exception e) { + get_exception = e; + close(false); + return null; + } + + close(true); + return null; + } + + /** + * Handles processing of row of data into the proper list + * @param key The row key, possibly mutated + * @param row The row of KVs to process + * @return True if processing should continue, false if an exception occurred + * or the scanner was already closed (possibly due to another scanner error) + */ + boolean process(final byte[] key, final ArrayList row) { + ++mul_get_rows_post_filter; + num_post_filter_data_points.addAndGet(row.size()); + + List notes = null; + if (annotMap != null) { + notes = annotations.get(key); + + if (notes == null) { + notes = new ArrayList(); + annotations.put(key, notes); + } + } + + List hists = new ArrayList(); + if (RollupQuery.isValidQuery(rollup_query)) { + processRollupQuery(key, row, notes, hists); + } else { + processNotRollupQuery(key, row, notes, hists); + } + + return true; + } + + private void processNotRollupQuery(final byte[] key, + final ArrayList row, + List notes, + List hists) { + KeyValue compacted = null; + try { + final long compaction_start = DateTime.nanoTime(); + compacted = tsdb.compact(row, notes, hists); + + // histogram row + if (hists.size() > 0) { + histograms.add(new SimpleEntry>(key, hists)); + mul_get_dps_post_filter += hists.size(); + } + + mul_get_compaction_time += (DateTime.nanoTime() - compaction_start); + if (compacted != null) { + final byte[] compact_value = compacted.value(); + final byte[] compact_qualifier = compacted.qualifier(); + mul_get_number_byte_fetched = mul_get_number_byte_fetched + + compacted.value().length + compacted.key().length; + number_byte_fetched.addAndGet(compacted.value().length + compacted.key().length); + if (number_byte_fetched.get() > max_bytes) { + handleException( + new QueryException(HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE, + "Sorry, you have attempted to fetch more than our maximum " + + "amount of " + (max_bytes / 1024 / 1024) + "MB from storage. " + + "Please try reducing your time range or adjust the query filters.")); + close(false); + return; + } + if (compact_qualifier.length % 2 == 0) { + // The length of the qualifier is even so this is a put type + // so the size of the data is the length of the qualifier by 2 + if (compact_value[compact_value.length - 1] == 0) { + // LOG.debug("All data points we have here are either in seconds + // or Ms"); + if (Internal.inMilliseconds(compact_qualifier[0])) { + mul_get_dps_post_filter += compact_qualifier.length / 4; + } else { + mul_get_dps_post_filter += compact_qualifier.length / 2; + } + } else { + // LOG.debug("Data Points we have here are stored in second and Ms + // precision"); + // We wil make a estimate here as iterating over each qualifer + // could be expensive. + // We will just divide the qualifier by 3 to estimate the value + mul_get_dps_post_filter += compact_qualifier.length / 3; + } + } + } + } catch (IllegalDataException idex) { + LOG.error("Caught IllegalDataException exception while parsing the " + "row " + key + ", skipping index", idex); + } + + if (compacted != null) { // Can be null if we ignored all KVs. + keyValues.add(compacted); + } + } + + private void processRollupQuery(final byte[] key, + final ArrayList row, + List notes, + final List hists) { + for (KeyValue kv : row) { + mul_get_number_byte_fetched = mul_get_number_byte_fetched + + kv.value().length + kv.key().length; + number_byte_fetched.addAndGet(kv.value().length + kv.key().length); + if (number_byte_fetched.get() > max_bytes) { + handleException( + new QueryException(HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE, + "Sorry, you have attempted to fetch more than our maximum " + + "amount of " + (max_bytes / 1024 / 1024) + "MB from storage. " + + "Please try reducing your time range or adjust the query filters.")); + close(false); + return; + } + final byte[] qual = kv.qualifier(); + + if (qual.length > 0) { + // Todo: Bug! Here we shouldn't use the first byte to check the type + // of this row + // Instead should parse the byte array to find the suffix and + // determine the actual type + if (qual[0] == Annotation.PREFIX()) { + // This could be a row with only an annotation in it + final Annotation note = JSON.parseToObject(kv.value(), Annotation.class); + notes.add(note); + } else if (qual[0] == HistogramDataPoint.PREFIX) { + try { + HistogramDataPoint histogram = Internal.decodeHistogramDataPoint(tsdb, kv); + hists.add(histogram); + } catch (Throwable t) { + LOG.error("Failed to decode histogram data point", t); + } + } else { + if (qual[0] == (byte) rollup_agg_id || + qual[0] == (byte) rollup_count_id || + rollup_query.getRollupAgg() == Aggregators.AVG || + rollup_query.getRollupAgg() == Aggregators.DEV) { + if (Bytes.memcmp(RollupQuery.SUM, qual, 0, RollupQuery.SUM.length) == 0 + || Bytes.memcmp(RollupQuery.COUNT, qual, 0, RollupQuery.COUNT.length) == 0) { + keyValues.add(kv); + } + } else if (Bytes.memcmp(rollup_query.getRollupAggPrefix(), qual, 0, + rollup_query.getRollupAggPrefix().length) == 0) { + keyValues.add(kv); + } + } + } + } // end for + + // histogram row + if (hists.size() > 0) { + histograms.add(new SimpleEntry>(key, hists)); + } + } + + void close(final boolean ok) { + if (LOG.isDebugEnabled()) { + LOG.debug("Finished multiget on concurrency index: " + concurrency_index + + ", seq id: " + seq_id + ". Fetched rows: " + mul_get_number_row_fetched + + ", cells: " + mul_get_number_column_fetched + ", mget time(ms): " + + mul_get_time / 1000000 + " and " + ok); + } + + if (query_stats != null) { + query_stats.addScannerStat(query_index, seq_id, QueryStat.SCANNER_TIME, + DateTime.nanoTime() - mul_get_start_time); + + // Scanner Stats + query_stats.addScannerStat(query_index, seq_id, QueryStat.ROWS_FROM_STORAGE, + mul_get_number_row_fetched); + + query_stats.addScannerStat(query_index, seq_id, QueryStat.COLUMNS_FROM_STORAGE, + mul_get_number_column_fetched); + + query_stats.addScannerStat(query_index, seq_id, QueryStat.BYTES_FROM_STORAGE, + mul_get_number_byte_fetched); + + query_stats.addScannerStat(query_index, seq_id, QueryStat.HBASE_TIME, + mul_get_time); + query_stats.addScannerStat(query_index, seq_id, QueryStat.SUCCESSFUL_SCAN, + ok ? 1 : 0); + + // Post Scan stats + query_stats.addScannerStat(query_index, seq_id, QueryStat.ROWS_POST_FILTER, + mul_get_rows_post_filter); + query_stats.addScannerStat(query_index, seq_id, QueryStat.DPS_POST_FILTER, + mul_get_dps_post_filter); + query_stats.addScannerStat(query_index, seq_id, QueryStat.SCANNER_UID_TO_STRING_TIME, + mul_get_uid_resolved_time); + query_stats.addScannerStat(query_index, seq_id, QueryStat.UID_PAIRS_RESOLVED, + mul_get_uids_resolved); + query_stats.addScannerStat(query_index, seq_id, QueryStat.COMPACTION_TIME, + mul_get_compaction_time); + } + + if (ok) { + validateMultigetData(keyValues, annotations, histograms); + } + else { + finished_multi_get_cnt.incrementAndGet(); + } + + // check we have finished all the multi get + if (!checkAllFinishAndTriggerCallback()) { + // check to fire a new multi get in this concurrency bucket + List salt_mul_get_tasks = multi_get_tasks.get(concurrency_index); + int task_index = multi_get_indexs.get(concurrency_index).incrementAndGet(); + if (task_index < salt_mul_get_tasks.size()) { + MultiGetTask task = salt_mul_get_tasks.get(task_index); + MulGetCB mgcb = new MulGetCB(concurrency_index, task.getTSUIDs(), task.getGets()); + mgcb.fetch(); + } + } + } + } + + /** + * Initiate the get requests and return the tree map of results. + * @return A non-null tree map of results (may be empty) + */ + public Deferred> fetch() { + if(tags.isEmpty()) { + return Deferred.fromResult(null); + } + startFetch(); + return results; + } + + /** + * Initiate the get requests and return the tree map of results. + * @return A non-null tree map of results (may be empty) + */ + public Deferred> fetchHistogram() { + startFetch(); + return histogramResults; + } + + /** + * Start the work of firing up X concurrent get requests. + */ + private void startFetch() { + prepareConcurrentMultiGetTasks(); + + // set the time of starting + fetch_start_time = System.currentTimeMillis(); + if (LOG.isDebugEnabled()) { + LOG.debug("Start to fetch data using multiget, there will be " + multi_get_wait_cnt + + " multigets to call"); + } + + for (int con_idx = 0; con_idx < concurrency_multi_get; ++con_idx) { + final List con_mul_get_tasks = multi_get_tasks.get(con_idx); + final int task_index = multi_get_indexs.get(con_idx).incrementAndGet(); + + if (task_index < con_mul_get_tasks.size()) { + final MultiGetTask task = con_mul_get_tasks.get(task_index); + final MulGetCB mgcb = new MulGetCB(con_idx, task.getTSUIDs(), task.getGets()); + mgcb.fetch(); + } + } // end for + } + + /** + * Compiles the list of TSUIDs and GetRequests to send to execute against + * storage. Each batch will only have requests for one salt, i.e a batch + * will not have requests with multiple salts. + */ + @VisibleForTesting + void prepareConcurrentMultiGetTasks() { + multi_get_wait_cnt = 0; + prepare_multi_get_start_time = DateTime.currentTimeMillis(); + + + final List row_base_time_list; + if (RollupQuery.isValidQuery(rollup_query)) { + row_base_time_list = prepareRowBaseTimesRollup(); + } else { + row_base_time_list = prepareRowBaseTimes(); + } + + int next_concurrency_index = 0; + List gets_to_prepare = new ArrayList(batch_size); + Set tsuids = new ByteSet(); + + final ByteMap>> all_tsuids_gets; + if (multiget_no_meta) { + // prepare the tagvs combinations and base time list + + final List tagv_compinations = prepareAllTagvCompounds(); + all_tsuids_gets = prepareRequestsNoMeta(tagv_compinations, row_base_time_list); + } else { + all_tsuids_gets = prepareRequests(row_base_time_list, tags); + + } + // Iterate over all salts + for (final Entry>> salts_entry : all_tsuids_gets.entrySet()) { + if (gets_to_prepare.size() > 0) { // if we have any gets_to_prepare for previous salt, create a + // request out of it. + final MultiGetTask task = new MultiGetTask(tsuids, gets_to_prepare); + final List mulget_task_list = + multi_get_tasks.get((next_concurrency_index++) % concurrency_multi_get); + mulget_task_list.add(task); + ++multi_get_wait_cnt; + multi_get_num_get_requests = multi_get_num_get_requests + gets_to_prepare.size(); + gets_to_prepare = new ArrayList(batch_size); + tsuids = new ByteSet(); + } + byte[] curr_salt = salts_entry.getKey(); + // Iterate over all tsuid's in curr_salt and add them to gets_to_prepare + for (final Entry> gets_entry : salts_entry.getValue()) { + byte[] tsuid = gets_entry.getKey(); + + for (GetRequest request : gets_entry.getValue()) { + if (gets_to_prepare.size() >= batch_size) { // close batch and create a MultiGetTask + final MultiGetTask task = new MultiGetTask(tsuids, gets_to_prepare); + final List mulget_task_list = + multi_get_tasks.get((next_concurrency_index++) % concurrency_multi_get); + mulget_task_list.add(task); + ++multi_get_wait_cnt; + multi_get_num_get_requests = multi_get_num_get_requests + gets_to_prepare.size(); + if (LOG.isDebugEnabled()) { + LOG.debug("Finished preparing MultiGetRequest with " + gets_to_prepare.size() + " requests for salt " + curr_salt + + " for tsuid " + Bytes.pretty(tsuid)); + } + // prepare a new task list and tsuids + gets_to_prepare = new ArrayList(batch_size); + tsuids = new ByteSet(); + } + gets_to_prepare.add(request); + tsuids.add(gets_entry.getKey()); + } + tsuids.add(gets_entry.getKey()); + } + } + + + if (gets_to_prepare.size() > 0) { + + final MultiGetTask task = new MultiGetTask(tsuids, gets_to_prepare); + final List mulget_task_list = + multi_get_tasks.get((next_concurrency_index++) % concurrency_multi_get); + mulget_task_list.add(task); + ++multi_get_wait_cnt; + multi_get_num_get_requests = multi_get_num_get_requests + gets_to_prepare.size(); + LOG.debug("Finished preparing MultiGetRequest with " + gets_to_prepare.size()); + gets_to_prepare = new ArrayList(batch_size); + tsuids = new ByteSet(); + } + + prepare_multi_get_end_time = DateTime.currentTimeMillis(); + if (LOG.isDebugEnabled()) { + LOG.debug("Finished preparing concurrency multi get task with " + + multi_get_wait_cnt + " tasks using " + + (prepare_multi_get_end_time - prepare_multi_get_start_time) + "ms"); + } + + } + + private void validateMultigetData(List kvs, + Map> annotations, + List>> histograms) { + int tasks = finished_multi_get_cnt.incrementAndGet(); + + if (kvs.size() > 0) { + kvsmap.put(tasks, kvs); + } + + if (annotMap != null) { + for (byte[] key : annotations.keySet()) { + List notes = annotations.get(key); + + if (notes.size() > 0) { + annotMap.put(key, notes); + } + } + } + + if (histograms.size() > 0) { + histMap.put(tasks, histograms); + } + } + + private boolean checkAllFinishAndTriggerCallback() { + if (multi_get_wait_cnt == finished_multi_get_cnt.get()) { + try { + if (exception == null) { + mergeAndReturnResults(); + } + } catch (Exception ex) { + LOG.error("Failed merging and returning results, calling back with " + + "exception", ex); + + if (!isHistogramScan()) { + results.callback(ex); + } else { + histogramResults.callback(ex); + } + } + return true; + } else { + return false; + } + } + + private void mergeAndReturnResults() throws Exception { + final long hbase_time = DateTime.currentTimeMillis(); + TsdbQuery.scanlatency.add((int) (hbase_time - fetch_start_time)); + if (LOG.isDebugEnabled()) { + LOG.debug("Finished fetching data for metric: " + Bytes.pretty(metric) + + " using " + (hbase_time - fetch_start_time) + "ms"); + } + LOG.info("Finished fetching data for metric: " + Bytes.pretty(metric) + + " using " + (hbase_time - fetch_start_time) + "ms"); + if (exception != null) { + LOG.error("After all of the multi-gets finished, at " + + "least one threw an exception", exception); + throw exception; + } + + final long merge_start = DateTime.nanoTime(); + + // Merge sorted spans together + if (!isHistogramScan()) { + mergeDataPoints(); + } else { + // Merge histogram data points + mergeHistogramDataPoints(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("It took " + (DateTime.currentTimeMillis() - hbase_time) + " ms, " + + " to merge and sort the rows into a tree map"); + } + + if (query_stats != null) { + query_stats.addStat(query_index, QueryStat.SCANNER_MERGE_TIME, + (DateTime.nanoTime() - merge_start)); + } + + if (!isHistogramScan()) { + results.callback(spans); + } else { + histogramResults.callback(histogramSpans); + } + } + + private boolean isHistogramScan() { + return histogramSpans != null; + } + + private void mergeHistogramDataPoints() { + if (histogramSpans != null) { + for (List>> rows : histMap.values()) { + if (null == rows || rows.isEmpty()) { + LOG.error("Found a histogram rows list that was null or empty"); + continue; + } + + // for all the rows with the same salt in the timestamp order + for (final SimpleEntry> row : rows) { + if (null == row) { + LOG.error("Found a histogram row item that was null"); + continue; + } + + HistogramSpan histSpan = null; + try { + histSpan = histogramSpans.get(row.getKey()); + } catch (RuntimeException e) { + LOG.error("Failed to fetch the histogram span", e); + } + + if (histSpan == null) { + histSpan = new HistogramSpan(tsdb); + histogramSpans.put(row.getKey(), histSpan); + } + + if (annotMap.containsKey(row.getKey())) { + histSpan.getAnnotations().addAll(annotMap.get(row.getKey())); + annotMap.remove(row.getKey()); + } + + try { + histSpan.addRow(row.getKey(), row.getValue()); + } catch (RuntimeException e) { + LOG.error("Exception adding row to histogram span", e); + } + } // end for + } // end for + + histMap.clear(); + + for (byte[] key : annotMap.keySet()) { + HistogramSpan histSpan = histogramSpans.get(key); + + if (histSpan == null) { + histSpan = new HistogramSpan(tsdb); + histogramSpans.put(key, histSpan); + } + + histSpan.getAnnotations().addAll(annotMap.get(key)); + } + + annotMap.clear(); + } + } + + private void mergeDataPoints() { + for (List kvs : kvsmap.values()) { + if (kvs == null || kvs.isEmpty()) { + LOG.error("Found a key value list that was null or empty"); + continue; + } + for (final KeyValue kv : kvs) { + + if (kv == null) { + LOG.error("Found a key value item that was null"); + continue; + } + if (kv.key() == null) { + LOG.error("A key for a kv was null"); + continue; + } + + Span datapoints = null; + try { + datapoints = spans.get(kv.key()); + } catch (RuntimeException e) { + LOG.error("Failed to fetch the span", e); + } + + // If this tsdb follows append logic, then there will not be any + // duplicates here. But if it is not, then there can be multiple + // non-compcated or out of order rows here + if (datapoints == null) { + datapoints = RollupQuery.isValidQuery(rollup_query) + ? new RollupSpan(tsdb, rollup_query) + : new Span(tsdb); + spans.put(kv.key(), datapoints); + } + + if (annotMap.containsKey(kv.key())) { + for (Annotation note : annotMap.get(kv.key())) { + datapoints.getAnnotations().add(note); + } + annotMap.remove(kv.key()); + } + try { + datapoints.addRow(kv); + } catch (RuntimeException e) { + LOG.error("Exception adding row to span", e); + } + } + } + + kvsmap.clear(); + + for (byte[] key : annotMap.keySet()) { + Span datapoints = (Span) spans.get(key); + + if (datapoints == null) { + datapoints = new Span(tsdb); + spans.put(key, datapoints); + } + + for (Annotation note : annotMap.get(key)) { + datapoints.getAnnotations().add(note); + } + } + + annotMap.clear(); + } + boolean exception1; + /** + * If one or more of the scanners throws an exception then we should close it + * and pass the exception here so that we can catch and return it to the + * caller. If all of the scanners have finished, this will callback to the + * caller immediately. + * @param e The exception to store. + */ + private void handleException(final Exception e) { + // make sure only one scanner can set the exception + finished_multi_get_cnt.incrementAndGet(); + if (exception == null) { + synchronized (this) { + if (exception == null) { + exception = e; + // fail once and fast on the first scanner to throw an exception + try { + if (exception != null) { + mergeAndReturnResults(); + } + } catch (Exception ex) { + + LOG.error("Failed merging and returning results, " + + "calling back with exception", ex); + results.callback(ex); + + } + } else { + // TODO - it would be nice to close and cancel the other scanners but + // for now we have to wait for them to finish and/or throw exceptions. + LOG.error("Another scanner threw an exception", e); + } + } + } + } + + /** + * Generates a list of Unix Epoch Timestamps for the row key base times given + * the start and end times of the query. + * @return A non-null list of at least one base row. + */ + @VisibleForTesting + List prepareRowBaseTimes() { + final ArrayList row_base_time_list = new ArrayList( + (int) ((end_row_time - start_row_time) / Const.MAX_TIMESPAN)); + // NOTE: inclusive end here + long ts = (start_row_time - (start_row_time % Const.MAX_TIMESPAN)); + while (ts <= end_row_time) { + row_base_time_list.add(ts); + ts += Const.MAX_TIMESPAN; + } + return row_base_time_list; + } + + /** + * Generates a list of Unix Epoch Timestamps for the row key base times given + * the start and end times of the query and rollup interval given. + * @return A non-null list of at least one base row. + */ + @VisibleForTesting + List prepareRowBaseTimesRollup() { + final RollupInterval interval = rollup_query.getRollupInterval(); + + // standard TSDB table format, i.e. we're using the default table and schema + if (interval.getUnits() == 'h') { + return prepareRowBaseTimes(); + } else { + final List row_base_times = new ArrayList( + (int) ((end_row_time - start_row_time) / interval.getIntervals())); + + long ts = RollupUtils.getRollupBasetime(start_row_time, interval); + while (ts <= end_row_time) { + row_base_times.add(ts); + // TODO - possible this could overshoot in some cases. It shouldn't + // if the rollups are properly configured, but... you know. Check it. + ts = RollupUtils.getRollupBasetime(ts + + (interval.getIntervalSeconds() * interval.getIntervals()), interval); + } + return row_base_times; + } + } + + /** + * We have multiple tagks and each tagk may has multiple possible values. + * This routine generates all the possible tagv compounds basing on the + * present sequence of the tagks. Each compound will be used to generate + * the final tsuid. + * For example, if there are two tag keys where the first tag key has 4 values + * and the second tagk has 2 values then the resulting list will have 6 + * entries, one for each permutation. + * + * @return a non-null list that contains all the possible compounds + */ + @VisibleForTesting + List prepareAllTagvCompounds() { + List pre_phase_tags = new LinkedList(); + pre_phase_tags.add(new byte[tags.get(0).size()][TSDB.tagv_width()]); + + List next_phase_tags = new LinkedList(); + int next_append_index = 0; + + for (final Map.Entry tag : tags.get(0)) { + byte[][] tagv = tag.getValue(); + for (int i = 0; i < tagv.length; ++i) { + for (byte[][] pre_phase_tag : pre_phase_tags) { + final byte[][] next_phase_tag = + new byte[tags.get(0).size()][tsdb.tag_values.width()]; + + // copy the tagv from index 0 ~ next_append_index - 1 + for (int k = 0; k < next_append_index; ++k) { + System.arraycopy(pre_phase_tag[k], 0, next_phase_tag[k], 0, + tsdb.tag_values.width()); + } + + // copy the tagv in next_append_index + System.arraycopy(tagv[i], 0, next_phase_tag[next_append_index], 0, + tsdb.tag_values.width()); + next_phase_tags.add(next_phase_tag); + } + } // end for + + ++next_append_index; + pre_phase_tags = next_phase_tags; + next_phase_tags = new LinkedList(); + } // end for + return pre_phase_tags; + } + + /** + * Generates a map of TSUIDs to get requests given the tag permutations. + * If all salts is enabled, each TSUID will have Const.SALT_BUCKETS() number + * of entries. Otherwise each TSUID will have one row key. + * @param tagv_compounds The permutations of tag key and value combinations to + * search for. + * @param base_time_list The list of base timestamps. + * @return A non-null map of TSUIDs to lists of get requests to send to HBase. + */ + @VisibleForTesting + ByteMap>> prepareRequestsNoMeta(final List tagv_compounds, + final List base_time_list) { + + final int row_size = (Const.SALT_WIDTH() + tsdb.metrics.width() + + Const.TIMESTAMP_BYTES + + (tsdb.tag_names.width() * tags.get(0).size()) + + (tsdb.tag_values.width() * tags.get(0).size())); + + final ByteMap> tsuid_rows = new ByteMap>(); + for (final byte[][] tagvs : tagv_compounds) { + // TSUID's don't have salts + // TODO: we reallly don't have to allocate tsuid's here. + // we can use the row_key array to fetch the tsuid. + // This will just double the memory utilization per time series. + final byte[] tsuid = new byte[tsdb.metrics.width() + + (tags.get(0).size() * tsdb.tag_names.width()) + + tags.get(0).size() * tsdb.tag_values.width()]; + final byte[] row_key = new byte[row_size]; + + // metric + System.arraycopy(metric, 0, row_key, Const.SALT_WIDTH(), tsdb.metrics.width()); + System.arraycopy(metric, 0, tsuid, 0, tsdb.metrics.width()); + + final List rows = + new ArrayList(base_time_list.size()); + + // copy tagks and tagvs to the row key + int tag_index = 0; + int row_key_copy_offset = Const.SALT_WIDTH() + tsdb.metrics.width() + + Const.TIMESTAMP_BYTES; + int tsuid_copy_offset = tsdb.metrics.width(); + for (Map.Entry tag : tags.get(0)) { + // tagk + byte[] tagk = tag.getKey(); + System.arraycopy(tagk, 0, row_key, row_key_copy_offset, tsdb.tag_names.width()); + System.arraycopy(tagk, 0, tsuid, tsuid_copy_offset, tsdb.tag_names.width()); + row_key_copy_offset += tsdb.tag_names.width(); + tsuid_copy_offset += tsdb.tag_names.width(); + + // tagv + System.arraycopy(tagvs[tag_index], 0, row_key, row_key_copy_offset, + tsdb.tag_values.width()); + System.arraycopy(tagvs[tag_index], 0, tsuid, tsuid_copy_offset, + tsdb.tag_values.width()); + row_key_copy_offset += tsdb.tag_values.width(); + tsuid_copy_offset += tsdb.tag_values.width(); + + // move to the next tag + ++tag_index; + } + + // iterate for each timestamp, making a copy of the key and tweaking it's + // timestamp. + for (final long row_base_time : base_time_list) { + final byte[] key_copy = Arrays.copyOf(row_key, row_key.length); + + // base time + Internal.setBaseTime(key_copy, (int) row_base_time); + + if (get_all_salts) { + for (int i = 0; i < Const.SALT_BUCKETS(); i++) { + final byte[] copy = Arrays.copyOf(key_copy, key_copy.length); + // TODO - handle multi byte salts + copy[0] = (byte) i; + rows.add(new GetRequest(table_to_fetch, copy, TSDB.FAMILY)); + } + } else { + // salt + RowKey.prefixKeyWithSalt(key_copy); + rows.add(new GetRequest(table_to_fetch, key_copy, TSDB.FAMILY)); + } + } // end for + + tsuid_rows.put(tsuid, rows); + } // end for + ByteMap>> return_obj = new ByteMap>>(); + // byte[] salt = new byte[Const.SALT_WIDTH()]; + // System.arraycopy("1".getBytes(), 0, salt, 0, Const.SALT_WIDTH()); + return_obj.put("0".getBytes(), tsuid_rows); + return return_obj; + } + + /** + * Generates a map of TSUIDs per salt to get requests given the tag permutations. + * If all salts is enabled, each TSUID will have Const.SALT_BUCKETS() number + * of entries. Otherwise each TSUID will have one row key. + * @param tagv_compounds The cardinality of tag key and value combinations to + * search for. + * @param base_time_list The list of base timestamps. + * @return A non-null map of TSUIDs to lists of get requests to send to HBase. + */ + @VisibleForTesting + ByteMap>> prepareRequests(final List base_time_list, List> tags) { + + final ByteMap>> tsuid_rows = new ByteMap>>(); + // TSUID's don't have salts + // final byte[] tsuid = new byte[tsdb.metrics.width() + // + (tags.size() * tsdb.tag_names.width()) + // + tags.size() * tsdb.tag_values.width()]; + // final byte[] row_key = new byte[row_size]; + + + // copy tagks and tagvs to the row key + for (ByteMap each_row_key : tags) { + + final int row_size = (Const.SALT_WIDTH() + tsdb.metrics.width() + + Const.TIMESTAMP_BYTES + + (tsdb.tag_names.width() * each_row_key.size()) + + (tsdb.tag_values.width() * each_row_key.size())); + byte[] tsuid = new byte[tsdb.metrics.width() + + (each_row_key.size() * tsdb.tag_names.width()) + + each_row_key.size() * tsdb.tag_values.width()]; + byte[] row_key = new byte[row_size]; + int row_key_copy_offset = Const.SALT_WIDTH() + tsdb.metrics.width() + + Const.TIMESTAMP_BYTES; + int tsuid_copy_offset = tsdb.metrics.width(); + + // metric + System.arraycopy(metric, 0, row_key, Const.SALT_WIDTH(), tsdb.metrics.width()); + System.arraycopy(metric, 0, tsuid, 0, tsdb.metrics.width()); + + final List rows = + new ArrayList(base_time_list.size()); + for (Map.Entry tag_arr : each_row_key.entrySet()) { + byte[] tagv = tag_arr.getValue()[0]; + // tagk + byte[] tagk = tag_arr.getKey(); + + System.arraycopy(tagk, 0, row_key, row_key_copy_offset, tsdb.tag_names.width()); + System.arraycopy(tagk, 0, tsuid, tsuid_copy_offset, tsdb.tag_names.width()); + row_key_copy_offset += tsdb.tag_names.width(); + tsuid_copy_offset += tsdb.tag_names.width(); + + // tagv + System.arraycopy(tagv, 0, row_key, row_key_copy_offset, + tsdb.tag_values.width()); + System.arraycopy(tagv, 0, tsuid, tsuid_copy_offset, + tsdb.tag_values.width()); + row_key_copy_offset += tsdb.tag_values.width(); + tsuid_copy_offset += tsdb.tag_values.width(); + + } + + // iterate for each timestamp, making a copy of the key and tweaking it's + // timestamp. + for (final long row_base_time : base_time_list) { + final byte[] key_copy = Arrays.copyOf(row_key, row_key.length); + + // base time + Internal.setBaseTime(key_copy, (int) row_base_time); + + if (get_all_salts) { + for (int i = 0; i < Const.SALT_BUCKETS(); i++) { + byte[] salt = RowKey.getSaltBytes(i); + final byte[] copy = Arrays.copyOf(key_copy, key_copy.length); + System.arraycopy(salt, 0, copy, 0, Const.SALT_WIDTH()); + rows.add(new GetRequest(table_to_fetch, copy, TSDB.FAMILY)); + } + } else { + // salt + RowKey.prefixKeyWithSalt(key_copy); + rows.add(new GetRequest(table_to_fetch, key_copy, TSDB.FAMILY)); + } + } // end for + for (GetRequest request : rows) { + byte[] salt = new byte[Const.SALT_WIDTH()]; + System.arraycopy(request.key(), 0, salt, 0, Const.SALT_WIDTH()); + if (tsuid_rows.containsKey(salt)) { + ByteMap> map = tsuid_rows.get(salt); + if (map.containsKey(tsuid)) { + List list = map.get(tsuid); + list.add(request); + } else { + List list = new ArrayList(); + list.add(request); + map.put(tsuid, list); + } + } else { + ByteMap> map = new ByteMap>(); + List list = new ArrayList(); + list.add(request); + map.put(tsuid, list); + tsuid_rows.put(salt, map); + } + } + } + + return tsuid_rows; + } + + @VisibleForTesting + List> getMultiGetTasks() { + return multi_get_tasks; + } +} diff --git a/src/core/MutableDataPoint.java b/src/core/MutableDataPoint.java index 2f51a4e3f9..00b6cb0147 100644 --- a/src/core/MutableDataPoint.java +++ b/src/core/MutableDataPoint.java @@ -92,6 +92,16 @@ public static MutableDataPoint ofLongValue(final long timestamp, return dp; } + /** + * Copy constructor + * + * @param value A datapoint value. + */ + public static MutableDataPoint fromPoint(final DataPoint value) { + if (value.isInteger()) return ofLongValue(value.timestamp(), value.longValue()); + else return ofDoubleValue(value.timestamp(), value.doubleValue()); + } + @Override public long timestamp() { return timestamp; @@ -132,4 +142,9 @@ public String toString() { is_integer + ", value=" + (is_integer ? value : Double.longBitsToDouble(value)) + ")"; } + + @Override + public long valueCount() { + return 1; + } } diff --git a/src/core/Query.java b/src/core/Query.java index 01e08b969c..5cec6c7c80 100644 --- a/src/core/Query.java +++ b/src/core/Query.java @@ -40,8 +40,8 @@ public interface Query { /** * Returns the start time of the graph. * @return A strictly positive integer. - * @throws IllegalStateException if {@link #setStartTime} was never called on - * this instance before. + * @throws IllegalStateException if {@link #setStartTime(long)} was never + * called on this instance before. */ long getStartTime(); @@ -66,7 +66,21 @@ public interface Query { * @return A strictly positive integer. */ long getEndTime(); + + /** + * Sets whether or not the data queried will be deleted. + * @param delete True if data should be deleted, false otherwise. + * @since 2.2 + */ + void setDelete(boolean delete); + /** + * Returns whether or not the data queried will be deleted. + * @return A boolean + * @since 2.2 + */ + boolean getDelete(); + /** * Sets the time series to the query. * @param metric The metric to retrieve from the TSDB. @@ -105,7 +119,7 @@ void setTimeSeries(String metric, Map tags, * to run asynchronously and use different scanners, we can allow different * TSUIDs. * Note: This method will not check to determine if the TSUIDs are - * valid, since that wastes time and we *assume* that the user provides TUSIDs + * valid, since that wastes time and we *assume* that the user provides TSUIDs * that are up to date. * @param tsuids A list of one or more TSUIDs to scan for * @param function The aggregation function to use on results @@ -125,7 +139,7 @@ public void setTimeSeries(final List tsuids, * to run asynchronously and use different scanners, we can allow different * TSUIDs. * Note: This method will not check to determine if the TSUIDs are - * valid, since that wastes time and we *assume* that the user provides TUSIDs + * valid, since that wastes time and we *assume* that the user provides TSUIDs * that are up to date. * @param tsuids A list of one or more TSUIDs to scan for * @param function The aggregation function to use on results @@ -140,6 +154,40 @@ public void setTimeSeries(final List tsuids, final Aggregator function, final boolean rate, final RateOptions rate_options); + /** + * Prepares a query against HBase by setting up group bys and resolving + * strings to UIDs asynchronously. This replaces calls to all of the setters + * like the {@link setTimeSeries}, {@link setStartTime}, etc. + * Make sure to wait on the deferred return before calling {@link runAsync}. + * @param query The main query to fetch the start and end time from + * @param index The index of which sub query we're executing + * @return A deferred to wait on for UID resolution. The result doesn't have + * any meaning and can be discarded. + * @throws IllegalArgumentException if the query was missing sub queries or + * the index was out of bounds. + * @throws NoSuchUniqueName if the name of a metric, or a tag name/value + * does not exist. (Bubbles up through the deferred) + * @since 2.2 + */ + public Deferred configureFromQuery(final TSQuery query, + final int index); + + /** + * Prepares a query against HBase by setting up group bys and resolving + * strings to UIDs asynchronously. This replaces calls to all of the setters + * like the {@link setTimeSeries}, {@link setStartTime}, etc. + * Make sure to wait on the deferred return before calling {@link runAsync}. + * @param query The main query to fetch the start and end time from + * @param index The index of which sub query we're executing + * @param force_raw If true, always get the data from the raw table; disables rollups + * @throws IllegalArgumentException if the query was missing sub queries or + * the index was out of bounds. + * @throws NoSuchUniqueName if the name of a metric, or a tag name/value + * does not exist. (Bubbles up through the deferred) + * @since 2.4 + */ + Deferred configureFromQuery(final TSQuery query, final int index, boolean force_raw); + /** * Downsamples the results by specifying a fixed interval between points. *

    @@ -149,12 +197,24 @@ public void setTimeSeries(final List tsuids, * way we get this one data point is by aggregating all the data points of * that interval together using an {@link Aggregator}. This enables you * to compute things like the 5-minute average or 10 minute 99th percentile. - * @param interval Number of seconds wanted between each data point. + * @param interval Number of milliseconds wanted between each data point. * @param downsampler Aggregation function to use to group data points * within an interval. */ void downsample(long interval, Aggregator downsampler); + /** + * Sets an optional downsampling function on this query + * @param interval The interval, in milliseconds to rollup data points + * @param downsampler An aggregation function to use when rolling up data points + * @param fill_policy Policy specifying whether to interpolate or to fill + * missing intervals with special values. + * @throws NullPointerException if the aggregation function is null + * @throws IllegalArgumentException if the interval is not greater than 0 + * @since 2.2 + */ + void downsample(long interval, Aggregator downsampler, FillPolicy fill_policy); + /** * Runs this query. * @return The data points matched by this query. @@ -167,6 +227,19 @@ public void setTimeSeries(final List tsuids, */ DataPoints[] run() throws HBaseException; + /** + * Runs this query. + * @return The data points matched by this query and applied with percentile calculation + *

    + * Each element in the non-{@code null} but possibly empty array returned + * corresponds to one time series for which some data points have been + * matched by the query. + * @throws HBaseException if there was a problem communicating with HBase to + * perform the search. + * @throws IllegalStateException if the query is not a histogram query + */ + DataPoints[] runHistogram() throws HBaseException; + /** * Executes the query asynchronously * @return The data points matched by this query. @@ -179,4 +252,50 @@ public void setTimeSeries(final List tsuids, * @since 1.2 */ public Deferred runAsync() throws HBaseException; + + /** + * Runs this query asynchronously. + * @return The data points matched by this query and applied with percentile calculation + *

    + * Each element in the non-{@code null} but possibly empty array returned + * corresponds to one time series for which some data points have been + * matched by the query. + * @throws HBaseException if there was a problem communicating with HBase to + * perform the search. + * @throws IllegalStateException if the query is not a histogram query + */ + Deferred runHistogramAsync() throws HBaseException; + + /** + * Returns an index for this sub-query in the original set of queries. + * @return A zero based index. + * @since 2.4 + */ + public int getQueryIdx(); + + /** + * Check this is a histogram query or not + * @return + */ + public boolean isHistogramQuery(); + + /** + * @return Whether or not this is a rollup query + * @since 2.4 + */ + public boolean isRollupQuery(); + + /** + * @return whether this query needs to be split. + * @since 2.4 + */ + public boolean needsSplitting(); + + /** + * Set the percentile calculation parameters for this query if this is + * a histogram query + * + * @param percentiles + */ + public void setPercentiles(List percentiles); } diff --git a/src/core/QueryException.java b/src/core/QueryException.java new file mode 100644 index 0000000000..5427cc1be2 --- /dev/null +++ b/src/core/QueryException.java @@ -0,0 +1,75 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import org.jboss.netty.handler.codec.http.HttpResponseStatus; + +/** + * An exception thrown during query execution such as a timeout or other + * type of error. + * @since 2.2 + */ +public final class QueryException extends RuntimeException { + + /** An optional, detailed error message */ + private final String details; + + /** The HTTP status code to return to the user */ + private final HttpResponseStatus status; + + /** + * Default ctor + * @param msg Message describing the problem. + */ + public QueryException(final String msg) { + super(msg); + status = HttpResponseStatus.BAD_REQUEST; + details = msg; + } + + /** + * Ctor setting the status + * @param status The status code to respond with for HTTP requests + * @param msg Message describing the problem. + */ + public QueryException(final HttpResponseStatus status, final String msg) { + super(msg); + this.status = status; + details = msg; + } + + /** + * Ctor setting status and the messages + * @param status The status code to respond with for HTTP requests + * @param msg Message describing the problem. + * @param details Extra details for the error + */ + public QueryException(final HttpResponseStatus status, final String msg, + final String details) { + super(msg); + this.status = status; + this.details = details; + } + + /** @return the HTTP status code */ + public final HttpResponseStatus getStatus() { + return this.status; + } + + /** @return the details, may be an empty string */ + public final String getDetails() { + return this.details; + } + + private static final long serialVersionUID = 9040020770546069974L; +} diff --git a/src/core/RateOptions.java b/src/core/RateOptions.java index abf1b0f1ee..aa1370d37c 100644 --- a/src/core/RateOptions.java +++ b/src/core/RateOptions.java @@ -12,12 +12,14 @@ // see . package net.opentsdb.core; +import com.google.common.base.Objects; + /** * Provides additional options that will be used when calculating rates. These * options are useful when working with metrics that are raw counter values, * where a counter is defined by a value that always increases until it hits * a maximum value and then it "rolls over" to start back at 0. - *

    + *

    * These options will only be utilized if the query is for a rate calculation * and if the "counter" options is set to true. * @since 2.0 @@ -31,6 +33,9 @@ public class RateOptions { * some maximum */ private boolean counter; + + /** Whether or not to simply drop rolled-over or reset data points */ + private boolean drop_resets; /** * If calculating a rate of change over a metric that is a counter, then this @@ -53,6 +58,7 @@ public RateOptions() { this.counter = false; this.counter_max = Long.MAX_VALUE; this.reset_value = DEFAULT_RESET_VALUE; + this.drop_resets = false; } /** @@ -67,9 +73,50 @@ public RateOptions() { */ public RateOptions(final boolean counter, final long counter_max, final long reset_value) { + this(counter, counter_max, reset_value, false); + } + + /** + * Ctor + * @param counter If true, indicates that the rate calculation should assume + * that the underlying data is from a counter + * @param counter_max Specifies the maximum value for the counter before it + * will roll over and restart at 0 + * @param reset_value Specifies the largest rate change that is considered + * acceptable, if a rate change is seen larger than this value then the + * counter is assumed to have been reset + * @param drop_resets Whether or not to drop rolled-over or reset counters + * @since 2.2 + */ + public RateOptions(final boolean counter, final long counter_max, + final long reset_value, final boolean drop_resets) { this.counter = counter; this.counter_max = counter_max; this.reset_value = reset_value; + this.drop_resets = drop_resets; + } + + @Override + public int hashCode() { + return Objects.hashCode(counter, counter_max, reset_value, drop_resets); + } + + @Override + public boolean equals(final Object obj) { + if (obj == null) { + return false; + } + if (!(obj instanceof RateOptions)) { + return false; + } + if (obj == this) { + return true; + } + final RateOptions options = (RateOptions)obj; + return Objects.equal(counter, options.counter) + && Objects.equal(counter_max, options.counter_max) + && Objects.equal(reset_value, options.reset_value) + && Objects.equal(drop_resets, options.drop_resets); } /** @return Whether or not the counter flag is set */ @@ -87,6 +134,11 @@ public long getResetValue() { return reset_value; } + /** @return Whether or not to drop rolled-over or reset counters */ + public boolean getDropResets() { + return drop_resets; + } + /** @param counter Whether or not the time series should be considered counters */ public void setIsCounter(boolean counter) { this.counter = counter; @@ -102,6 +154,11 @@ public void setResetValue(long reset_value) { this.reset_value = reset_value; } + /** @param drop_resets Whether or not to drop rolled-over or reset counters */ + public void setDropResets(boolean drop_resets) { + this.drop_resets = drop_resets; + } + /** * Generates a String version of the rate option instance in a format that * can be utilized in a query. diff --git a/src/core/RateSpan.java b/src/core/RateSpan.java index ed3540e6da..c9fc43f350 100644 --- a/src/core/RateSpan.java +++ b/src/core/RateSpan.java @@ -147,6 +147,11 @@ private void populateNextRate() { } if (options.isCounter() && difference < 0) { + if (options.getDropResets()) { + populateNextRate(); + return; + } + if (prev_data.isInteger() && next_data.isInteger()) { // NOTE: Calculates in the long type to avoid precision loss // while converting long values to double values if both values are long. diff --git a/src/core/RequestBuilder.java b/src/core/RequestBuilder.java new file mode 100644 index 0000000000..e8aa47e504 --- /dev/null +++ b/src/core/RequestBuilder.java @@ -0,0 +1,34 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import org.hbase.async.AppendRequest; +import org.hbase.async.HBaseRpc; +import org.hbase.async.PutRequest; + +import net.opentsdb.utils.Config; + +public class RequestBuilder { + + public static PutRequest buildPutRequest(Config config, byte[] tableName, byte[] row, byte[] family, byte[] qualifier, byte[] value, long timestamp) { + + if(config.use_otsdb_timestamp()) + if((timestamp & Const.SECOND_MASK) != 0) + return new PutRequest(tableName, row, family, qualifier, value, timestamp); + else + return new PutRequest(tableName, row, family, qualifier, value, timestamp * 1000); + else + return new PutRequest(tableName, row, family, qualifier, value); + } + +} diff --git a/src/core/RowKey.java b/src/core/RowKey.java index ee733daa23..2a8a5f7064 100644 --- a/src/core/RowKey.java +++ b/src/core/RowKey.java @@ -13,6 +13,9 @@ package net.opentsdb.core; import java.util.Arrays; +import java.util.Comparator; + +import net.opentsdb.uid.NoSuchUniqueId; import org.hbase.async.Bytes; @@ -30,6 +33,7 @@ private RowKey() { * @param tsdb The TSDB to use. * @param row The actual row key. * @return The name of the metric. + * @throws NoSuchUniqueId if the UID could not resolve to a string */ static String metricName(final TSDB tsdb, final byte[] row) { try { @@ -45,26 +49,42 @@ static String metricName(final TSDB tsdb, final byte[] row) { * Extracts the name of the metric ID contained in a row key. * @param tsdb The TSDB to use. * @param row The actual row key. - * @return The name of the metric. + * @return A deferred to wait on that will return the name of the metric. + * @throws IllegalArgumentException if the row key is too short due to missing + * salt or metric or if it's null/empty. + * @throws NoSuchUniqueId if the UID could not resolve to a string * @since 1.2 */ public static Deferred metricNameAsync(final TSDB tsdb, final byte[] row) { - final byte[] id = Arrays.copyOfRange(row, 0, tsdb.metrics.width()); + if (row == null || row.length < 1) { + throw new IllegalArgumentException("Row key cannot be null or empty"); + } + if (row.length < Const.SALT_WIDTH() + tsdb.metrics.width()) { + throw new IllegalArgumentException("Row key is too short"); + } + final byte[] id = Arrays.copyOfRange( + row, Const.SALT_WIDTH(), tsdb.metrics.width() + Const.SALT_WIDTH()); return tsdb.metrics.getNameAsync(id); } /** * Generates a row key given a TSUID and an absolute timestamp. The timestamp - * will be normalized to an hourly base time. + * will be normalized to an hourly base time. If salting is enabled then + * empty salt bytes will be prepended to the key and must be filled in later. * @param tsdb The TSDB to use for fetching tag widths * @param tsuid The TSUID to use for the key * @param timestamp An absolute time from which we generate the row base time * @return A row key for use in fetching data from OpenTSDB + * @throws IllegalArgumentException if the TSUID is too short, i.e. doesn't + * contain a metric * @since 2.0 */ public static byte[] rowKeyFromTSUID(final TSDB tsdb, final byte[] tsuid, final long timestamp) { + if (tsuid.length < tsdb.metrics.width()) { + throw new IllegalArgumentException("TSUID appears to be missing the metric"); + } final long base_time; if ((timestamp & Const.SECOND_MASK) != 0) { // drop the ms timestamp to seconds to calculate the base timestamp @@ -73,12 +93,116 @@ public static byte[] rowKeyFromTSUID(final TSDB tsdb, final byte[] tsuid, } else { base_time = (timestamp - (timestamp % Const.MAX_TIMESPAN)); } - final byte[] row = new byte[tsuid.length + Const.TIMESTAMP_BYTES]; - System.arraycopy(tsuid, 0, row, 0, TSDB.metrics_width()); - Bytes.setInt(row, (int) base_time, TSDB.metrics_width()); - System.arraycopy(tsuid, TSDB.metrics_width(), row, - TSDB.metrics_width() + Const.TIMESTAMP_BYTES, - tsuid.length - TSDB.metrics_width()); + final byte[] row = + new byte[Const.SALT_WIDTH() + tsuid.length + Const.TIMESTAMP_BYTES]; + System.arraycopy(tsuid, 0, row, Const.SALT_WIDTH(), tsdb.metrics.width()); + Bytes.setInt(row, (int) base_time, Const.SALT_WIDTH() + tsdb.metrics.width()); + System.arraycopy(tsuid, tsdb.metrics.width(), row, + Const.SALT_WIDTH() + tsdb.metrics.width() + Const.TIMESTAMP_BYTES, + tsuid.length - tsdb.metrics.width()); + RowKey.prefixKeyWithSalt(row); return row; } + + /** + * Returns the byte array for the given salt id + * WARNING: Don't use this one unless you know what you're doing. It's here + * for unit testing. + * @param bucket The ID of the bucket to get the salt for + * @return The salt as a byte array based on the width in bytes + * @since 2.2 + */ + public static byte[] getSaltBytes(final int bucket) { + final byte[] bytes = new byte[Const.SALT_WIDTH()]; + int shift = 0; + for (int i = 1;i <= Const.SALT_WIDTH(); i++) { + bytes[Const.SALT_WIDTH() - i] = (byte) (bucket >>> shift); + shift += 8; + } + return bytes; + } + + /** + * Calculates and writes an array of one or more salt bytes at the front of + * the given row key. + * + * The salt is calculated by taking the Java hash code of the metric and + * tag UIDs and returning a modulo based on the number of salt buckets. + * The result will always be a positive integer from 0 to salt buckets. + * + * NOTE: The row key passed in MUST have allocated the {@code width} number of + * bytes at the front of the row key or this call will overwrite data. + * + * WARNING: If the width is set to a positive value, then the bucket must be + * at least 1 or greater. + * @param row_key The pre-allocated row key to write the salt to + * @since 2.2 + */ + public static void prefixKeyWithSalt(final byte[] row_key) { + if (Const.SALT_WIDTH() > 0) { + if (row_key.length < (Const.SALT_WIDTH() + TSDB.metrics_width()) || + (Bytes.memcmp(row_key, new byte[Const.SALT_WIDTH() + TSDB.metrics_width()], + Const.SALT_WIDTH(), TSDB.metrics_width()) == 0)) { + // ^ Don't salt the global annotation row, leave it at zero + return; + } + final int tags_start = Const.SALT_WIDTH() + TSDB.metrics_width() + + Const.TIMESTAMP_BYTES; + + // we want the metric and tags, not the timestamp + final byte[] salt_base = + new byte[row_key.length - Const.SALT_WIDTH() - Const.TIMESTAMP_BYTES]; + System.arraycopy(row_key, Const.SALT_WIDTH(), salt_base, 0, TSDB.metrics_width()); + System.arraycopy(row_key, tags_start,salt_base, TSDB.metrics_width(), + row_key.length - tags_start); + int modulo = Arrays.hashCode(salt_base) % Const.SALT_BUCKETS(); + if (modulo < 0) { + // make sure we return a positive salt. + modulo = modulo * -1; + } + + final byte[] salt = getSaltBytes(modulo); + System.arraycopy(salt, 0, row_key, 0, Const.SALT_WIDTH()); + } // else salting is disabled so it's a no-op + } + + /** + * Checks a row key to determine if it contains the metric UID. If salting is + * enabled, we skip the salt bytes. + * @param metric The metric UID to match + * @param row_key The row key to match on + * @return 0 if the two arrays are identical, otherwise the difference + * between the first two different bytes (treated as unsigned), otherwise + * the different between their lengths. + * @throws IndexOutOfBoundsException if either array isn't large enough. + */ + public static int rowKeyContainsMetric(final byte[] metric, + final byte[] row_key) { + int idx = Const.SALT_WIDTH(); + for (int i = 0; i < metric.length; i++, idx++) { + if (metric[i] != row_key[idx]) { + return (metric[i] & 0xFF) - (row_key[idx] & 0xFF); // "promote" to unsigned. + } + } + return 0; + } + + /** + * A comparator that ignores the salt in row keys + */ + public static class SaltCmp implements Comparator { + public int compare(final byte[] a, final byte[] b) { + final int length = Math.min(a.length, b.length); + if (a == b) { // Do this after accessing a.length and b.length + return 0; // in order to NPE if either a or b is null. + } + // Skip salt + for (int i = Const.SALT_WIDTH(); i < length; i++) { + if (a[i] != b[i]) { + return (a[i] & 0xFF) - (b[i] & 0xFF); // "promote" to unsigned. + } + } + return a.length - b.length; + } + } } diff --git a/src/core/RowSeq.java b/src/core/RowSeq.java index 8238f872e1..e614ff40b9 100644 --- a/src/core/RowSeq.java +++ b/src/core/RowSeq.java @@ -24,6 +24,7 @@ import org.hbase.async.Bytes; import org.hbase.async.KeyValue; +import org.hbase.async.Bytes.ByteMap; import com.stumbleupon.async.Deferred; @@ -35,7 +36,7 @@ * are stored in two byte arrays: one for the time offsets/flags and another * for the values. Access is granted via pointers. */ -final class RowSeq implements DataPoints { +public final class RowSeq implements iRowSeq { /** The {@link TSDB} instance we belong to. */ private final TSDB tsdb; @@ -63,13 +64,9 @@ final class RowSeq implements DataPoints { RowSeq(final TSDB tsdb) { this.tsdb = tsdb; } - - /** - * Sets the row this instance holds in RAM using a row from a scanner. - * @param row The compacted HBase row to set. - * @throws IllegalStateException if this method was already called. - */ - void setRow(final KeyValue row) { + + @Override + public void setRow(final KeyValue row) { if (this.key != null) { throw new IllegalStateException("setRow was already called on " + this); } @@ -83,19 +80,22 @@ void setRow(final KeyValue row) { * Merges data points for the same HBase row into the local object. * When executing multiple async queries simultaneously, they may call into * this method with data sets that are out of order. This may ONLY be called - * after setRow() has initiated the rowseq. + * after setRow() has initiated the rowseq. It also allows for rows with + * different salt bucket IDs to be merged into the same sequence. * @param row The compacted HBase row to merge into this instance. * @throws IllegalStateException if {@link #setRow} wasn't called first. * @throws IllegalArgumentException if the data points in the argument * do not belong to the same row as this RowSeq */ - void addRow(final KeyValue row) { + @Override + public void addRow(final KeyValue row) { if (this.key == null) { throw new IllegalStateException("setRow was never called on " + this); } final byte[] key = row.key(); - if (!Bytes.equals(this.key, key)) { + if (Bytes.memcmp(this.key, key, Const.SALT_WIDTH(), + key.length - Const.SALT_WIDTH()) != 0) { throw new IllegalDataException("Attempt to add a different row=" + row + ", this=" + this); } @@ -282,6 +282,12 @@ public Deferred metricNameAsync() { return RowKey.metricNameAsync(tsdb, key); } + @Override + public byte[] metricUID() { + return Arrays.copyOfRange(key, Const.SALT_WIDTH(), + Const.SALT_WIDTH() + TSDB.metrics_width()); + } + public Map getTags() { try { return getTagsAsync().joinUninterruptibly(); @@ -292,6 +298,11 @@ public Map getTags() { } } + @Override + public ByteMap getTagUids() { + return Tags.getTagUids(key); + } + public Deferred> getTagsAsync() { return Tags.getTagsAsync(tsdb, key); } @@ -306,6 +317,11 @@ public Deferred> getAggregatedTagsAsync() { return Deferred.fromResult(empty); } + @Override + public List getAggregatedTagUids() { + return Collections.emptyList(); + } + public List getTSUIDs() { return Collections.emptyList(); } @@ -347,16 +363,21 @@ public int aggregatedSize() { public SeekableView iterator() { return internalIterator(); } - - /** Package private iterator method to access it as a {@link Iterator}. */ - Iterator internalIterator() { + + @Override + public Iterator internalIterator() { // XXX this is now grossly inefficient, need to walk the arrays once. return new Iterator(); } - /** Extracts the base timestamp from the row key. */ - long baseTime() { - return Bytes.getUnsignedInt(key, tsdb.metrics.width()); + @Override + public long baseTime() { + return Bytes.getUnsignedInt(key, Const.SALT_WIDTH() + tsdb.metrics.width()); + } + + @Override + public byte[] key() { + return key; } /** @throws IndexOutOfBoundsException if {@code i} is out of bounds. */ @@ -493,8 +514,8 @@ public String toString() { * on the {@code RowSeq#baseTime()} * @since 2.0 */ - public static final class RowSeqComparator implements Comparator { - public int compare(final RowSeq a, final RowSeq b) { + public static final class RowSeqComparator implements Comparator { + public int compare(final iRowSeq a, final iRowSeq b) { if (a.baseTime() == b.baseTime()) { return 0; } @@ -503,7 +524,7 @@ public int compare(final RowSeq a, final RowSeq b) { } /** Iterator for {@link RowSeq}s. */ - final class Iterator implements SeekableView, DataPoint { + final class Iterator implements iRowSeq.Iterator { /** Current qualifier. */ private int qualifier; @@ -656,5 +677,23 @@ public String toString() { return toStringSummary() + ", seq=" + RowSeq.this + ')'; } + @Override + public long valueCount() { + return 1; + } + + } + + public int getQueryIndex() { + throw new UnsupportedOperationException("Not mapped to a query"); + } + @Override + public boolean isPercentile() { + return false; + } + + @Override + public float getPercentile() { + throw new UnsupportedOperationException("getPercentile not supported"); } } diff --git a/src/core/RpcResponder.java b/src/core/RpcResponder.java new file mode 100644 index 0000000000..97e7b22bb8 --- /dev/null +++ b/src/core/RpcResponder.java @@ -0,0 +1,110 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2010-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import net.opentsdb.utils.Config; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + * This class is responsible for building result of requests and + * respond to clients asynchronously. + * + * It can reduce requests that stacking in AsyncHBase, especially put requests. + * When a HBase's RPC has completed, the "AsyncHBase I/O worker" just decodes + * the response, and then do callback by this class asynchronously. We should + * take up workers as short as possible time so that workers can remove RPCs + * from in-flight state more quickly. + * + */ +public class RpcResponder { + + private static final Logger LOG = LoggerFactory.getLogger(RpcResponder.class); + + public static final String TSD_RESPONSE_ASYNC_KEY = "tsd.core.response.async"; + public static final boolean TSD_RESPONSE_ASYNC_DEFAULT = true; + + public static final String TSD_RESPONSE_WORKER_NUM_KEY = + "tsd.core.response.worker.num"; + public static final int TSD_RESPONSE_WORKER_NUM_DEFAULT = 10; + + private final boolean async; + private ExecutorService responders; + private volatile boolean running = true; + + RpcResponder(final Config config) { + async = config.getBoolean(TSD_RESPONSE_ASYNC_KEY, + TSD_RESPONSE_ASYNC_DEFAULT); + + if (async) { + int threads = config.getInt(TSD_RESPONSE_WORKER_NUM_KEY, + TSD_RESPONSE_WORKER_NUM_DEFAULT); + responders = Executors.newFixedThreadPool(threads, + new ThreadFactoryBuilder() + .setNameFormat("OpenTSDB Responder #%d") + .setDaemon(true) + .setUncaughtExceptionHandler(new ExceptionHandler()) + .build()); + } + + LOG.info("RpcResponder mode: {}", async ? "async" : "sync"); + } + + public void response(Runnable run) { + if (async) { + if (running) { + responders.execute(run); + } else { + throw new IllegalStateException("RpcResponder is closing or closed."); + } + } else { + run.run(); + } + } + + public void close() { + if (running) { + running = false; + responders.shutdown(); + } + + boolean completed; + try { + completed = responders.awaitTermination(5, TimeUnit.MINUTES); + } catch (InterruptedException e) { + completed = false; + } + + if (!completed) { + LOG.warn( + "There are still some results that are not returned to the clients."); + } + } + + public boolean isAsync() { + return async; + } + + private class ExceptionHandler implements Thread.UncaughtExceptionHandler { + @Override + public void uncaughtException(Thread t, Throwable e) { + LOG.error("Run into an uncaught exception in thread: " + t.getName(), e); + } + } +} diff --git a/src/core/SaltScanner.java b/src/core/SaltScanner.java new file mode 100644 index 0000000000..76ec83bb01 --- /dev/null +++ b/src/core/SaltScanner.java @@ -0,0 +1,1002 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.AbstractMap.SimpleEntry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import net.opentsdb.meta.Annotation; +import net.opentsdb.query.filter.TagVFilter; +import net.opentsdb.rollup.RollupQuery; +import net.opentsdb.rollup.RollupSpan; +import net.opentsdb.stats.QueryStats; +import net.opentsdb.stats.QueryStats.QueryStat; +import net.opentsdb.uid.UniqueId; +import net.opentsdb.utils.DateTime; +import net.opentsdb.utils.JSON; + +import org.hbase.async.Bytes.ByteMap; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.hbase.async.Bytes; +import org.hbase.async.DeleteRequest; +import org.hbase.async.KeyValue; +import org.hbase.async.Scanner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; +import com.stumbleupon.async.Callback; +import com.stumbleupon.async.Deferred; + +/** + * A class that handles coordinating the various scanners created for each + * salt bucket when salting is enabled. Each scanner stores it's results in + * local maps and once everyone has reported in, then the maps are parsed and + * combined into a proper set of spans to return to the {@link TsdbQuery} class. + * + * Note that if one or more of the scanners throws an exception, then that + * exception will be returned to the caller in the deferred. Unfortunately we + * don't have a good way to cancel a scan in progress so the first scanner with + * an error will store it, then we wait for all of the other scanners to + * complete. + * + * Concurrency is important in this class as the scanners are executing + * asynchronously and can modify variables at any time. + * + * @since 2.2 + */ +public class SaltScanner { + private static final Logger LOG = LoggerFactory.getLogger(SaltScanner.class); + + /** This is a map that the caller must supply. We'll fill it with data. + * WARNING: The salted row comparator should be applied to this map. */ + private final SortedMap spans; + + private final SortedMap histSpans; + + /** The list of pre-configured scanners. One scanner should be created per + * salt bucket. */ + private final List scanners; + + /** Stores the compacted columns from each scanner as it completes. After all + * scanners are done, we process this into the span map above. */ + private final Map> kv_map = + new ConcurrentHashMap>(); + + /** Stores annotations from each scanner as it completes */ + private final Map> annotation_map = + Collections.synchronizedMap( + new TreeMap>(new RowKey.SaltCmp())); + + private final Map>>> + histMap = new ConcurrentHashMap>>>(); + + /** A deferred to call with the spans on completion */ + private final Deferred> results = + new Deferred>(); + + private final Deferred> histogramResults = + new Deferred>(); + + /** The metric this scanner set is dealing with. If a row comes in with a + * different metric we toss an exception. This shouldn't happen though. */ + private final byte[] metric; + + /** The TSDB to which we belong */ + private final TSDB tsdb; + + /** A stats object associated with the sub query used for storing stats + * about scanner operations. */ + private final QueryStats query_stats; + + /** Index of the sub query in the main query list */ + private final int query_index; + private final boolean is_rollup; + private final int rollup_agg_id; + private final int rollup_count_id; + + /** Settings and counters to determine when we need to cancel a query. */ + private final AtomicLong num_data_points; + private final AtomicBoolean max_data_points_flag; + private AtomicLong bytes_fetched = new AtomicLong(); + private final long max_data_points; + private final long max_bytes; + + /** A latch used to determine how many scanners are still running */ + private final AtomicInteger countdown; + + /** When the scanning started. We store the scan latency once all scanners + * are done.*/ + private long start_time; // milliseconds. + + /** Whether or not to delete the queried data */ + private final boolean delete; + + /** A rollup query configuration if scanning for rolled up data. */ + private final RollupQuery rollup_query; + + /** A list of filters to iterate over when processing rows */ + private final List filters; + + /** A holder for storing the first exception thrown by a scanner if something + * goes pear shaped. Make sure to synchronize on this object when checking + * for null or assigning from a scanner's callback. */ + private volatile Exception exception; + + /** + * Default ctor that performs some validation. Call {@link scan} after + * construction to actually start fetching data. + * @param tsdb The TSDB to which we belong + * @param metric The metric we're expecting to fetch + * @param scanners A list of HBase scanners, one for each bucket + * @param spans The span map to store results in + * @param filters A list of filters for processing + * @throws IllegalArgumentException if any required data was missing or + * we had invalid parameters. + */ + public SaltScanner(final TSDB tsdb, final byte[] metric, + final List scanners, + final TreeMap spans, + final List filters) { + this(tsdb, metric, scanners, spans, filters, false, null, null, 0, null, 0, 0); + } + + /** + * Default ctor that performs some validation. Call {@link scan} after + * construction to actually start fetching data. + * @param tsdb The TSDB to which we belong + * @param metric The metric we're expecting to fetch + * @param scanners A list of HBase scanners, one for each bucket + * @param spans The span map to store results in + * @param delete Whether or not to delete the queried data + * @param rollup_query An optional rollup query config. May be null. + * @param filters A list of filters for processing + * @param query_stats A stats object for tracking timing + * @param query_index The index of the sub query in the main query list + * @param histogramSpans The histo map to populate. + * @param max_bytes The maximum number of bytes pulled out from all scanners + * combined. + * @param max_data_points The maximum number of data points pulled out from all + * scanners (estimated). + * @throws IllegalArgumentException if any required data was missing or + * we had invalid parameters. + */ + public SaltScanner(final TSDB tsdb, final byte[] metric, + final List scanners, + final TreeMap spans, + final List filters, + final boolean delete, + final RollupQuery rollup_query, + final QueryStats query_stats, + final int query_index, + final TreeMap histogramSpans, + final long max_bytes, + final long max_data_points) { + if (tsdb == null) { + throw new IllegalArgumentException("The TSDB argument was null."); + } + if (spans == null && histogramSpans == null) { + throw new IllegalArgumentException("Both Span map and HistogramSpan map were null."); + } + if (spans != null && !spans.isEmpty()) { + throw new IllegalArgumentException("The span map should be empty."); + } + if (histogramSpans != null && !histogramSpans.isEmpty()) { + throw new IllegalArgumentException("The histogram span map should be empty."); + } + if (scanners == null || scanners.isEmpty()) { + throw new IllegalArgumentException("Missing or empty scanners list. " + + "Please provide a list of scanners for each salt."); + } + if (Const.SALT_WIDTH() > 0 && scanners.size() != Const.SALT_BUCKETS()) { + throw new IllegalArgumentException("Not enough or too many scanners " + + scanners.size() + " when the salt bucket count is " + + Const.SALT_BUCKETS()); + } else if (Const.SALT_WIDTH() <= 0 && scanners.size() > 1) { + throw new IllegalArgumentException("Not enough or too many scanners " + + scanners.size() + " when the salting is disabled."); + } + if (metric == null) { + throw new IllegalArgumentException("The metric array was null."); + } + if (metric.length != TSDB.metrics_width()) { + throw new IllegalArgumentException("The metric was too short. It must be " + + TSDB.metrics_width() + "bytes wide."); + } + + this.scanners = scanners; + this.spans = spans != null ? Collections.synchronizedSortedMap(spans) : null; + this.histSpans = histogramSpans != null ? Collections.synchronizedSortedMap(histogramSpans) : null; + this.metric = metric; + this.tsdb = tsdb; + this.filters = filters; + this.delete = delete; + this.rollup_query = rollup_query; + this.query_stats = query_stats; + this.query_index = query_index; + countdown = new AtomicInteger(scanners.size()); + if (rollup_query != null && RollupQuery.isValidQuery(rollup_query)) { + is_rollup = true; + if (rollup_query.getRollupAgg() == Aggregators.AVG) { + rollup_agg_id = tsdb.getRollupConfig().getIdForAggregator("sum"); + rollup_count_id = tsdb.getRollupConfig().getIdForAggregator("count"); + } else { + rollup_agg_id = tsdb.getRollupConfig().getIdForAggregator( + rollup_query.getRollupAgg().toString()); + rollup_count_id = -1; + } + } else { + is_rollup = false; + rollup_agg_id = rollup_count_id = -1; + } + this.max_bytes = max_bytes; + this.max_data_points = max_data_points; + num_data_points = new AtomicLong(); + bytes_fetched = new AtomicLong(); + max_data_points_flag = new AtomicBoolean(); + } + + /** + * Starts all of the scanners asynchronously and returns the data fetched + * once all of the scanners have completed. Note that the result may be an + * exception if one or more of the scanners encountered an exception. The + * first error will be returned, others will be logged. + * @return A deferred to wait on for results. + */ + public Deferred> scan() { + start_time = System.currentTimeMillis(); + int i = 0; + for (final Scanner scanner: scanners) { + new ScannerCB(scanner, i++).scan(); + } + return results; + } + + public Deferred> scanHistogram() { + start_time = DateTime.currentTimeMillis(); + + int index = 0; + for (Scanner scanner: scanners) { + ScannerCB scnr = new ScannerCB(scanner, index++); + scnr.scan(); + } + + return histogramResults; + } + + /** + * Called once all of the scanners have reported back in to record our + * latency and merge the results into the spans map. If there was an exception + * stored then we'll return that instead. + */ + private void mergeAndReturnResults() { + final long hbase_time = DateTime.currentTimeMillis(); + TsdbQuery.scanlatency.add((int)(hbase_time - start_time)); + + if (exception != null) { + LOG.error("After all of the scanners finished, at " + + "least one threw an exception", exception); + results.callback(exception); + return; + } + + final long merge_start = DateTime.nanoTime(); + //Merge sorted spans together + if (!isHistogramScan()) { + mergeDataPoints(); + } else { + // Merge histogram data points + mergeHistogramDataPoints(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("It took " + (DateTime.currentTimeMillis() - hbase_time) + " ms, " + + " to merge and sort the rows into a tree map"); + } + + if (query_stats != null) { + query_stats.addStat(query_index, QueryStat.SCANNER_MERGE_TIME, + (DateTime.nanoTime() - merge_start)); + } + + if (!isHistogramScan()) { + results.callback(spans); + } else { + histogramResults.callback(histSpans); + } + } + + private boolean isHistogramScan() { + return histSpans != null; + } + + private void mergeHistogramDataPoints() { + if (histSpans != null) { + for (List>> rows : histMap.values()) { + if (null == rows || rows.isEmpty()) { + LOG.error("Found a histogram rows list that was null or empty"); + continue; + } + + // for all the rows with the same salt in the scann order - timestamp order + for (final SimpleEntry> row : rows) { + if (null == row) { + LOG.error("Found a histogram row item that was null"); + continue; + } + + HistogramSpan histSpan = null; + try { + histSpan = histSpans.get(row.getKey()); + } catch (RuntimeException e) { + LOG.error("Failed to fetch the histogram span", e); + } + + if (histSpan == null) { + histSpan = new HistogramSpan(tsdb); + histSpans.put(row.getKey(), histSpan); + } + + if (annotation_map.containsKey(row.getKey())) { + histSpan.getAnnotations().addAll(annotation_map.get(row.getKey())); + annotation_map.remove(row.getKey()); + } + + try { + histSpan.addRow(row.getKey(), row.getValue()); + } catch (RuntimeException e) { + LOG.error("Exception adding row to histogram span", e); + } + } // end for + } // end for + + histMap.clear(); + + for (byte[] key : annotation_map.keySet()) { + HistogramSpan histSpan = histSpans.get(key); + + if (histSpan == null) { + histSpan = new HistogramSpan(tsdb); + histSpans.put(key, histSpan); + } + + histSpan.getAnnotations().addAll(annotation_map.get(key)); + } + + annotation_map.clear(); + } + } + + /** + * Called once all of the scanners have reported back in to record our + * latency and merge the results into the spans map. If there was an exception + * stored then we'll return that instead. + */ + private void mergeDataPoints() { + // Merge sorted spans together + final long merge_start = DateTime.nanoTime(); + for (final List kvs : kv_map.values()) { + if (kvs == null || kvs.isEmpty()) { + LOG.warn("Found a key value list that was null or empty"); + continue; + } + + for (final KeyValue kv : kvs) { + if (kv == null) { + LOG.warn("Found a key value item that was null"); + continue; + } + if (kv.key() == null) { + LOG.warn("A key for a kv was null"); + continue; + } + + Span datapoints = spans.get(kv.key()); + if (datapoints == null) { + datapoints = RollupQuery.isValidQuery(rollup_query) ? + new RollupSpan(tsdb, this.rollup_query) : new Span(tsdb); + spans.put(kv.key(), datapoints); + } + + if (annotation_map.containsKey(kv.key())) { + for (final Annotation note: annotation_map.get(kv.key())) { + datapoints.getAnnotations().add(note); + } + annotation_map.remove(kv.key()); + } + try { + datapoints.addRow(kv); + } catch (RuntimeException e) { + LOG.error("Exception adding row to span", e); + throw e; + } + } + } + + kv_map.clear(); + + for (final byte[] key : annotation_map.keySet()) { + Span datapoints = spans.get(key); + if (datapoints == null) { + datapoints = new Span(tsdb); + spans.put(key, datapoints); + } + + for (final Annotation note: annotation_map.get(key)) { + datapoints.getAnnotations().add(note); + } + } + + annotation_map.clear(); + } + + /** + * Scanner callback executed recursively each time we get a set of data + * from storage. This is responsible for determining what columns are + * returned and issuing requests to load leaf objects. + * When the scanner returns a null set of rows, the method initiates the + * final callback. + */ + final class ScannerCB implements Callback>> { + private final Scanner scanner; + private final int index; + private final List kvs = Collections.synchronizedList(new ArrayList()); + private final ByteMap> annotations = + new ByteMap>(); + private final Set skips = Collections.newSetFromMap( + new ConcurrentHashMap()); + private final Set keepers = Collections.newSetFromMap( + new ConcurrentHashMap()); + + // use list here because we want to keep the rows in the scan order - + // timestamp order. I don't want to define an additional class to store the + // information of the row key and the histogram data points in the row, + // then use {@link SimpleEntry} + private List>> histograms = + Collections.synchronizedList(Lists.>>newArrayList()); + + private long scanner_start = -1; + /** nanosecond timestamps */ + private long fetch_start = 0; // reset each time we send an RPC to HBase + private long fetch_time = 0; // cumulation of time waiting on HBase + private long uid_resolve_time = 0; // cumulation of time resolving UIDs + private long uids_resolved = 0; + private long compaction_time = 0; // cumulation of time compacting + private long dps_pre_filter = 0; + private long rows_pre_filter = 0; + private long dps_post_filter = 0; + private long rows_post_filter = 0; + private long query_timeout = tsdb.getConfig().getLong("tsd.query.timeout"); + + public ScannerCB(final Scanner scanner, final int index) { + this.scanner = scanner; + this.index = index; + if (query_stats != null) { + query_stats.addScannerId(query_index, index, scanner.toString()); + } + } + + /** Error callback that will capture an exception from AsyncHBase and store + * it so we can bubble it up to the caller. + */ + class ErrorCb implements Callback { + @Override + public Object call(final Exception e) throws Exception { + LOG.error("Scanner " + scanner + " threw an exception", e); + close(false); + handleException(e); + return null; + } + } + + /** + * Starts the scanner and is called recursively to fetch the next set of + * rows from the scanner. + * @return The map of spans if loaded successfully, null if no data was + * found + */ + public Object scan() { + if (scanner_start < 0) { + scanner_start = DateTime.nanoTime(); + } + fetch_start = DateTime.nanoTime(); + return scanner.nextRows().addCallback(this).addErrback(new ErrorCb()); + } + + /** + * Iterate through each row of the scanner results, parses out data + * points (and optional meta data). + * @return null if no rows were found, otherwise the SortedMap with spans + */ + @Override + public Object call(final ArrayList> rows) + throws Exception { + try { + fetch_time += DateTime.nanoTime() - fetch_start; + if (rows == null) { + close(true); + return null; + } else if (exception != null) { + close(false); + // don't need to handleException here as it's already taken care of + // due to the fact that exception was set. + if (LOG.isDebugEnabled()) { + LOG.debug("Closing scanner as there was an exception: " + scanner); + } + return null; + } + + // used for UID resolution if a filter is involved + final List> lookups = + filters != null && !filters.isEmpty() ? + new ArrayList>(rows.size()) : null; + + // fail the query when the timeout exceeded + if (this.query_timeout > 0 && fetch_time > (this.query_timeout * 1000000)) { + try { + close(false); + handleException( + new QueryException(HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE, + "Sorry, your query timed out. Time limit: " + + this.query_timeout + " ms, fetch time: " + + (double)(fetch_time)/1000000 + " ms. Please try filtering " + + "using more tags or decrease your time range.")); + return false; + } catch (Exception e) { + LOG.error("Sorry, Scanner is closed: " + scanner, e); + return false; + } + } + + // validation checking before processing the next set of results. It's + // kinda funky but we want to allow queries to sneak through that were + // just a *tad* over the limits so that's why we don't check at the + // end of a scan call. + if (max_data_points > 0 && num_data_points.get() >= max_data_points) { + max_data_points_flag.getAndSet(true); + try { + close(false); + handleException( + new QueryException(HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE, + "Sorry, you have attempted to fetch more than our limit of " + + max_data_points + " data points. Please try filtering " + + "using more tags or decrease your time range.")); + return false; + } catch (Exception e) { + LOG.error("Sorry, Scanner is closed: " + scanner, e); + return false; + } + } + + if (max_bytes > 0 && bytes_fetched.get() > max_bytes) { + max_data_points_flag.getAndSet(true); + try { + close(false); + handleException( + new QueryException(HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE, + "Sorry, you have attempted to fetch more than our maximum " + + "amount of " + (max_bytes / 1024 / 1024) + "MB from storage. " + + "Please try filtering using more tags or decrease your time range.")); + return false; + } catch (Exception e) { + LOG.error("Sorry, Scanner is closed: " + scanner, e); + return false; + } + } + + rows_pre_filter += rows.size(); + for (final ArrayList row : rows) { + final byte[] key = row.get(0).key(); + num_data_points.addAndGet(row.size()); + if (RowKey.rowKeyContainsMetric(metric, key) != 0) { + close(false); + handleException(new IllegalDataException( + "HBase returned a row that doesn't match" + + " our scanner (" + scanner + ")! " + row + " does not start" + + " with " + Arrays.toString(metric) + " on scanner " + this)); + return null; + } + + // calculate estimated data point count. We don't want to deserialize + // the byte arrays so we'll just get a rough estimate of compacted + // columns. + long bytes = 0; + for (final KeyValue kv : row) { + // rough estimate of the # of bytes returned from storage. + bytes += key.length + kv.qualifier().length | kv.value().length; + bytes_fetched.addAndGet(bytes); + + if (kv.qualifier().length % 2 == 0) { + if (kv.qualifier().length == 2 || kv.qualifier().length == 4) { + ++dps_pre_filter; + } else { + // for now we'll assume that all compacted columns are of the + // same precision. This is likely incorrect. + if (Internal.inMilliseconds(kv.qualifier())) { + dps_pre_filter += (kv.qualifier().length / 4); + } else { + dps_pre_filter += (kv.qualifier().length / 2); + } + } + } else if (kv.qualifier()[0] == AppendDataPoints.APPEND_COLUMN_PREFIX) { + // with appends we don't have a good rough estimate as the length + // can vary widely with the value length variability. Therefore we + // have to iterate. + int idx = 0; + int qlength = 0; + while (idx < kv.value().length) { + qlength = Internal.getQualifierLength(kv.value(), idx); + idx += qlength + Internal.getValueLengthFromQualifier(kv.value(), idx); + ++dps_pre_filter; + } + } + } + + // If any filters have made it this far then we need to resolve + // the row key UIDs to their names for string comparison. We'll + // try to avoid the resolution with some sets but we may dupe + // resolve a few times. + // TODO - more efficient resolution + // TODO - byte set instead of a string for the uid may be faster + if (filters != null && !filters.isEmpty()) { + final String tsuid = + UniqueId.uidToString(UniqueId.getTSUIDFromKey(key, + TSDB.metrics_width(), Const.TIMESTAMP_BYTES)); + if (skips.contains(tsuid)) { + continue; + } + if (!keepers.contains(tsuid)) { + final long uid_start = DateTime.nanoTime(); + + /** CB to called after all of the UIDs have been resolved */ + class MatchCB implements Callback> { + @Override + public Object call(final ArrayList matches) + throws Exception { + for (final boolean matched : matches) { + if (!matched) { + skips.add(tsuid); + return null; + } + } + // matched all, good data + keepers.add(tsuid); + processRow(key, row); + return null; + } + } + + /** Resolves all of the row key UIDs to their strings for filtering */ + class GetTagsCB implements + Callback>, Map> { + @Override + public Deferred> call( + final Map tags) throws Exception { + uid_resolve_time += (DateTime.nanoTime() - uid_start); + uids_resolved += tags.size(); + final List> matches = + new ArrayList>(filters.size()); + + for (final TagVFilter filter : filters) { + matches.add(filter.match(tags)); + } + + return Deferred.group(matches); + } + } + + lookups.add(Tags.getTagsAsync(tsdb, key) + .addCallbackDeferring(new GetTagsCB()) + .addBoth(new MatchCB())); + } else { + processRow(key, row); + } + } else { + processRow(key, row); + } + } + + // either we need to wait on the UID resolutions or we can go ahead + // if we don't have filters. + if (lookups != null && lookups.size() > 0) { + class GroupCB implements Callback> { + @Override + public Object call(final ArrayList group) throws Exception { + return scan(); + } + } + return Deferred.group(lookups).addCallback(new GroupCB()); + } else { + return scan(); + } + } catch (final RuntimeException e) { + LOG.error("Unexpected exception on scanner " + this, e); + close(false); + handleException(e); + return null; + } + } + + /** + * Finds or creates the span for this row, compacts it and stores it. Also + * fires off a delete request for the row if told to. + * @param key The row key to use for fetching the span + * @param row The row to add + */ + void processRow(final byte[] key, final ArrayList row) { + ++rows_post_filter; + if (delete) { + final DeleteRequest del = new DeleteRequest(tsdb.dataTable(), key); + tsdb.getClient().delete(del); + } + + List hists = new ArrayList(); + + //TODO rollup doesn't use the column qualifier prefix right now + //Please move this logic to @CompactionQueue.compact API, if the + //qualifier prefix is set for rollup. Right now there is no way to + //identify whether a cell belong to rollup or default data table + //from the KeyValue/Hbase cell object + if (RollupQuery.isValidQuery(rollup_query)) { + //It is the rollup search result and rollup cells will not be + //compacted, so don't need to worry about complex or trivial + //compactions. It just need to consider the cells are different key + //values + for (KeyValue kv:row) { + final byte[] qual = kv.qualifier(); + + if (qual.length > 0) { + // TODO - allow rollups for annotations and histos? Probably will + // want to encode those on 4 bytes or something + if (!is_rollup && qual[0] == Annotation.PREFIX()) { + // This could be a row with only an annotation in it + final Annotation note = JSON.parseToObject(kv.value(), + Annotation.class); + synchronized (annotations) { + List map_notes = annotations.get(key); + if (map_notes == null) { + map_notes = new ArrayList(); + annotations.put(key, map_notes); + } + map_notes.add(note); + } + } else if (!is_rollup && qual[0] == HistogramDataPoint.PREFIX) { + try { + HistogramDataPoint histogram = + Internal.decodeHistogramDataPoint(tsdb, kv); + hists.add(histogram); + } catch (Throwable t) { + LOG.error("Failed to decode histogram data point", t); + } + } else { + if (rollup_query.getRollupAgg() == Aggregators.AVG || + rollup_query.getRollupAgg() == Aggregators.DEV) { + if (qual[0] == (byte) rollup_agg_id || + qual[0] == (byte) rollup_count_id || + Bytes.memcmp(RollupQuery.SUM, qual, 0, RollupQuery.SUM.length) == 0 || + Bytes.memcmp(RollupQuery.COUNT, qual, 0, RollupQuery.COUNT.length) == 0) { + kvs.add(kv); + } + } else if (qual[0] == (byte) rollup_agg_id || + Bytes.memcmp(rollup_query.getRollupAggPrefix(), + qual, 0, rollup_query.getRollupAggPrefix().length) == 0) { + kvs.add(kv); + } + } + } + } // end for + + // histogram row + if (hists.size() > 0) { + this.histograms.add(new SimpleEntry>(key, hists)); + } + } else { + // calculate estimated data point count. We don't want to deserialize + // the byte arrays so we'll just get a rough estimate of compacted + // columns. + for (final KeyValue kv : row) { + if (kv.qualifier().length % 2 == 0) { + if (kv.qualifier().length == 2 || kv.qualifier().length == 4) { + ++dps_post_filter; + } else { + // for now we'll assume that all compacted columns are of the + // same precision. This is likely incorrect. + if (Internal.inMilliseconds(kv.qualifier())) { + dps_post_filter += (kv.qualifier().length / 4); + } else { + dps_post_filter += (kv.qualifier().length / 2); + } + } + } else if (kv.qualifier()[0] == AppendDataPoints.APPEND_COLUMN_PREFIX) { + // with appends we don't have a good rough estimate as the length + // can vary widely with the value length variability. Therefore we + // have to iterate. + int idx = 0; + int qlength = 0; + while (idx < kv.value().length) { + qlength = Internal.getQualifierLength(kv.value(), idx); + idx += qlength + Internal.getValueLengthFromQualifier(kv.value(), idx); + ++dps_post_filter; + } + } + } + + final KeyValue compacted; + // let IllegalDataExceptions bubble up so the handler above can close + // the scanner + final long compaction_start = DateTime.nanoTime(); + try { + final List notes = Lists.newArrayList(); + compacted = tsdb.compact(row, notes, hists); + + // histogram row + if (hists.size() > 0) { + this.histograms.add(new SimpleEntry>(key, hists)); + } + + if (!notes.isEmpty()) { + synchronized (annotations) { + List map_notes = annotations.get(key); + if (map_notes == null) { + annotations.put(key, notes); + } else { + map_notes.addAll(notes); + } + } + } + } catch (IllegalDataException idex) { + compaction_time += (DateTime.nanoTime() - compaction_start); + close(false); + handleException(idex); + return; + } + compaction_time += (DateTime.nanoTime() - compaction_start); + if (compacted != null) { // Can be null if we ignored all KVs. + kvs.add(compacted); + } + } + } + + /** + * Closes the scanner and sets the various stats after filtering + * @param ok Whether or not the scanner closed with an exception or + * closed due to natural causes (e.g. ran out of data or we wanted to stop + * it early) + */ + void close(final boolean ok) { + scanner.close(); + + if (query_stats != null) { + query_stats.addScannerStat(query_index, index, QueryStat.SCANNER_TIME, + DateTime.nanoTime() - scanner_start); + + // Scanner Stats + /* Uncomment when AsyncHBase has this feature: + query_stats.addScannerStat(query_index, index, + QueryStat.ROWS_FROM_STORAGE, scanner.getRowsFetched()); + query_stats.addScannerStat(query_index, index, + QueryStat.COLUMNS_FROM_STORAGE, scanner.getColumnsFetched()); + query_stats.addScannerStat(query_index, index, + QueryStat.BYTES_FROM_STORAGE, scanner.getBytesFetched()); */ + query_stats.addScannerStat(query_index, index, + QueryStat.HBASE_TIME, fetch_time); + query_stats.addScannerStat(query_index, index, + QueryStat.SUCCESSFUL_SCAN, ok ? 1 : 0); + + // Post Scan stats + query_stats.addScannerStat(query_index, index, + QueryStat.ROWS_PRE_FILTER, rows_pre_filter); + query_stats.addScannerStat(query_index, index, + QueryStat.DPS_PRE_FILTER, dps_pre_filter); + query_stats.addScannerStat(query_index, index, + QueryStat.ROWS_POST_FILTER, rows_post_filter); + query_stats.addScannerStat(query_index, index, + QueryStat.DPS_POST_FILTER, dps_post_filter); + query_stats.addScannerStat(query_index, index, + QueryStat.SCANNER_UID_TO_STRING_TIME, uid_resolve_time); + query_stats.addScannerStat(query_index, index, + QueryStat.UID_PAIRS_RESOLVED, uids_resolved); + query_stats.addScannerStat(query_index, index, + QueryStat.COMPACTION_TIME, compaction_time); + } + + if (ok && exception == null) { + validateAndTriggerCallback(kvs, annotations, histograms); + } else { + countdown.decrementAndGet(); + } + } + } + + /** + * Called each time a scanner completes with valid or empty data. + * @param kvs The compacted columns fetched by the scanner + * @param annotations The annotations fetched by the scanners + */ + private void validateAndTriggerCallback( + final List kvs, + final Map> annotations, + final List>> histograms) { + + int scannersRunning = countdown.decrementAndGet(); + if (kvs.size() > 0) { + kv_map.put(scannersRunning, kvs); + } + + for (final byte[] key : annotations.keySet()) { + final List notes = annotations.get(key); + if (notes.size() > 0) { + // Optimistic write, expecting unique row keys + annotation_map.put(key, notes); + } + } + + if (histograms.size() > 0) { + histMap.put(scannersRunning, histograms); + } + + if (scannersRunning <= 0) { + try { + mergeAndReturnResults(); + } catch (final Exception ex) { + results.callback(ex); + } + } + } + + /** + * If one or more of the scanners throws an exception then we should close it + * and pass the exception here so that we can catch and return it to the + * caller. If all of the scanners have finished, this will callback to the + * caller immediately. + * @param e The exception to store. + */ + private void handleException(final Exception e) { + // make sure only one scanner can set the exception + countdown.decrementAndGet(); + if (exception == null) { + synchronized (this) { + if (exception == null) { + exception = e; + // fail once and fast on the first scanner to throw an exception + try { + mergeAndReturnResults(); + } catch (Exception ex) { + LOG.error("Failed merging and returning results, " + + "calling back with exception", ex); + results.callback(ex); + } + } else { + // TODO - it would be nice to close and cancel the other scanners but + // for now we have to wait for them to finish and/or throw exceptions. + LOG.error("Another scanner threw an exception", e); + } + } + } + } +} diff --git a/src/core/SeekableViewChain.java b/src/core/SeekableViewChain.java new file mode 100644 index 0000000000..63c0454240 --- /dev/null +++ b/src/core/SeekableViewChain.java @@ -0,0 +1,97 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2014 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import java.util.List; +import java.util.NoSuchElementException; + +public class SeekableViewChain implements SeekableView { + + private final List iterators; + private int currentIterator; + + SeekableViewChain(List iterators) { + this.iterators = iterators; + } + + /** + * Returns {@code true} if this view has more elements. + */ + @Override + public boolean hasNext() { + SeekableView iterator = getCurrentIterator(); + return iterator != null && iterator.hasNext(); + } + + /** + * Returns a view on the next data point. + * No new object gets created, the referenced returned is always the same + * and must not be stored since its internal data structure will change the + * next time {@code next()} is called. + * + * @throws NoSuchElementException if there were no more elements to iterate + * on (in which case {@link #hasNext} would have returned {@code false}. + */ + @Override + public DataPoint next() { + SeekableView iterator = getCurrentIterator(); + if (iterator == null || !iterator.hasNext()) { + throw new NoSuchElementException("No elements left in iterator"); + } + + DataPoint next = iterator.next(); + + if (!iterator.hasNext()) { + currentIterator++; + } + + return next; + } + + /** + * Unsupported operation. + * + * @throws UnsupportedOperationException always. + */ + @Override + public void remove() { + throw new UnsupportedOperationException("Removing items is not supported"); + } + + /** + * Advances the iterator to the given point in time. + *

    + * This allows the iterator to skip all the data points that are strictly + * before the given timestamp. + * + * @param timestamp A strictly positive 32 bit UNIX timestamp (in seconds). + * @throws IllegalArgumentException if the timestamp is zero, or negative, + * or doesn't fit on 32 bits (think "unsigned int" -- yay Java!). + */ + @Override + public void seek(long timestamp) { + for (final SeekableView it : iterators) { + it.seek(timestamp); + } + } + + private SeekableView getCurrentIterator() { + while (currentIterator < iterators.size()) { + if (iterators.get(currentIterator).hasNext()) { + return iterators.get(currentIterator); + } + currentIterator++; + } + return null; + } +} diff --git a/src/core/SimpleHistogram.java b/src/core/SimpleHistogram.java new file mode 100644 index 0000000000..e5e152cebb --- /dev/null +++ b/src/core/SimpleHistogram.java @@ -0,0 +1,366 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import org.hbase.async.Bytes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; + +import net.opentsdb.core.HistogramDataPoint.HistogramBucket; +import net.opentsdb.core.HistogramDataPoint.HistogramBucket.BucketType; + +/** + * A simple bucketed histogram with a fixed number of buckets, an underflow + * counter and an overflow counter. + * + * @since 3.0 + */ +public class SimpleHistogram implements Histogram { + private static Logger LOG = LoggerFactory.getLogger(SimpleHistogram.class); + + private final int id; + + @JsonProperty("buckets") + TreeMap buckets = new TreeMap(); + + @JsonProperty("underflow") + Long underflow = 0L; + + @JsonProperty("overflow") + Long overflow = 0L; + + public SimpleHistogram(final int id) { + this.id = id; + } + + public void addBucket(Float min, Float max, Long count) { + if (min == null || max == null) { + return; + } + if (count == null) { + count = 0L; //Prevent Null Exception + } + + buckets.put(new HistogramBucket(BucketType.REGULAR, min, max), count); + } + + public byte[] histogram(final boolean include_id) { + ByteArrayOutputStream outBuffer = new ByteArrayOutputStream(); + Output output = new Output(outBuffer); + + int bucketCount = buckets.size(); + try { + if (include_id) { + output.writeByte(id); + } + output.writeShort(bucketCount); + + for (Map.Entry bucket : buckets.entrySet()) { + output.writeFloat(bucket.getKey().getLowerBound()); + output.writeFloat(bucket.getKey().getUpperBound()); + output.writeLong(bucket.getValue(), true); + } + + output.writeLong(this.getUnderflow(), true); + output.writeLong(this.getOverflow(), true); + } finally { + output.close(); + } + return outBuffer.toByteArray(); + } + + public void fromHistogram(byte[] raw, final boolean include_id) { + if (raw.length < 6) { + throw new IllegalArgumentException("Byte array shorter than 6 bytes " + + "detected: " + Bytes.pretty(raw)); + } + Input input = null; + try { + input = new Input(new ByteArrayInputStream(raw)); + if (include_id) { + input.readByte(); // pull out the id. + } + int bucketCount = input.readShort(); + + for (int i = 0; i < bucketCount; i++) { + buckets.put(new HistogramBucket(BucketType.REGULAR, input.readFloat(), + input.readFloat()), input.readLong(true)); + } + + this.setUnderflow(input.readLong(true)); + this.setOverflow(input.readLong(true)); + } finally { + if (input != null) { + input.close(); + } + } + } + + private int calcBucketSum () { + int sum = 0; + for (Map.Entry entry : buckets.entrySet()) { + sum += entry.getValue().intValue(); + } + + return sum; + } + + public double percentile(double perc) { + if (perc < 1.0 || perc > 100.0) { + return -1.0; + } + + int bucketSum = calcBucketSum(); + + long runningCount = 0; + double prevBucketArea = 0.0; + double percValue = 0.0; + + for (Map.Entry entry : buckets.entrySet()) { + runningCount += entry.getValue().intValue(); + Double currBucketArea = runningCount * 100.0 / bucketSum; + + if (currBucketArea >= perc) { + //Find closest ranks + //Float currBucketStart = entry.getKey().getLowerBound(); + //Float nextBucketStart = entry.getKey().getUpperBound(); + + //percValue = currBucketStart + ((nextBucketStart - currBucketStart) * + // (perc - prevBucketArea) / (currBucketArea - prevBucketArea)); + percValue = (entry.getKey().getLowerBound() + + entry.getKey().getUpperBound()) / 2; + break; + } else { + prevBucketArea = runningCount * 100.0 / bucketSum; + } + } + + return percValue; + } + + @Override + public List percentiles(List percs) { + List percValues = new ArrayList(); + + for (Double perc : percs) { + percValues.add(percentile(perc)); + } + + return percValues; + } + + @JsonIgnore + @Override + public Map getHistogram() { + return Collections.unmodifiableMap(buckets); + } + + @Override + public SimpleHistogram clone() { + SimpleHistogram cloneObj = new SimpleHistogram(id); + + for (Map.Entry bucket : buckets.entrySet()) { + cloneObj.addBucket(bucket.getKey().getLowerBound(), + bucket.getKey().getUpperBound(), bucket.getValue()); + } + cloneObj.setUnderflow(underflow); + cloneObj.setOverflow(overflow); + + return cloneObj; + } + + @Override + public int getId() { + return id; + } + + public Long getBucketCount(Float min, Float max) { + HistogramBucket qryBucket = new HistogramBucket(BucketType.REGULAR, min, max); + if (buckets.containsKey(qryBucket)) { + Long bucketCount = buckets.get(qryBucket); + if (bucketCount == null) { + return 0L; + } else { + return bucketCount; + } + } else { + return 0L; + } + } + + public void write(Kryo kryo, Output output) { + int bucketCount = buckets.size(); + output.writeShort(bucketCount); + + for (Map.Entry bucket : buckets.entrySet()) { + bucket.getKey().write(kryo, output); + output.writeLong(bucket.getValue(), true); + } + + output.writeLong(this.getUnderflow(), true); + output.writeLong(this.getOverflow(), true); + } + + public void read(Kryo kryo, Input input) { + int bucketCount = input.readShort(); + if (bucketCount < 1) { + LOG.debug("Byte array passed has less than 1 histogram entry"); + return; + } + + for (int i = 0; i < bucketCount; i++) { + HistogramBucket bucket = new HistogramBucket(); + bucket.read(kryo, input); + buckets.put(bucket, input.readLong(true)); + } + + this.setUnderflow(input.readLong(true)); + this.setOverflow(input.readLong(true)); + } + + public void aggregate(Histogram histo, HistogramAggregation func) { + if (func == HistogramAggregation.SUM) { + SimpleHistogram y1Histo = (SimpleHistogram) histo; + for (Map.Entry bucket : + (Set>) histo.getHistogram().entrySet()) { + Long newCount = this.getBucketCount(bucket.getKey().getLowerBound(), + bucket.getKey().getUpperBound()) + bucket.getValue(); + this.addBucket(bucket.getKey().getLowerBound(), + bucket.getKey().getUpperBound(), newCount); + } + this.setOverflow(y1Histo.getOverflow() + this.getOverflow()); + this.setUnderflow(y1Histo.getUnderflow() + this.getUnderflow()); + } else { + LOG.debug("Unsupported histogram aggregation used"); + } + } + + public void aggregate(List histos, HistogramAggregation func) { + if (func == HistogramAggregation.SUM) { + for (Histogram histo : histos) { + this.aggregate(histo, HistogramAggregation.SUM); + } + } else { + LOG.debug("Unsupported histogram aggregation used"); + } + } + + public Long getUnderflow() { + return underflow; + } + + public void setUnderflow(Long underflow) { + this.underflow = underflow; + } + + public Long getOverflow() { + return overflow; + } + + public void setOverflow(Long overflow) { + this.overflow = overflow; + } + + /** + * Generates an array of bucket lower bounds from {@code start} to {@code end} + * with a fixed error interval (i.e. the span of the buckets). Up to 100 + * buckets can be created using this method. Additionally, a range of + * measurements can be provided to "focus" on with a smaller bucket span while + * the remaining buckets have a wider span. + * @param start The starting bucket measurement (lower bound). + * @param end The ending bucket measurement (upper bound). + * @param focusRangeStart The focus range start bound. Can be the same as + * {@code start}. + * @param focusRangeEnd The focus rang end bound. Can be the same as + * {@code end}. + * @param errorPct The acceptable error with respect to bucket width. E.g. + * 0.05 for 5%. + * @return An array of bucket lower bounds. + */ + public static double[] initializeHistogram (final float start, + final float end, + final float focusRangeStart, + final float focusRangeEnd, + final float errorPct) { + if (Float.compare(start, end) >= 0) { + throw new IllegalArgumentException("Histogram start (" + start + ") must be " + + "less than Histogram end (" + end +")"); + } else if (Float.compare(focusRangeStart, focusRangeEnd) >= 0) { + throw new IllegalArgumentException("Histogram focus range start (" + + focusRangeStart + ") must be less than Histogram focus end (" + + focusRangeEnd + ")"); + } else if (Float.compare(start, focusRangeStart) > 0 || + Float.compare(focusRangeStart, end) >= 0 || + Float.compare(start, focusRangeEnd) >= 0 || + Float.compare(focusRangeEnd, end) > 0) { + throw new IllegalArgumentException("Focus range start (" + focusRangeStart + + ") and Focus range end (" + focusRangeEnd + ") must be greater " + + "than Histogram start (" + start + ") and less than " + + "Histogram end (" + end + ")"); + } else if (Float.compare(errorPct, 0.0f) <= 0) { + throw new IllegalArgumentException("Error rate (" + errorPct + ") must be " + + "greater than zero"); + } + int MAX_BUCKETS = 100; + float stepSize = (1 + errorPct)/(1 - errorPct); + int bucketcount = Double.valueOf(Math.ceil( + Math.log(focusRangeEnd/focusRangeStart) / Math.log(stepSize))) + .intValue() + 1; + + if (Float.compare(start, focusRangeStart) < 0) { + bucketcount++; + } + + if (Float.compare(focusRangeEnd, end) < 0) { + bucketcount++; + } + + if (bucketcount > MAX_BUCKETS) { + throw new IllegalArgumentException("A max of " + MAX_BUCKETS + + " buckets are supported. " + bucketcount + " were requested"); + } + + double[] retval = new double[bucketcount]; + int j = 0; + if (Float.compare(start, focusRangeStart) < 0) { + retval[j] = start; + j++; + } + + for (float i = focusRangeStart; i < focusRangeEnd; i*=stepSize, j++) { + retval[j] = i; + } + + if (Float.compare(focusRangeEnd, end) < 0) { + retval[j++] = focusRangeEnd; + } + retval[j] = end; + + return retval; + } +} diff --git a/src/core/SimpleHistogramDataPointAdapter.java b/src/core/SimpleHistogramDataPointAdapter.java new file mode 100644 index 0000000000..84c69cd9d7 --- /dev/null +++ b/src/core/SimpleHistogramDataPointAdapter.java @@ -0,0 +1,139 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/** + * An adapter of TSDB's {@code HistogramDataPoint} interface with Yamas's + * {@code Histogram} interface. + * + * @since 2.4 + */ +public class SimpleHistogramDataPointAdapter implements HistogramDataPoint { + + private final Histogram histogram; + + private final long timestamp; + + public SimpleHistogramDataPointAdapter(final Histogram histogram, + final long timestamp) { + this.histogram = histogram; + this.timestamp = timestamp; + } + + protected SimpleHistogramDataPointAdapter( + final SimpleHistogramDataPointAdapter rhs) { + histogram = rhs.histogram.clone(); + timestamp = rhs.timestamp; + } + + protected SimpleHistogramDataPointAdapter( + final SimpleHistogramDataPointAdapter rhs, + final long timestamp) { + histogram = rhs.histogram.clone(); + this.timestamp = timestamp; + } + + @Override + public long timestamp() { + return timestamp; + } + + @Override + public byte[] getRawData(final boolean include_id) { + return histogram.histogram(include_id); + } + + @Override + public void resetFromRawData(final byte[] raw_data, final boolean includes_id) { + histogram.fromHistogram(raw_data, includes_id); + } + + @Override + public double percentile(final double p) { + return histogram.percentile(p); + } + + @Override + public List percentile(final List p) { + return histogram.percentiles(p); + } + + @Override + public void aggregate(final HistogramDataPoint histo, + final HistogramAggregation func) { + if (!(histo instanceof SimpleHistogramDataPointAdapter)) { + throw new IllegalArgumentException("The object must be an instance of the " + + "YamasHistogramDataPointAdapter"); + } + histogram.aggregate(((SimpleHistogramDataPointAdapter) histo).histogram, + mapAggregation(func)); + } + + @Override + public HistogramDataPoint clone() { + return new SimpleHistogramDataPointAdapter(this); + } + + @Override + public HistogramDataPoint cloneAndSetTimestamp(final long timestamp) { + return new SimpleHistogramDataPointAdapter(this, timestamp); + } + + @Override + public int getId() { + return histogram.getId(); + } + + private HistogramAggregation mapAggregation(final HistogramAggregation func) { + switch (func) { + case SUM: + return HistogramAggregation.SUM; + } + + throw new UnsupportedOperationException("Failed to map the aggregator."); + } + + @Override + public Map getHistogramBucketsIfHas() { + if (histogram instanceof SimpleHistogram) { + SimpleHistogram yms1_histogram = (SimpleHistogram) (histogram); + Map buckets = yms1_histogram.getHistogram(); + if (null != buckets) { + Map result = new TreeMap(); + for (Map.Entry entry : buckets.entrySet()) { + HistogramBucket bucket = new HistogramBucket( + HistogramBucket.BucketType.REGULAR, + entry.getKey().getLowerBound(), entry.getKey().getUpperBound()); + result.put(bucket, entry.getValue()); + } + + HistogramBucket underflow_bucket = new HistogramBucket( + HistogramBucket.BucketType.UNDERFLOW, 0.0f, 0.0f); + result.put(underflow_bucket, yms1_histogram.getUnderflow()); + + HistogramBucket overflow_bucket = new HistogramBucket( + HistogramBucket.BucketType.OVERFLOW, 0.0f, 0.0f); + result.put(overflow_bucket, yms1_histogram.getOverflow()); + return result; + } + } else { + throw new UnsupportedOperationException("The founding histogram object " + + "is not one of class Yamas1Histogram"); + } + return null; + } +} diff --git a/src/core/SimpleHistogramDecoder.java b/src/core/SimpleHistogramDecoder.java new file mode 100644 index 0000000000..31d0fd3146 --- /dev/null +++ b/src/core/SimpleHistogramDecoder.java @@ -0,0 +1,59 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import java.util.Arrays; + +/** + *

    + * Histogram decoder for Simple style histograms. + *

    + *

    + * It currently checks the first byte of the encoded data and decide the type of + * the histogram. Then it uses that histogram types builtin factory method to + * create the instance. + *

    + *

    + * This class is thread safe as it has no state. + *

    + * @since 2.4 + */ +public class SimpleHistogramDecoder extends HistogramDataPointCodec { + @Override + public Histogram decode(final byte[] raw_data, + final boolean includes_type) { + if (raw_data == null) { + throw new IllegalArgumentException("The data array cannot be null."); + } + if (includes_type && raw_data.length < 1) { + throw new IllegalArgumentException("The data array cannot be empty."); + } + if (includes_type && (int) raw_data[0] != id) { + throw new IllegalArgumentException("Data ID " + (int) raw_data[0] + + " did not match the codec ID " + id); + } + final Histogram histogram = new SimpleHistogram(id); + histogram.fromHistogram(raw_data, includes_type); + return histogram; + } + + @Override + public byte[] encode(final Histogram data_point, + final boolean include_id) { + if (!(data_point instanceof SimpleHistogram)) { + throw new IllegalArgumentException("The given histogram is not a " + + "SimpleHistogram: " + data_point.getClass()); + } + return data_point.histogram(include_id); + } +} diff --git a/src/core/Span.java b/src/core/Span.java index 6cf0d0c79f..ad413193db 100644 --- a/src/core/Span.java +++ b/src/core/Span.java @@ -1,5 +1,5 @@ // This file is part of OpenTSDB. -// Copyright (C) 2010-2012 The OpenTSDB Authors. +// Copyright (C) 2010-2015 The OpenTSDB Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU Lesser General Public License as published by @@ -20,10 +20,13 @@ import java.util.NoSuchElementException; import net.opentsdb.meta.Annotation; +import net.opentsdb.rollup.RollupQuery; +import net.opentsdb.uid.NoSuchUniqueId; import net.opentsdb.uid.UniqueId; import org.hbase.async.Bytes; import org.hbase.async.KeyValue; +import org.hbase.async.Bytes.ByteMap; import com.stumbleupon.async.Deferred; @@ -32,17 +35,17 @@ *

    * This class stores a continuous sequence of {@link RowSeq}s in memory. */ -final class Span implements DataPoints { +public class Span implements DataPoints { /** The {@link TSDB} instance we belong to. */ - private final TSDB tsdb; + protected final TSDB tsdb; /** All the rows in this span. */ - private final ArrayList rows = new ArrayList(); + protected final ArrayList rows = new ArrayList(); /** A list of annotations for this span. We can't lazily initialize since we * have to pass a collection to the compaction queue */ - private final ArrayList annotations = new ArrayList(0); + protected final ArrayList annotations = new ArrayList(0); /** * Whether or not the rows have been sorted. This should be toggled by the @@ -54,7 +57,7 @@ final class Span implements DataPoints { * Default constructor. * @param tsdb The TSDB to which we belong */ - Span(final TSDB tsdb) { + protected Span(final TSDB tsdb) { this.tsdb = tsdb; } @@ -85,6 +88,12 @@ public Deferred metricNameAsync() { return rows.get(0).metricNameAsync(); } + @Override + public byte[] metricUID() { + checkNotEmpty(); + return rows.get(0).metricUID(); + } + /** * @return the list of tag pairs for the rows in this span * @throws IllegalStateException if the span was empty @@ -105,6 +114,12 @@ public Deferred> getTagsAsync() { return rows.get(0).getTagsAsync(); } + @Override + public ByteMap getTagUids() { + checkNotEmpty(); + return rows.get(0).getTagUids(); + } + /** @return an empty list since aggregated tags cannot exist on a single span */ public List getAggregatedTags() { return Collections.emptyList(); @@ -114,13 +129,18 @@ public Deferred> getAggregatedTagsAsync() { final List empty = Collections.emptyList(); return Deferred.fromResult(empty); } + + @Override + public List getAggregatedTagUids() { + return Collections.emptyList(); + } /** @return the number of data points in this span, O(n) * Unfortunately we must walk the entire array for every row as there may be a * mix of second and millisecond timestamps */ public int size() { int size = 0; - for (final RowSeq row : rows) { + for (final iRowSeq row : rows) { size += row.size(); } return size; @@ -135,7 +155,7 @@ public List getTSUIDs() { if (rows.size() < 1) { return null; } - final byte[] tsuid = UniqueId.getTSUIDFromKey(rows.get(0).key, + final byte[] tsuid = UniqueId.getTSUIDFromKey(rows.get(0).key(), TSDB.metrics_width(), Const.TIMESTAMP_BYTES); final List tsuids = new ArrayList(1); tsuids.add(UniqueId.uidToString(tsuid)); @@ -154,26 +174,28 @@ public List getAnnotations() { * @throws IllegalArgumentException if the argument and this span are for * two different time series. */ - void addRow(final KeyValue row) { + protected void addRow(final KeyValue row) { long last_ts = 0; if (rows.size() != 0) { // Verify that we have the same metric id and tags. final byte[] key = row.key(); - final RowSeq last = rows.get(rows.size() - 1); + final iRowSeq last = rows.get(rows.size() - 1); final short metric_width = tsdb.metrics.width(); - final short tags_offset = (short) (metric_width + Const.TIMESTAMP_BYTES); + final short tags_offset = + (short) (Const.SALT_WIDTH() + metric_width + Const.TIMESTAMP_BYTES); final short tags_bytes = (short) (key.length - tags_offset); String error = null; - if (key.length != last.key.length) { + if (key.length != last.key().length) { error = "row key length mismatch"; - } else if (Bytes.memcmp(key, last.key, 0, metric_width) != 0) { + } else if ( + Bytes.memcmp(key, last.key(), Const.SALT_WIDTH(), metric_width) != 0) { error = "metric ID mismatch"; - } else if (Bytes.memcmp(key, last.key, tags_offset, tags_bytes) != 0) { + } else if (Bytes.memcmp(key, last.key(), tags_offset, tags_bytes) != 0) { error = "tags mismatch"; } if (error != null) { throw new IllegalArgumentException(error + ". " - + "This Span's last row key is " + Arrays.toString(last.key) + + "This Span's last row key is " + Arrays.toString(last.key()) + " whereas the row key being added is " + Arrays.toString(key) + " and metric_width=" + metric_width); } @@ -185,8 +207,9 @@ void addRow(final KeyValue row) { sorted = false; if (last_ts >= rowseq.timestamp(0)) { // scan to see if we need to merge into an existing row - for (final RowSeq rs : rows) { - if (Bytes.memcmp(rs.key, row.key()) == 0) { + for (final iRowSeq rs : rows) { + if (Bytes.memcmp(rs.key(), row.key(), Const.SALT_WIDTH(), + (rs.key().length - Const.SALT_WIDTH())) == 0) { rs.addRow(row); return; } @@ -205,7 +228,8 @@ void addRow(final KeyValue row) { */ static long lastTimestampInRow(final short metric_width, final KeyValue row) { - final long base_time = Bytes.getUnsignedInt(row.key(), metric_width); + final long base_time = Bytes.getUnsignedInt(row.key(), metric_width + + Const.SALT_WIDTH()); final byte[] qual = row.qualifier(); if (qual.length >= 4 && Internal.inMilliseconds(qual[qual.length - 4])) { return (base_time * 1000) + ((Bytes.getUnsignedInt(qual, qual.length - 4) & @@ -232,7 +256,7 @@ private long getIdxOffsetFor(final int i) { checkRowOrder(); int idx = 0; int offset = 0; - for (final RowSeq row : rows) { + for (final iRowSeq row : rows) { final int sz = row.size(); if (offset + sz > i) { break; @@ -336,12 +360,14 @@ public String toString() { private int seekRow(final long timestamp) { checkRowOrder(); int row_index = 0; - RowSeq row = null; + iRowSeq row = null; final int nrows = rows.size(); for (int i = 0; i < nrows; i++) { row = rows.get(i); final int sz = row.size(); - if (row.timestamp(sz - 1) < timestamp) { + if (sz < 1) { + row_index++; + } else if (row.timestamp(sz - 1) < timestamp) { row_index++; // The last DP in this row is before 'timestamp'. } else { break; @@ -381,32 +407,60 @@ final class Iterator implements SeekableView { private int row_index; /** Iterator on the current row. */ - private RowSeq.Iterator current_row; + private iRowSeq.Iterator current_row; Iterator() { current_row = rows.get(0).internalIterator(); } + // ------------------ // + // Iterator interface // + // ------------------ // + + @Override public boolean hasNext() { - return (current_row.hasNext() // more points in this row - || row_index < rows.size() - 1); // or more rows + if (current_row.hasNext()) { + return true; + } + // handle situations where a row in the middle may be empty due to some + // kind of logic kicking out data points + while (row_index < rows.size() - 1) { + row_index++; + current_row = rows.get(row_index).internalIterator(); + if (current_row.hasNext()) { + return true; + } + } + return false; } + @Override public DataPoint next() { if (current_row.hasNext()) { return current_row.next(); - } else if (row_index < rows.size() - 1) { + } + // handle situations where a row in the middle may be empty due to some + // kind of logic kicking out data points + while (row_index < rows.size() - 1) { row_index++; current_row = rows.get(row_index).internalIterator(); - return current_row.next(); + if (current_row.hasNext()) { + return current_row.next(); + } } throw new NoSuchElementException("no more elements"); } + @Override public void remove() { throw new UnsupportedOperationException(); } + // ---------------------- // + // SeekableView interface // + // ---------------------- // + + @Override public void seek(final long timestamp) { int row_index = seekRow(timestamp); if (row_index != this.row_index) { @@ -416,6 +470,7 @@ public void seek(final long timestamp) { current_row.seek(timestamp); } + @Override public String toString() { return "Span.Iterator(row_index=" + row_index + ", current_row=" + current_row + ", span=" + Span.this + ')'; @@ -424,13 +479,107 @@ public String toString() { } /** - * Package private iterator method to access data while downsampling. + * Package private iterator method to access data while downsampling with the + * option to force interpolation. + * @param start_time The time in milliseconds at which the data begins. + * @param end_time The time in milliseconds at which the data ends. * @param interval_ms The interval in milli seconds wanted between each data * point. * @param downsampler The downsampling function to use. + * @param fill_policy Policy specifying whether to interpolate or to fill + * missing intervals with special values. + * @return A new downsampler. */ - Downsampler downsampler(final long interval_ms, - final Aggregator downsampler) { - return new Downsampler(spanIterator(), interval_ms, downsampler); + Downsampler downsampler(final long start_time, + final long end_time, + final long interval_ms, + final Aggregator downsampler, + final FillPolicy fill_policy) { + if (FillPolicy.NONE == fill_policy) { + // The default downsampler simply skips missing intervals, causing the + // span group to linearly interpolate. + return new Downsampler(spanIterator(), interval_ms, downsampler); + } else { + // Otherwise, we need to instantiate a downsampler that can fill missing + // intervals with special values. + return new FillingDownsampler(spanIterator(), start_time, end_time, + interval_ms, downsampler, fill_policy); + } + } + + /** + * @param start_time The time in milliseconds at which the data begins. + * @param end_time The time in milliseconds at which the data ends. + * @param downsampler The downsampling specification to use + * @param query_start Start of the actual query + * @param query_end End of the actual query + * @return A new downsampler. + * @since 2.3 + */ + Downsampler downsampler(final long start_time, + final long end_time, + final DownsamplingSpecification downsampler, + final long query_start, + final long query_end) { + if (downsampler == null) { + return null; + } + if (FillPolicy.NONE == downsampler.getFillPolicy()) { + return new Downsampler(spanIterator(), downsampler, + query_start, query_end); + } + return new FillingDownsampler(spanIterator(), start_time, end_time, + downsampler, query_start, query_end); + } + + /** + * @param start_time The time in milliseconds at which the data begins. + * @param end_time The time in milliseconds at which the data ends. + * @param downsampler The downsampling specification to use + * @param query_start Start of the actual query + * @param query_end End of the actual query + * @param rollup_query An optional rollup query. + * @return A new downsampler. + * @since 2.4 + */ + Downsampler downsampler(final long start_time, + final long end_time, + final DownsamplingSpecification downsampler, + final long query_start, + final long query_end, + final RollupQuery rollup_query) { + if (downsampler == null) { + return null; + } + if (FillPolicy.NONE == downsampler.getFillPolicy()) { + return new Downsampler(spanIterator(), downsampler, + query_start, query_end, rollup_query); + } + return new FillingDownsampler(spanIterator(), start_time, end_time, + downsampler, query_start, query_end, rollup_query); + } + + /** + * RowSeq abstract factory API implementation + * @param tsdb The TSDB to which we belong + * @return RowSeq object which stores read-only sequence of continuous + * HBase rows + * @since 2.4 + */ + protected iRowSeq createRowSequence(TSDB tsdb) { + return new RowSeq(tsdb); + } + + public int getQueryIndex() { + throw new UnsupportedOperationException("Not mapped to a query"); + } + @Override + public boolean isPercentile() { + return false; + } + + @Override + public float getPercentile() { + throw new UnsupportedOperationException("getPercentile not supported"); } } diff --git a/src/core/SpanGroup.java b/src/core/SpanGroup.java index 976bab108b..07beaaa5d5 100644 --- a/src/core/SpanGroup.java +++ b/src/core/SpanGroup.java @@ -12,16 +12,16 @@ // see . package net.opentsdb.core; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; + +import org.hbase.async.Bytes; +import org.hbase.async.Bytes.ByteMap; import com.stumbleupon.async.Callback; import com.stumbleupon.async.Deferred; + import net.opentsdb.meta.Annotation; +import net.opentsdb.rollup.RollupQuery; /** * Groups multiple spans together and offers a dynamic "view" on them. @@ -45,7 +45,7 @@ * {@link Aggregator}) are given. This is done by using a special * iterator when using the {@link Span.DownsamplingIterator}. */ -final class SpanGroup implements DataPoints { +final class SpanGroup extends AbstractSpanGroup { /** Annotations */ private final ArrayList annotations; @@ -61,7 +61,8 @@ final class SpanGroup implements DataPoints { * in this group. * @see #computeTags */ - private HashMap tags; + private Map tags; + private ByteMap tag_uids; /** * The names of the tags that aren't shared by every single data point. @@ -69,7 +70,8 @@ final class SpanGroup implements DataPoints { * in this group. * @see #computeTags */ - private ArrayList aggregated_tags; + private List aggregated_tags; + private Set aggregated_tag_uids; /** Spans in this group. They must all be for the same metric. */ private final ArrayList spans = new ArrayList(); @@ -83,14 +85,26 @@ final class SpanGroup implements DataPoints { /** Aggregator to use to aggregate data points from different Spans. */ private final Aggregator aggregator; - /** - * Downsampling function to use, if any (can be {@code null}). - * If this is non-null, {@code sample_interval} must be strictly positive. - */ - private final Aggregator downsampler; + /** Downsampling specification to use, if any (can be {@code null}). */ + private DownsamplingSpecification downsampler; + + /** Start timestamp of the query for filtering */ + private final long query_start; + + /** End timestamp of the query for filtering */ + private final long query_end; - /** Minimum time interval (in seconds) wanted between each data point. */ - private final long sample_interval; + /** Index of the query in the TSQuery class */ + private final int query_index; + + /** An optional rollup query. */ + private final RollupQuery rollup_query; + + /** The TSDB to which we belong, used for resolution */ + private final TSDB tsdb; + + /** The group we belong to */ + private byte[] group; /** * Ctor. @@ -143,21 +157,134 @@ final class SpanGroup implements DataPoints { final boolean rate, final RateOptions rate_options, final Aggregator aggregator, final long interval, final Aggregator downsampler) { - annotations = new ArrayList(); - this.start_time = (start_time & Const.SECOND_MASK) == 0 ? start_time * 1000 : start_time; - this.end_time = (end_time & Const.SECOND_MASK) == 0 ? end_time * 1000 : end_time; - if (spans != null) { - for (final Span span : spans) { - add(span); - } - } - this.rate = rate; - this.rate_options = rate_options; - this.aggregator = aggregator; - this.downsampler = downsampler; - this.sample_interval = interval; + this(tsdb, start_time, end_time, spans, rate, rate_options, aggregator, + interval, downsampler, -1, FillPolicy.NONE); } + /** + * Ctor. + * @param tsdb The TSDB we belong to. + * @param start_time Any data point strictly before this timestamp will be + * ignored. + * @param end_time Any data point strictly after this timestamp will be + * ignored. + * @param spans A sequence of initial {@link Spans} to add to this group. + * Ignored if {@code null}. Additional spans can be added with {@link #add}. + * @param rate If {@code true}, the rate of the series will be used instead + * of the actual values. + * @param rate_options Specifies the optional additional rate calculation options. + * @param aggregator The aggregation function to use. + * @param interval Number of milliseconds wanted between each data point. + * @param downsampler Aggregation function to use to group data points + * within an interval. + * @param query_index index of the original query + * @param fill_policy Policy specifying whether to interpolate or to fill + * missing intervals with special values. + * @since 2.2 + */ + SpanGroup(final TSDB tsdb, + final long start_time, final long end_time, + final Iterable spans, + final boolean rate, final RateOptions rate_options, + final Aggregator aggregator, + final long interval, final Aggregator downsampler, final int query_index, + final FillPolicy fill_policy) { + this(tsdb, start_time, end_time, spans, rate, rate_options, aggregator, + downsampler != null ? + new DownsamplingSpecification(interval, downsampler, fill_policy) : + null, + 0, 0, query_index); + } + + /** + * Ctor. + * @param tsdb The TSDB we belong to. + * @param start_time Any data point strictly before this timestamp will be + * ignored. + * @param end_time Any data point strictly after this timestamp will be + * ignored. + * @param spans A sequence of initial {@link Spans} to add to this group. + * Ignored if {@code null}. Additional spans can be added with {@link #add}. + * @param rate If {@code true}, the rate of the series will be used instead + * of the actual values. + * @param rate_options Specifies the optional additional rate calculation options. + * @param aggregator The aggregation function to use. + * @param downsampler The specification to use for downsampling, may be null. + * @param query_start Start of the actual query + * @param query_end End of the actual query + * @param query_index index of the original query + * @since 2.3 + */ + SpanGroup(final TSDB tsdb, + final long start_time, + final long end_time, + final Iterable spans, + final boolean rate, + final RateOptions rate_options, + final Aggregator aggregator, + final DownsamplingSpecification downsampler, + final long query_start, + final long query_end, + final int query_index) { + this(tsdb, start_time, end_time, spans, rate, rate_options, aggregator, + downsampler, query_start, query_end, query_index, null, new byte[0]); + } + + /** + * Ctor. + * @param tsdb The TSDB we belong to. + * @param start_time Any data point strictly before this timestamp will be + * ignored. + * @param end_time Any data point strictly after this timestamp will be + * ignored. + * @param spans A sequence of initial {@link Spans} to add to this group. + * Ignored if {@code null}. Additional spans can be added with {@link #add}. + * @param rate If {@code true}, the rate of the series will be used instead + * of the actual values. + * @param rate_options Specifies the optional additional rate calculation options. + * @param aggregator The aggregation function to use. + * @param downsampler The specification to use for downsampling, may be null. + * @param query_start Start of the actual query + * @param query_end End of the actual query + * @param query_index index of the original query + * @param rollup_query An optional rollup query. + * @since 2.4 + */ + SpanGroup(final TSDB tsdb, + final long start_time, + final long end_time, + final Iterable spans, + final boolean rate, + final RateOptions rate_options, + final Aggregator aggregator, + final DownsamplingSpecification downsampler, + final long query_start, + final long query_end, + final int query_index, + final RollupQuery rollup_query, + byte[] group) { + annotations = new ArrayList(); + this.start_time = (start_time & Const.SECOND_MASK) == 0 ? + start_time * 1000 : start_time; + this.end_time = (end_time & Const.SECOND_MASK) == 0 ? + end_time * 1000 : end_time; + if (spans != null) { + for (final Span span : spans) { + add(span); + } + } + this.rate = rate; + this.rate_options = rate_options; + this.aggregator = aggregator; + this.downsampler = downsampler; + this.query_start = query_start; + this.query_end = query_end; + this.query_index = query_index; + this.rollup_query = rollup_query; + this.tsdb = tsdb; + this.group = group; + } + /** * Adds a span to this group, provided that it's in the right time range. * Must not be called once {@link #getTags} or @@ -213,63 +340,50 @@ void add(final Span span) { /** * Computes the intersection set + symmetric difference of tags in all spans. - * @param spans A collection of spans for which to find the common tags. - * @return A (possibly empty) map of the tags common to all the spans given. + * This method loads the UID aggregated list and tag pair maps with byte arrays + * but does not actually resolve the UIDs to strings. + * On the first run, it will initialize the UID collections (which may be empty) + * and subsequent calls will skip processing. */ - private Deferred computeTags() { + private void computeTags() { + if (tag_uids != null && aggregated_tag_uids != null) { + return; + } if (spans.isEmpty()) { - tags = new HashMap(0); - aggregated_tags = new ArrayList(0); - return Deferred.fromResult(null); + tag_uids = new ByteMap(); + aggregated_tag_uids = new HashSet(); + return; } - - final Iterator it = spans.iterator(); - /** - * This is the last callback that will determine what tags are aggregated in - * the results. - */ - class SpanTagsCB implements Callback>> { - public Object call(final ArrayList> lookups) - throws Exception { - final HashSet discarded_tags = new HashSet(tags.size()); - for (Map lookup : lookups) { - final Iterator> i = tags.entrySet().iterator(); - while (i.hasNext()) { - final Map.Entry entry = i.next(); - final String name = entry.getKey(); - final String value = lookup.get(name); - if (value == null || !value.equals(entry.getValue())) { - i.remove(); - discarded_tags.add(name); - } - } - } - SpanGroup.this.aggregated_tags = new ArrayList(discarded_tags); - return null; - } - } + // local tag uids + final ByteMap tag_set = new ByteMap(); - /** - * We have to wait for the first set of tags to be resolved so we can - * create a map with the proper size. Then we iterate through the rest of - * the tags for the different spans and work on each set. - */ - class FirstTagSetCB implements Callback> { - public Object call(final Map first_tags) throws Exception { - tags = new HashMap(first_tags); - final ArrayList>> deferreds = - new ArrayList>>(tags.size()); - - while (it.hasNext()) { - deferreds.add(it.next().getTagsAsync()); + // value is always null, we just want the set of unique keys + final ByteMap discards = new ByteMap(); + final Iterator it = spans.iterator(); + while (it.hasNext()) { + final Span span = it.next(); + final ByteMap uids = span.getTagUids(); + + for (final Map.Entry tag_pair : uids.entrySet()) { + // we already know it's an aggregated tag + if (discards.containsKey(tag_pair.getKey())) { + continue; } - return Deferred.groupInOrder(deferreds).addCallback(new SpanTagsCB()); + final byte[] tag_value = tag_set.get(tag_pair.getKey()); + if (tag_value == null) { + tag_set.put(tag_pair.getKey(), tag_pair.getValue()); + } else if (Bytes.memcmp(tag_value, tag_pair.getValue()) != 0) { + // bump to aggregated tags + discards.put(tag_pair.getKey(), null); + tag_set.remove(tag_pair.getKey()); + } } } - - return it.next().getTagsAsync().addCallback(new FirstTagSetCB()); + + aggregated_tag_uids = discards.keySet(); + tag_uids = tag_set; } public String metricName() { @@ -287,6 +401,11 @@ public Deferred metricNameAsync() { spans.get(0).metricNameAsync(); } + @Override + public byte[] metricUID() { + return spans.isEmpty() ? new byte[] {} : spans.get(0).metricUID(); + } + public Map getTags() { try { return getTagsAsync().joinUninterruptibly(); @@ -299,19 +418,29 @@ public Map getTags() { public Deferred> getTagsAsync() { if (tags != null) { - final Map local_tags = tags; - return Deferred.fromResult(local_tags); + return Deferred.fromResult(tags); } - class ComputeCB implements Callback, Object> { - public Map call(final Object obj) { - return tags; - } + if (spans.isEmpty()) { + tags = new HashMap(0); + return Deferred.fromResult(tags); + } + + if (tag_uids == null) { + computeTags(); } - return computeTags().addCallback(new ComputeCB()); + return resolveTags(tag_uids); } + @Override + public ByteMap getTagUids() { + if (tag_uids == null) { + computeTags(); + } + return tag_uids; + } + public List getAggregatedTags() { try { return getAggregatedTagsAsync().joinUninterruptibly(); @@ -324,17 +453,35 @@ public List getAggregatedTags() { public Deferred> getAggregatedTagsAsync() { if (aggregated_tags != null) { - final List agg_tags = aggregated_tags; - return Deferred.fromResult(agg_tags); + return Deferred.fromResult(aggregated_tags); } - class ComputeCB implements Callback, Object> { - public List call(final Object obj) { - return aggregated_tags; - } + if (spans.isEmpty()) { + aggregated_tags = new ArrayList(0); + return Deferred.fromResult(aggregated_tags); + } + + if (aggregated_tag_uids == null) { + computeTags(); } - return computeTags().addCallback(new ComputeCB()); + return resolveAggTags(aggregated_tag_uids); + } + + @Override + public List getAggregatedTagUids() { + if (aggregated_tag_uids != null) { + return new ArrayList(aggregated_tag_uids); + } + + if (spans.isEmpty()) { + return Collections.emptyList(); + } + + if (aggregated_tag_uids == null) { + computeTags(); + } + return new ArrayList(aggregated_tag_uids); } public List getTSUIDs() { @@ -378,36 +525,26 @@ public int aggregatedSize() { public SeekableView iterator() { return AggregationIterator.create(spans, start_time, end_time, aggregator, aggregator.interpolationMethod(), - downsampler, sample_interval, - rate, rate_options); - } - - /** - * Finds the {@code i}th data point of this group in {@code O(n)}. - * Where {@code n} is the number of data points in this group. - */ - private DataPoint getDataPoint(int i) { - if (i < 0) { - throw new IndexOutOfBoundsException("negative index: " + i); - } - final int saved_i = i; - final SeekableView it = iterator(); - DataPoint dp = null; - while (it.hasNext() && i >= 0) { - dp = it.next(); - i--; - } - if (i != -1 || dp == null) { - throw new IndexOutOfBoundsException("index " + saved_i - + " too large (it's >= " + size() + ") for " + this); - } - return dp; + downsampler, query_start, query_end, + rate, rate_options, rollup_query); } public long timestamp(final int i) { return getDataPoint(i).timestamp(); } + /** + * Returns the group the spans in here belong to. + * + * Returns null if the NONE aggregator was requested in the query + * Returns an empty array if there were no group bys and they're all in the same group + * Returns the group otherwise + * @return The group + */ + public byte[] group() { + return group; + } + public boolean isInteger(final int i) { return getDataPoint(i).isInteger(); } @@ -435,8 +572,104 @@ private String toStringSharedAttributes() { + ", rate=" + rate + ", aggregator=" + aggregator + ", downsampler=" + downsampler - + ", sample_interval=" + sample_interval + + ", query_start=" + query_start + + ", query_end=" + query_end + + ", group=" + Arrays.toString(group) + ')'; } + public int getQueryIndex() { + return query_index; + } + + @Override + public boolean isPercentile() { + return false; + } + + @Override + public float getPercentile() { + throw new UnsupportedOperationException("getPercentile not supported"); + } + + public List getSpans() { + return spans; + } + + /** + * Resolves the set of tag keys to their string names. + * @param tagks The set of unique tag names + * @return a deferred to wait on for all of the tag keys to be resolved. The + * result should be null. + */ + private Deferred> resolveAggTags(final Set tagks) { + if (aggregated_tags != null) { + return Deferred.fromResult(null); + } + aggregated_tags = new ArrayList(tagks.size()); + + final List> names = + new ArrayList>(tagks.size()); + for (final byte[] tagk : tagks) { + names.add(tsdb.tag_names.getNameAsync(tagk)); + } + + /** Adds the names to the aggregated_tags list */ + final class ResolveCB implements Callback, ArrayList> { + @Override + public List call(final ArrayList names) throws Exception { + for (final String name : names) { + aggregated_tags.add(name); + } + return aggregated_tags; + } + } + + return Deferred.group(names).addCallback(new ResolveCB()); + } + + /** + * Resolves the tags to their names, loading them into {@link tags} after + * initializing that map. + * @param tag_uids The tag UIDs + * @return A defeferred to wait on for resolution to complete, the result + * should be null. + */ + private Deferred> resolveTags(final ByteMap tag_uids) { + if (tags != null) { + return Deferred.fromResult(null); + } + tags = new HashMap(tag_uids.size()); + + final List> deferreds = + new ArrayList>(tag_uids.size()); + + /** Dumps the pairs into the map in the correct order */ + final class PairCB implements Callback> { + @Override + public Object call(final ArrayList pair) throws Exception { + tags.put(pair.get(0), pair.get(1)); + return null; + } + } + + /** Callback executed once all of the pairs are resolved and stored in the map */ + final class GroupCB implements Callback, ArrayList> { + @Override + public Map call(final ArrayList group) + throws Exception { + return tags; + } + } + + for (Map.Entry tag_pair : tag_uids.entrySet()) { + final List> resolve_pair = + new ArrayList>(2); + resolve_pair.add(tsdb.tag_names.getNameAsync(tag_pair.getKey())); + resolve_pair.add(tsdb.tag_values.getNameAsync(tag_pair.getValue())); + deferreds.add(Deferred.groupInOrder(resolve_pair).addCallback(new PairCB())); + } + + return Deferred.group(deferreds).addCallback(new GroupCB()); + } } diff --git a/src/core/SplitRollupQuery.java b/src/core/SplitRollupQuery.java new file mode 100644 index 0000000000..f422985377 --- /dev/null +++ b/src/core/SplitRollupQuery.java @@ -0,0 +1,480 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2010-2021 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import com.stumbleupon.async.Callback; +import com.stumbleupon.async.Deferred; +import net.opentsdb.rollup.RollupQuery; +import net.opentsdb.uid.NoSuchUniqueName; +import org.hbase.async.Bytes; +import org.hbase.async.Bytes.ByteMap; +import org.hbase.async.HBaseException; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.TreeSet; + +public class SplitRollupQuery extends AbstractQuery { + + private TSDB tsdb; + + private TsdbQuery rollupQuery; + private TsdbQuery rawQuery; + + private Deferred rollupResolution; + private Deferred rawResolution; + + SplitRollupQuery(final TSDB tsdb, TsdbQuery rollupQuery, Deferred rollupResolution) { + if (rollupQuery == null) { + throw new IllegalArgumentException("Rollup query cannot be null"); + } + + this.tsdb = tsdb; + + this.rollupQuery = rollupQuery; + this.rollupResolution = rollupResolution; + } + + /** + * Returns the start time of the graph. + * + * @return A strictly positive integer. + * @throws IllegalStateException if {@link #setStartTime(long)} was never + * called on this instance before. + */ + @Override + public long getStartTime() { + return rollupQuery != null ? rollupQuery.getStartTime() : rawQuery.getStartTime(); + } + + /** + * Sets the start time of the graph. Converts the timestamp to milliseconds if necessary. + * + * @param timestamp The start time, all the data points returned will have a + * timestamp greater than or equal to this one. + * @throws IllegalArgumentException if timestamp is less than or equal to 0, + * or if it can't fit on 32 bits. + * @throws IllegalArgumentException if + * {@code timestamp >= }{@link #getEndTime getEndTime}. + */ + @Override + public void setStartTime(long timestamp) { + if ((timestamp & Const.SECOND_MASK) == 0) { timestamp *= 1000L; } + + if (rollupQuery == null) { + rawQuery.setStartTime(timestamp); + return; + } + + if (rollupQuery.getEndTime() <= timestamp) { + rollupQuery = null; + rawQuery.setStartTime(timestamp); + return; + } + + rollupQuery.setStartTime(timestamp); + } + + /** + * Returns the end time of the graph. + *

    + * If {@link #setEndTime} was never called before, this method will + * automatically execute + * {@link #setEndTime setEndTime}{@code (System.currentTimeMillis() / 1000)} + * to set the end time. + * + * @return A strictly positive integer. + */ + @Override + public long getEndTime() { + return rawQuery != null ? rawQuery.getEndTime() : rollupQuery.getEndTime(); + } + + /** + * Sets the end time of the graph. Converts the timestamp to milliseconds if necessary. + * + * @param timestamp The end time, all the data points returned will have a + * timestamp less than or equal to this one. + * @throws IllegalArgumentException if timestamp is less than or equal to 0, + * or if it can't fit on 32 bits. + * @throws IllegalArgumentException if + * {@code timestamp <= }{@link #getStartTime getStartTime}. + */ + @Override + public void setEndTime(long timestamp) { + if ((timestamp & Const.SECOND_MASK) == 0) { timestamp *= 1000L; } + + rawQuery.setEndTime(timestamp); + } + + /** + * Returns whether or not the data queried will be deleted. + * + * @return A boolean + * @since 2.4 + */ + @Override + public boolean getDelete() { + return rawQuery.getDelete(); + } + + /** + * Sets whether or not the data queried will be deleted. + * + * @param delete True if data should be deleted, false otherwise. + * @since 2.4 + */ + @Override + public void setDelete(boolean delete) { + if (rollupQuery != null) { + rollupQuery.setDelete(delete); + } + rawQuery.setDelete(delete); + } + + /** + * Sets the time series to the query. + * + * @param metric The metric to retrieve from the TSDB. + * @param tags The set of tags of interest. + * @param function The aggregation function to use. + * @param rate If true, the rate of the series will be used instead of the + * actual values. + * @param rate_options If included specifies additional options that are used + * when calculating and graph rate values + * @throws NoSuchUniqueName if the name of a metric, or a tag name/value + * does not exist. + * @since 2.4 + */ + @Override + public void setTimeSeries(String metric, + Map tags, + Aggregator function, + boolean rate, + RateOptions rate_options) throws NoSuchUniqueName { + if (rollupQuery != null) { + rollupQuery.setTimeSeries(metric, tags, function, rate, rate_options); + } + rawQuery.setTimeSeries(metric, tags, function, rate, rate_options); + } + + /** + * Sets the time series to the query. + * + * @param metric The metric to retrieve from the TSDB. + * @param tags The set of tags of interest. + * @param function The aggregation function to use. + * @param rate If true, the rate of the series will be used instead of the + * actual values. + * @throws NoSuchUniqueName if the name of a metric, or a tag name/value + * does not exist. + */ + @Override + public void setTimeSeries(String metric, Map tags, Aggregator function, boolean rate) throws NoSuchUniqueName { + if (rollupQuery != null) { + rollupQuery.setTimeSeries(metric, tags, function, rate); + } + rawQuery.setTimeSeries(metric, tags, function, rate); + } + + /** + * Sets up a query for the given timeseries UIDs. For now, all TSUIDs in the + * group must share a common metric. This is to avoid issues where the scanner + * may have to traverse the entire data table if one TSUID has a metric of + * 000001 and another has a metric of FFFFFF. After modifying the query code + * to run asynchronously and use different scanners, we can allow different + * TSUIDs. + * Note: This method will not check to determine if the TSUIDs are + * valid, since that wastes time and we *assume* that the user provides TSUIDs + * that are up to date. + * + * @param tsuids A list of one or more TSUIDs to scan for + * @param function The aggregation function to use on results + * @param rate Whether or not the results should be converted to a rate + * @throws IllegalArgumentException if the tsuid list is null, empty or the + * TSUIDs do not share a common metric + * @since 2.4 + */ + @Override + public void setTimeSeries(List tsuids, Aggregator function, boolean rate) { + if (rollupQuery != null) { + rollupQuery.setTimeSeries(tsuids, function, rate); + } + rawQuery.setTimeSeries(tsuids, function, rate); + } + + /** + * Sets up a query for the given timeseries UIDs. For now, all TSUIDs in the + * group must share a common metric. This is to avoid issues where the scanner + * may have to traverse the entire data table if one TSUID has a metric of + * 000001 and another has a metric of FFFFFF. After modifying the query code + * to run asynchronously and use different scanners, we can allow different + * TSUIDs. + * Note: This method will not check to determine if the TSUIDs are + * valid, since that wastes time and we *assume* that the user provides TSUIDs + * that are up to date. + * + * @param tsuids A list of one or more TSUIDs to scan for + * @param function The aggregation function to use on results + * @param rate Whether or not the results should be converted to a rate + * @param rate_options If included specifies additional options that are used + * when calculating and graph rate values + * @throws IllegalArgumentException if the tsuid list is null, empty or the + * TSUIDs do not share a common metric + * @since 2.4 + */ + @Override + public void setTimeSeries(List tsuids, Aggregator function, boolean rate, RateOptions rate_options) { + if (rollupQuery != null) { + rollupQuery.setTimeSeries(tsuids, function, rate, rate_options); + } + rawQuery.setTimeSeries(tsuids, function, rate, rate_options); + } + + /** + * Prepares a query against HBase by setting up group bys and resolving + * strings to UIDs asynchronously. This replaces calls to all of the setters + * like the {@link setTimeSeries}, {@link setStartTime}, etc. + * Make sure to wait on the deferred return before calling {@link runAsync}. + * + * @param query The main query to fetch the start and end time from + * @param index The index of which sub query we're executing + * @return A deferred to wait on for UID resolution. The result doesn't have + * any meaning and can be discarded. + * @throws IllegalArgumentException if the query was missing sub queries or + * the index was out of bounds. + * @throws NoSuchUniqueName if the name of a metric, or a tag name/value + * does not exist. (Bubbles up through the deferred) + * @since 2.4 + */ + @Override + public Deferred configureFromQuery(TSQuery query, int index) { + return configureFromQuery(query, index, false); + } + + @Override + public Deferred configureFromQuery(TSQuery query, int index, boolean force_raw) { + if (force_raw) { + throw new UnsupportedOperationException("Not implemented yet"); + } + + if (!rollupQuery.needsSplitting()) { + return rollupResolution; + } + + rawQuery = new TsdbQuery(tsdb); + rawResolution = rollupQuery.split(query, index, rawQuery); + + if (rollupQuery.getRollupQuery().getLastRollupTimestampSeconds() * 1000L < rollupQuery.getStartTime()) { + // We're looking at a query that would normally hit a rollup table, but the table doesn't + // have data guaranteed to be available for the requested time period or any part of it + // (i.e. the last guaranteed rollup point is before the query actually starts) + // So we won't bother running it. + rollupQuery = null; + } + + return Deferred.group(rollupResolution, rawResolution).addCallback(new GroupCallback()); + } + + /** + * Downsamples the results by specifying a fixed interval between points. + *

    + * Technically, downsampling means reducing the sampling interval. Here + * the idea is similar. Instead of returning every single data point that + * matched the query, we want one data point per fixed time interval. The + * way we get this one data point is by aggregating all the data points of + * that interval together using an {@link Aggregator}. This enables you + * to compute things like the 5-minute average or 10 minute 99th percentile. + * + * @param interval Number of seconds wanted between each data point. + * @param downsampler Aggregation function to use to group data points + */ + @Override + public void downsample(long interval, Aggregator downsampler) { + if (rollupQuery != null) { + rollupQuery.downsample(interval, downsampler); + } + rawQuery.downsample(interval, downsampler); + } + + /** + * Sets an optional downsampling function on this query + * + * @param interval The interval, in milliseconds to rollup data points + * @param downsampler An aggregation function to use when rolling up data points + * @param fill_policy Policy specifying whether to interpolate or to fill + * missing intervals with special values. + * @throws NullPointerException if the aggregation function is null + * @throws IllegalArgumentException if the interval is not greater than 0 + * @since 2.4 + */ + @Override + public void downsample(long interval, Aggregator downsampler, FillPolicy fill_policy) { + if (rollupQuery != null) { + rollupQuery.downsample(interval, downsampler, fill_policy); + } + rawQuery.downsample(interval, downsampler, fill_policy); + } + + /** + * Executes the query asynchronously + * + * @return The data points matched by this query. + *

    + * Each element in the non-{@code null} but possibly empty array returned + * corresponds to one time series for which some data points have been + * matched by the query. + * @throws HBaseException if there was a problem communicating with HBase to + * perform the search. + * @since 1.2 + */ + @Override + public Deferred runAsync() throws HBaseException { + Deferred rollupResults = Deferred.fromResult(new DataPoints[0]); + if (rollupQuery != null) { + rollupResults = rollupQuery.runAsync(); + } + Deferred rawResults = rawQuery.runAsync(); + + return Deferred.groupInOrder(Arrays.asList(rollupResults, rawResults)).addCallback(new RunCB()); + } + + /** + * Runs this query asynchronously. + * + * @return The data points matched by this query and applied with percentile calculation + *

    + * Each element in the non-{@code null} but possibly empty array returned + * corresponds to one time series for which some data points have been + * matched by the query. + * @throws HBaseException if there was a problem communicating with HBase to + * perform the search. + * @throws IllegalStateException if the query is not a histogram query + */ + @Override + public Deferred runHistogramAsync() throws HBaseException { + Deferred rollupResults = Deferred.fromResult(new DataPoints[0]); + if (rollupQuery != null) { + rollupResults = rollupQuery.runHistogramAsync(); + } Deferred rawResults = rawQuery.runHistogramAsync(); + + return Deferred.groupInOrder(Arrays.asList(rollupResults, rawResults)).addCallback(new RunCB()); + } + + /** + * Returns an index for this sub-query in the original set of queries. + * + * @return A zero based index. + * @since 2.4 + */ + @Override + public int getQueryIdx() { + return rawQuery.getQueryIdx(); + } + + /** + * Check this is a histogram query or not + * + * @return + */ + @Override + public boolean isHistogramQuery() { + return rawQuery.isHistogramQuery(); + } + + /** + * Check this is a rollup query or not + * + * @return Whether or not this is a rollup query + * @since 2.4 + */ + @Override + public boolean isRollupQuery() { + return rollupQuery != null && RollupQuery.isValidQuery(rollupQuery.getRollupQuery()); + } + + /** + * @since 2.4 + */ + @Override + public boolean needsSplitting() { + // No further splitting supported + return false; + } + + /** + * Set the percentile calculation parameters for this query if this is + * a histogram query + * + * @param percentiles + */ + @Override + public void setPercentiles(List percentiles) { + if (rollupQuery != null) { + rollupQuery.setPercentiles(percentiles); + } + rawQuery.setPercentiles(percentiles); + } + + private class RunCB implements Callback> { + + private ByteMap makeSpanGroupMap(DataPoints[] dataPointsArray) { + ByteMap map = new ByteMap(); + + for (DataPoints points : dataPointsArray) { + if (!(points instanceof SpanGroup)) { + throw new IllegalArgumentException("Only SpanGroups implemented"); + } + SpanGroup spanGroup = (SpanGroup) points; + map.put(spanGroup.group(), spanGroup); + } + + return map; + } + + private DataPoints[] merge(DataPoints[] rollup, DataPoints[] raw) { + ByteMap rollupResults = makeSpanGroupMap(rollup); + ByteMap rawResults = makeSpanGroupMap(raw); + + TreeSet allGroups = new TreeSet(Bytes.MEMCMP); + allGroups.addAll(rollupResults.keySet()); + allGroups.addAll(rawResults.keySet()); + + List results = new ArrayList(allGroups.size()); + + for (byte[] group : allGroups) { + SpanGroup rawGroup = rawResults.get(group); + SpanGroup rollupGroup = rollupResults.get(group); + results.add(new SplitRollupSpanGroup(rollupGroup, rawGroup)); + } + + return results.toArray(new DataPoints[0]); + } + + /** + * After both queries have run, merge their results + * + * @param dataPointArrays The results from both queries + * @return The merged data points + */ + @Override + public DataPoints[] call(ArrayList dataPointArrays) { + DataPoints[] rollupResults = dataPointArrays.get(0); + DataPoints[] rawResults = dataPointArrays.get(1); + + return merge(rollupResults, rawResults); + } + } +} diff --git a/src/core/SplitRollupSpanGroup.java b/src/core/SplitRollupSpanGroup.java new file mode 100644 index 0000000000..188fdd4b44 --- /dev/null +++ b/src/core/SplitRollupSpanGroup.java @@ -0,0 +1,430 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2012-2021 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import com.stumbleupon.async.Callback; +import com.stumbleupon.async.Deferred; +import net.opentsdb.meta.Annotation; +import org.hbase.async.Bytes; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class SplitRollupSpanGroup extends AbstractSpanGroup { + private final List spanGroups = new ArrayList(); + + public SplitRollupSpanGroup(SpanGroup... groups) { + for (SpanGroup group : groups) { + if (group != null) { + spanGroups.add(group); + } + } + + if (spanGroups.isEmpty()) { + throw new IllegalArgumentException("At least one SpanGroup must be non-null"); + } + } + + /** + * Returns the name of the series. + */ + @Override + public String metricName() { + return spanGroups.get(0).metricName(); + } + + /** + * Returns the name of the series. + * + * @since 1.2 + */ + @Override + public Deferred metricNameAsync() { + return spanGroups.get(0).metricNameAsync(); + } + + /** + * @return the metric UID + * @since 2.3 + */ + @Override + public byte[] metricUID() { + return spanGroups.get(0).metricUID(); + } + + /** + * Returns the tags associated with these data points. + * + * @return A non-{@code null} map of tag names (keys), tag values (values). + */ + @Override + public Map getTags() { + Map tags = new HashMap(); + + for (SpanGroup group : spanGroups) { + tags.putAll(group.getTags()); + } + + return tags; + } + + /** + * Returns the tags associated with these data points. + * + * @return A non-{@code null} map of tag names (keys), tag values (values). + * @since 1.2 + */ + @Override + public Deferred> getTagsAsync() { + class GetTagsCB implements Callback, ArrayList>> { + @Override + public Map call(ArrayList> resolvedTags) throws Exception { + Map tags = new HashMap(); + for (Map groupTags : resolvedTags) { + tags.putAll(groupTags); + } + return tags; + } + } + + List>> deferreds = new ArrayList>>(spanGroups.size()); + + for (SpanGroup group : spanGroups) { + deferreds.add(group.getTagsAsync()); + } + + return Deferred.groupInOrder(deferreds).addCallback(new GetTagsCB()); + } + + /** + * Returns a map of tag pairs as UIDs. + * When used on a span or row, it returns the tag set. When used on a span + * group it will return only the tag pairs that are common across all + * time series in the group. + * + * @return A potentially empty map of tagk to tagv pairs as UIDs + * @since 2.2 + */ + @Override + public Bytes.ByteMap getTagUids() { + Bytes.ByteMap tagUids = new Bytes.ByteMap(); + + for (SpanGroup group : spanGroups) { + tagUids.putAll(group.getTagUids()); + } + + return tagUids; + } + + /** + * Returns the tags associated with some but not all of the data points. + *

    + * When this instance represents the aggregation of multiple time series + * (same metric but different tags), {@link #getTags} returns the tags that + * are common to all data points (intersection set) whereas this method + * returns all the tags names that are not common to all data points (union + * set minus the intersection set, also called the symmetric difference). + *

    + * If this instance does not represent an aggregation of multiple time + * series, the list returned is empty. + * + * @return A non-{@code null} list of tag names. + */ + @Override + public List getAggregatedTags() { + List aggregatedTags = new ArrayList(); + + for (SpanGroup group : spanGroups) { + aggregatedTags.addAll(group.getAggregatedTags()); + } + + return aggregatedTags; + } + + /** + * Returns the tags associated with some but not all of the data points. + *

    + * When this instance represents the aggregation of multiple time series + * (same metric but different tags), {@link #getTags} returns the tags that + * are common to all data points (intersection set) whereas this method + * returns all the tags names that are not common to all data points (union + * set minus the intersection set, also called the symmetric difference). + *

    + * If this instance does not represent an aggregation of multiple time + * series, the list returned is empty. + * + * @return A non-{@code null} list of tag names. + * @since 1.2 + */ + @Override + public Deferred> getAggregatedTagsAsync() { + class GetAggregatedTagsCB implements Callback, ArrayList>> { + @Override + public List call(ArrayList> resolvedTags) throws Exception { + List aggregatedTags = new ArrayList(); + for (List groupTags : resolvedTags) { + aggregatedTags.addAll(groupTags); + } + return aggregatedTags; + } + } + + List>> deferreds = + new ArrayList>>(spanGroups.size()); + for (SpanGroup group : spanGroups) { + deferreds.add(group.getAggregatedTagsAsync()); + } + + return Deferred.groupInOrder(deferreds).addCallback(new GetAggregatedTagsCB()); + } + + /** + * Returns the tagk UIDs associated with some but not all of the data points. + * + * @return a non-{@code null} list of tagk UIDs. + * @since 2.3 + */ + @Override + public List getAggregatedTagUids() { + List aggTagUids = new ArrayList(); + + for (SpanGroup group : spanGroups) { + aggTagUids.addAll(group.getAggregatedTagUids()); + } + + return aggTagUids; + } + + /** + * Returns a list of unique TSUIDs contained in the results + * + * @return an empty list if there were no results, otherwise a list of TSUIDs + */ + @Override + public List getTSUIDs() { + List tsuids = new ArrayList(); + + for (SpanGroup group : spanGroups) { + tsuids.addAll(group.getTSUIDs()); + } + + return tsuids; + } + + /** + * Compiles the annotations for each span into a new array list + * + * @return Null if none of the spans had any annotations, a list if one or + * more were found + */ + @Override + public List getAnnotations() { + List annotations = new ArrayList(); + + for (SpanGroup group : spanGroups) { + List groupAnnotations = group.getAnnotations(); + if (groupAnnotations != null) { + annotations.addAll(group.getAnnotations()); + } + } + + return annotations; + } + + /** + * Returns the number of data points. + *

    + * This method must be implemented in {@code O(1)} or {@code O(n)} + * where n = {@link #aggregatedSize} > 0. + * + * @return A positive integer. + */ + @Override + public int size() { + int size = 0; + for (SpanGroup group : spanGroups) { + size += group.size(); + } + return size; + } + + /** + * Returns the number of data points aggregated in this instance. + *

    + * When this instance represents the aggregation of multiple time series + * (same metric but different tags), {@link #size} returns the number of data + * points after aggregation, whereas this method returns the number of data + * points before aggregation. + *

    + * If this instance does not represent an aggregation of multiple time + * series, then 0 is returned. + * + * @return A positive integer. + */ + @Override + public int aggregatedSize() { + int aggregatedSize = 0; + for (SpanGroup group : spanGroups) { + aggregatedSize += group.aggregatedSize(); + } + return aggregatedSize; + } + + /** + * Returns a zero-copy view to go through {@code size()} data points. + *

    + * The iterator returned must return each {@link DataPoint} in {@code O(1)}. + * The {@link DataPoint} returned must not be stored and gets + * invalidated as soon as {@code next} is called on the iterator. If you + * want to store individual data points, you need to copy the timestamp + * and value out of each {@link DataPoint} into your own data structures. + */ + @Override + public SeekableView iterator() { + List iterators = new ArrayList(); + for (SpanGroup group : spanGroups) { + iterators.add(group.iterator()); + } + return new SeekableViewChain(iterators); + } + + /** + * Returns the timestamp associated with the {@code i}th data point. + * The first data point has index 0. + *

    + * This method must be implemented in + * O({@link #aggregatedSize}) or better. + *

    + * It is guaranteed that 

    timestamp(i) < timestamp(i+1)
    + * + * @param i + * @return A strictly positive integer. + * @throws IndexOutOfBoundsException if {@code i} is not in the range + * [0, {@link #size} - 1] + */ + @Override + public long timestamp(int i) { + return getDataPoint(i).timestamp(); + } + + /** + * Tells whether or not the {@code i}th value is of integer type. + * The first data point has index 0. + *

    + * This method must be implemented in + * O({@link #aggregatedSize}) or better. + * + * @param i + * @return {@code true} if the {@code i}th value is of integer type, + * {@code false} if it's of floating point type. + * @throws IndexOutOfBoundsException if {@code i} is not in the range + * [0, {@link #size} - 1] + */ + @Override + public boolean isInteger(int i) { + return getDataPoint(i).isInteger(); + } + + /** + * Returns the value of the {@code i}th data point as a long. + * The first data point has index 0. + *

    + * This method must be implemented in + * O({@link #aggregatedSize}) or better. + * Use {@link #iterator} to get successive {@code O(1)} accesses. + * + * @param i + * @throws IndexOutOfBoundsException if {@code i} is not in the range + * [0, {@link #size} - 1] + * @throws ClassCastException if the + * {@link #isInteger isInteger(i)} == false. + * @see #iterator + */ + @Override + public long longValue(int i) { + return getDataPoint(i).longValue(); + } + + /** + * Returns the value of the {@code i}th data point as a float. + * The first data point has index 0. + *

    + * This method must be implemented in + * O({@link #aggregatedSize}) or better. + * Use {@link #iterator} to get successive {@code O(1)} accesses. + * + * @param i + * @throws IndexOutOfBoundsException if {@code i} is not in the range + * [0, {@link #size} - 1] + * @throws ClassCastException if the + * {@link #isInteger isInteger(i)} == true. + * @see #iterator + */ + @Override + public double doubleValue(int i) { + return getDataPoint(i).doubleValue(); + } + + /** + * Return the query index that maps this datapoints to the original TSSubQuery. + * + * @return index of the query in the TSQuery class + * @throws UnsupportedOperationException if the implementing class can't map + * to a sub query. + * @since 2.2 + */ + @Override + public int getQueryIndex() { + return spanGroups.get(0).getQueryIndex(); + } + + /** + * Return whether these data points are the result of the percentile calculation + * on the histogram data points. The client can call {@code getPercentile} to get + * the percentile calculation parameter. + * + * @return true or false + * @since 2.4 + */ + @Override + public boolean isPercentile() { + return spanGroups.get(0).isPercentile(); + } + + /** + * Return the percentile calculation parameter. This interface and {@code isPercentile} are used + * to convert {@code HistogramDataPoints} to {@code DataPoints} + * + * @return the percentile parameter + * @since 2.4 + */ + @Override + public float getPercentile() { + return spanGroups.get(0).getPercentile(); + } + + /** + * Returns the group the spans in here belong to. + *

    + * Returns null if the NONE aggregator was requested in the query + * Returns an empty array if there were no group bys and they're all in the same group + * Returns the group otherwise + * + * @return The group + */ + public byte[] group() { + return spanGroups.get(0).group(); + } +} diff --git a/src/core/TSDB.java b/src/core/TSDB.java index a44f9ef094..53e6fb15fc 100644 --- a/src/core/TSDB.java +++ b/src/core/TSDB.java @@ -1,5 +1,5 @@ // This file is part of OpenTSDB. -// Copyright (C) 2010-2012 The OpenTSDB Authors. +// Copyright (C) 2010-2017 The OpenTSDB Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU Lesser General Public License as published by @@ -12,18 +12,29 @@ // see . package net.opentsdb.core; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; +import org.hbase.async.RegionLocation; +import org.hbase.async.HBaseRpc; +import com.google.common.base.Strings; +import com.google.common.io.Files; import com.stumbleupon.async.Callback; import com.stumbleupon.async.Deferred; import com.stumbleupon.async.DeferredGroupException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.hbase.async.AppendRequest; import org.hbase.async.Bytes; import org.hbase.async.Bytes.ByteMap; import org.hbase.async.ClientStats; @@ -33,22 +44,40 @@ import org.hbase.async.HBaseException; import org.hbase.async.KeyValue; import org.hbase.async.PutRequest; +import org.hbase.async.TableNotFoundException; +import org.jboss.netty.util.HashedWheelTimer; +import org.jboss.netty.util.Timeout; +import org.jboss.netty.util.Timer; +import net.opentsdb.auth.Authentication; import net.opentsdb.tree.TreeBuilder; import net.opentsdb.tsd.RTPublisher; -import net.opentsdb.tsd.RpcPlugin; +import net.opentsdb.tsd.StorageExceptionHandler; +import net.opentsdb.uid.NoSuchUniqueId; import net.opentsdb.uid.NoSuchUniqueName; import net.opentsdb.uid.UniqueId; +import net.opentsdb.uid.UniqueIdFilterPlugin; import net.opentsdb.uid.UniqueId.UniqueIdType; import net.opentsdb.utils.Config; import net.opentsdb.utils.DateTime; +import net.opentsdb.utils.JSON; import net.opentsdb.utils.PluginLoader; +import net.opentsdb.utils.Threads; import net.opentsdb.meta.Annotation; +import net.opentsdb.meta.MetaDataCache; import net.opentsdb.meta.TSMeta; import net.opentsdb.meta.UIDMeta; +import net.opentsdb.query.QueryLimitOverride; +import net.opentsdb.query.expression.ExpressionFactory; +import net.opentsdb.query.filter.TagVFilter; +import net.opentsdb.rollup.RollupConfig; +import net.opentsdb.rollup.RollupInterval; +import net.opentsdb.rollup.RollupUtils; import net.opentsdb.search.SearchPlugin; import net.opentsdb.search.SearchQuery; +import net.opentsdb.tools.StartupPlugin; import net.opentsdb.stats.Histogram; +import net.opentsdb.stats.QueryStats; import net.opentsdb.stats.StatsCollector; /** @@ -59,21 +88,61 @@ */ public final class TSDB { private static final Logger LOG = LoggerFactory.getLogger(TSDB.class); - + static final byte[] FAMILY = { 't' }; /** Charset used to convert Strings to byte arrays and back. */ private static final Charset CHARSET = Charset.forName("ISO-8859-1"); private static final String METRICS_QUAL = "metrics"; - private static final short METRICS_WIDTH = 3; + private static short METRICS_WIDTH = 3; private static final String TAG_NAME_QUAL = "tagk"; - private static final short TAG_NAME_WIDTH = 3; + private static short TAG_NAME_WIDTH = 3; private static final String TAG_VALUE_QUAL = "tagv"; - private static final short TAG_VALUE_WIDTH = 3; + private static short TAG_VALUE_WIDTH = 3; + private static final int MIN_HISTOGRAM_BYTES = 1; + + /** The operation mode (role) of the TSD. */ + public enum OperationMode { + READWRITE(true, true), + READONLY(true, false), + WRITEONLY(false, true); + + private final boolean read; + private final boolean write; + + OperationMode(boolean read, boolean write) { + this.read = read; + this.write = write; + } + + /** Whether this mode allows reading */ + public boolean isRead() { + return read; + } + + /** Whether this mode allows writing */ + public boolean isWrite() { + return write; + } + } + + /** Whether tables are fully available, partially available, or unavailable. + * + * The order matters, since we do ordinal comparison—lower should mean + * less available. + */ + public enum TableAvailability { + NONE, + PARTIAL, + FULL, + } /** Client for the HBase cluster to use. */ final HBaseClient client; + /** The operation mode (role) of the TSD. */ + final OperationMode mode; + /** Name of the table in which timeseries are stored. */ final byte[] table; /** Name of the table in which UID information is stored. */ @@ -93,6 +162,12 @@ public final class TSDB { /** Configuration object for all TSDB components */ final Config config; + /** Timer used for various tasks such as idle timeouts or query timeouts */ + private final HashedWheelTimer timer; + + /** RpcResponder for doing response asynchronously*/ + private final RpcResponder rpcResponder; + /** * Row keys that need to be compacted. * Whenever we write a new data point to a row, we add the row key to this @@ -101,14 +176,68 @@ public final class TSDB { */ private final CompactionQueue compactionq; - /** Search indexer to use if configure */ + /** Authentication Plugin to use if configured */ + private Authentication authentication = null; + + /** Search indexer to use if configured */ private SearchPlugin search = null; - + + /** Optional Startup Plugin to use if configured */ + private StartupPlugin startup = null; + /** Optional real time pulblisher plugin to use if configured */ private RTPublisher rt_publisher = null; + + /** Optional plugin for handling meta data caching and updating */ + private MetaDataCache meta_cache = null; + + /** Plugin for dealing with data points that can't be stored */ + private StorageExceptionHandler storage_exception_handler = null; + + /** A filter plugin for allowing or blocking time series */ + private WriteableDataPointFilterPlugin ts_filter; + + /** A filter plugin for allowing or blocking UIDs */ + private UniqueIdFilterPlugin uid_filter; + + /** The rollup config object for storing and querying rollups */ + private final RollupConfig rollup_config; + + /** The default rollup interval. */ + private final RollupInterval default_interval; + + /** Name of the tag we use to determine aggregates */ + private final String agg_tag_key; + + /** Name of the tag we use use for raw data. */ + private final String raw_agg_tag_value; + + /** Whether or not to tag raw data with the raw value tag */ + private final boolean tag_raw_data; - /** List of activated RPC plugins */ - private List rpc_plugins = null; + /** Whether or not to block writing of derived rollups/pre-ags */ + private final boolean rollups_block_derived; + + /** + * Whether or not to enable splitting rollup queries if the rollup table is lagging + * Global config setting: tsd.rollups.split_query.enable = true + */ + private final boolean rollups_split_queries; + + /** An optional histogram manger used when the TSD will be dealing with + * histograms and sketches. Instantiated ONLY if + * {@link #initializePlugins(boolean)} was called.*/ + private HistogramCodecManager histogram_manager; + + /** A list of query overrides for the scanners */ + private final QueryLimitOverride query_limits; + + /** Writes rejected by the filter */ + private final AtomicLong rejected_dps = new AtomicLong(); + private final AtomicLong rejected_aggregate_dps = new AtomicLong(); + + /** Datapoints Added */ + private static final AtomicLong datapoints_added = new AtomicLong(); /** * Constructor @@ -118,30 +247,125 @@ public final class TSDB { */ public TSDB(final HBaseClient client, final Config config) { this.config = config; - this.client = client; + if (client == null) { + final org.hbase.async.Config async_config; + if (config.configLocation() != null && !config.configLocation().isEmpty()) { + try { + async_config = new org.hbase.async.Config(config.configLocation()); + } catch (final IOException e) { + throw new RuntimeException("Failed to read the config file: " + + config.configLocation(), e); + } + } else { + async_config = new org.hbase.async.Config(); + } + async_config.overrideConfig("hbase.zookeeper.znode.parent", + config.getString("tsd.storage.hbase.zk_basedir")); + async_config.overrideConfig("hbase.zookeeper.quorum", + config.getString("tsd.storage.hbase.zk_quorum")); + this.client = new HBaseClient(async_config); + } else { + this.client = client; + } + + String string_mode = config.getString("tsd.mode"); + if (Strings.isNullOrEmpty(string_mode)) { + mode = OperationMode.READWRITE; + } else if (string_mode.toLowerCase().equals("ro")) { + mode = OperationMode.READONLY; + } else if (string_mode.toLowerCase().equals("wo")) { + mode = OperationMode.WRITEONLY; + } else { + mode = OperationMode.READWRITE; + } + + // SALT AND UID WIDTHS + // Users really wanted this to be set via config instead of having to + // compile. Hopefully they know NOT to change these after writing data. + if (config.hasProperty("tsd.storage.uid.width.metric")) { + METRICS_WIDTH = config.getShort("tsd.storage.uid.width.metric"); + } + if (config.hasProperty("tsd.storage.uid.width.tagk")) { + TAG_NAME_WIDTH = config.getShort("tsd.storage.uid.width.tagk"); + } + if (config.hasProperty("tsd.storage.uid.width.tagv")) { + TAG_VALUE_WIDTH = config.getShort("tsd.storage.uid.width.tagv"); + } + if (config.hasProperty("tsd.storage.max_tags")) { + Const.setMaxNumTags(config.getShort("tsd.storage.max_tags")); + } + if (config.hasProperty("tsd.storage.salt.buckets")) { + Const.setSaltBuckets(config.getInt("tsd.storage.salt.buckets")); + } + if (config.hasProperty("tsd.storage.salt.width")) { + Const.setSaltWidth(config.getInt("tsd.storage.salt.width")); + } - this.client.setFlushInterval(config.getShort("tsd.storage.flush_interval")); table = config.getString("tsd.storage.hbase.data_table").getBytes(CHARSET); uidtable = config.getString("tsd.storage.hbase.uid_table").getBytes(CHARSET); treetable = config.getString("tsd.storage.hbase.tree_table").getBytes(CHARSET); meta_table = config.getString("tsd.storage.hbase.meta_table").getBytes(CHARSET); - - metrics = new UniqueId(client, uidtable, METRICS_QUAL, METRICS_WIDTH); - tag_names = new UniqueId(client, uidtable, TAG_NAME_QUAL, TAG_NAME_WIDTH); - tag_values = new UniqueId(client, uidtable, TAG_VALUE_QUAL, TAG_VALUE_WIDTH); + + if (config.getBoolean("tsd.core.uid.random_metrics")) { + metrics = new UniqueId(this, uidtable, METRICS_QUAL, METRICS_WIDTH, true); + } else { + metrics = new UniqueId(this, uidtable, METRICS_QUAL, METRICS_WIDTH, false); + } + tag_names = new UniqueId(this, uidtable, TAG_NAME_QUAL, TAG_NAME_WIDTH, false); + tag_values = new UniqueId(this, uidtable, TAG_VALUE_QUAL, TAG_VALUE_WIDTH, false); compactionq = new CompactionQueue(this); if (config.hasProperty("tsd.core.timezone")) { DateTime.setDefaultTimezone(config.getString("tsd.core.timezone")); } - if (config.enable_realtime_ts() || config.enable_realtime_uid()) { - // this is cleaner than another constructor and defaults to null. UIDs - // will be refactored with DAL code anyways - metrics.setTSDB(this); - tag_names.setTSDB(this); - tag_values.setTSDB(this); + + timer = Threads.newTimer("TSDB Timer"); + + if (config.getBoolean("tsd.rollups.enable")) { + String conf = config.getString("tsd.rollups.config"); + if (Strings.isNullOrEmpty(conf)) { + throw new IllegalArgumentException("Rollups were enabled but " + + "'tsd.rollups.config' is null or empty."); + } + if (conf.endsWith(".json")) { + try { + conf = Files.toString(new File(conf), Const.UTF8_CHARSET); + } catch (IOException e) { + throw new IllegalArgumentException("Failed to open conf file: " + + conf, e); + } + } + rollup_config = JSON.parseToObject(conf, RollupConfig.class); + RollupInterval config_default = null; + for (final RollupInterval interval: rollup_config.getRollups().values()) { + if (interval.isDefaultInterval()) { + config_default = interval; + break; + } + } + if (config_default == null) { + throw new IllegalArgumentException("None of the rollup intervals were " + + "marked as the \"default\"."); + } + default_interval = config_default; + tag_raw_data = config.getBoolean("tsd.rollups.tag_raw"); + agg_tag_key = config.getString("tsd.rollups.agg_tag_key"); + raw_agg_tag_value = config.getString("tsd.rollups.raw_agg_tag_value"); + rollups_block_derived = config.getBoolean("tsd.rollups.block_derived"); + rollups_split_queries = config.getBoolean("tsd.rollups.split_query.enable"); + } else { + rollup_config = null; + default_interval = null; + tag_raw_data = false; + agg_tag_key = null; + raw_agg_tag_value = null; + rollups_block_derived = false; + rollups_split_queries = false; } + QueryStats.setEnableDuplicates( + config.getBoolean("tsd.query.allow_simultaneous_duplicates")); + if (config.getBoolean("tsd.core.preload_uid_cache")) { final ByteMap uid_cache_map = new ByteMap(); uid_cache_map.put(METRICS_QUAL.getBytes(CHARSET), metrics); @@ -149,6 +373,22 @@ public TSDB(final HBaseClient client, final Config config) { uid_cache_map.put(TAG_VALUE_QUAL.getBytes(CHARSET), tag_values); UniqueId.preloadUidCache(this, uid_cache_map); } + + if (config.getString("tsd.core.tag.allow_specialchars") != null) { + Tags.setAllowSpecialChars(config.getString("tsd.core.tag.allow_specialchars")); + } + + query_limits = new QueryLimitOverride(this); + + // load up the functions that require the TSDB object + ExpressionFactory.addTSDBFunctions(this); + + // set any extra tags from the config for stats + StatsCollector.setGlobalTags(config); + + + rpcResponder = new RpcResponder(config); + LOG.debug(config.dumpConfiguration()); } @@ -158,16 +398,30 @@ public TSDB(final HBaseClient client, final Config config) { * @since 2.0 */ public TSDB(final Config config) { - this(new HBaseClient(config.getString("tsd.storage.hbase.zk_quorum"), - config.getString("tsd.storage.hbase.zk_basedir")), - config); + this(null, config); } - + /** @return The data point column family name */ public static byte[] FAMILY() { return FAMILY; } - + + /** + * Called by initializePlugins, also used to load startup plugins. + * @since 2.3 + */ + public static void loadPluginPath(final String plugin_path) { + if (plugin_path != null && !plugin_path.isEmpty()) { + try { + PluginLoader.loadJARs(plugin_path); + } catch (Exception e) { + LOG.error("Error loading plugins from plugin path: " + plugin_path, e); + throw new RuntimeException("Error loading plugins from plugin path: " + + plugin_path, e); + } + } + } + /** * Should be called immediately after construction to initialize plugins and * objects that rely on such. It also moves most of the potential exception @@ -180,13 +434,39 @@ public static byte[] FAMILY() { */ public void initializePlugins(final boolean init_rpcs) { final String plugin_path = config.getString("tsd.core.plugin_path"); - if (plugin_path != null && !plugin_path.isEmpty()) { + loadPluginPath(plugin_path); + + try { + TagVFilter.initializeFilterMap(this); + // @#$@%$%#$ing typed exceptions + } catch (SecurityException e) { + throw new RuntimeException("Failed to instantiate filters", e); + } catch (IllegalArgumentException e) { + throw new RuntimeException("Failed to instantiate filters", e); + } catch (ClassNotFoundException e) { + throw new RuntimeException("Failed to instantiate filters", e); + } catch (NoSuchMethodException e) { + throw new RuntimeException("Failed to instantiate filters", e); + } catch (NoSuchFieldException e) { + throw new RuntimeException("Failed to instantiate filters", e); + } catch (IllegalAccessException e) { + throw new RuntimeException("Failed to instantiate filters", e); + } catch (InvocationTargetException e) { + throw new RuntimeException("Failed to instantiate filters", e); + } + + // load the authentication plugin if enabled + if (config.getBoolean("tsd.core.authentication.enable")) { + authentication = PluginLoader.loadSpecificPlugin( + config.getString("tsd.core.authentication.plugin"), Authentication.class); + if (authentication == null) { + throw new IllegalArgumentException("Unable to locate authentication " + + "plugin: " + config.getString("tsd.core.authentication.plugin")); + } try { - PluginLoader.loadJARs(plugin_path); + authentication.initialize(this); } catch (Exception e) { - LOG.error("Error loading plugins from plugin path: " + plugin_path, e); - throw new RuntimeException("Error loading plugins from plugin path: " + - plugin_path, e); + throw new RuntimeException("Failed to initialize authentication plugin", e); } } @@ -195,7 +475,7 @@ public void initializePlugins(final boolean init_rpcs) { search = PluginLoader.loadSpecificPlugin( config.getString("tsd.search.plugin"), SearchPlugin.class); if (search == null) { - throw new IllegalArgumentException("Unable to locate search plugin: " + + throw new IllegalArgumentException("Unable to locate search plugin: " + config.getString("tsd.search.plugin")); } try { @@ -203,20 +483,20 @@ public void initializePlugins(final boolean init_rpcs) { } catch (Exception e) { throw new RuntimeException("Failed to initialize search plugin", e); } - LOG.info("Successfully initialized search plugin [" + - search.getClass().getCanonicalName() + "] version: " + LOG.info("Successfully initialized search plugin [" + + search.getClass().getCanonicalName() + "] version: " + search.version()); } else { search = null; } - + // load the real time publisher plugin if enabled if (config.getBoolean("tsd.rtpublisher.enable")) { rt_publisher = PluginLoader.loadSpecificPlugin( config.getString("tsd.rtpublisher.plugin"), RTPublisher.class); if (rt_publisher == null) { throw new IllegalArgumentException( - "Unable to locate real time publisher plugin: " + + "Unable to locate real time publisher plugin: " + config.getString("tsd.rtpublisher.plugin")); } try { @@ -225,58 +505,176 @@ public void initializePlugins(final boolean init_rpcs) { throw new RuntimeException( "Failed to initialize real time publisher plugin", e); } - LOG.info("Successfully initialized real time publisher plugin [" + - rt_publisher.getClass().getCanonicalName() + "] version: " + LOG.info("Successfully initialized real time publisher plugin [" + + rt_publisher.getClass().getCanonicalName() + "] version: " + rt_publisher.version()); } else { rt_publisher = null; } - - if (init_rpcs && config.hasProperty("tsd.rpc.plugins")) { - final String[] plugins = config.getString("tsd.rpc.plugins").split(","); - for (final String plugin : plugins) { - final RpcPlugin rpc = PluginLoader.loadSpecificPlugin(plugin.trim(), - RpcPlugin.class); - if (rpc == null) { - throw new IllegalArgumentException( - "Unable to locate RPC plugin: " + plugin.trim()); - } - try { - rpc.initialize(this); - } catch (Exception e) { - throw new RuntimeException( - "Failed to initialize RPC plugin", e); - } - - if (rpc_plugins == null) { - rpc_plugins = new ArrayList(1); - } - rpc_plugins.add(rpc); - LOG.info("Successfully initialized RPC plugin [" + - rpc.getClass().getCanonicalName() + "] version: " - + rpc.version()); + + // load the meta cache plugin if enabled + if (config.getBoolean("tsd.core.meta.cache.enable")) { + meta_cache = PluginLoader.loadSpecificPlugin( + config.getString("tsd.core.meta.cache.plugin"), MetaDataCache.class); + if (meta_cache == null) { + throw new IllegalArgumentException( + "Unable to locate meta cache plugin: " + + config.getString("tsd.core.meta.cache.plugin")); + } + try { + meta_cache.initialize(this); + } catch (Exception e) { + throw new RuntimeException( + "Failed to initialize meta cache plugin", e); + } + LOG.info("Successfully initialized meta cache plugin [" + + meta_cache.getClass().getCanonicalName() + "] version: " + + meta_cache.version()); + } + + // load the storage exception plugin if enabled + if (config.getBoolean("tsd.core.storage_exception_handler.enable")) { + storage_exception_handler = PluginLoader.loadSpecificPlugin( + config.getString("tsd.core.storage_exception_handler.plugin"), + StorageExceptionHandler.class); + if (storage_exception_handler == null) { + throw new IllegalArgumentException( + "Unable to locate storage exception handler plugin: " + + config.getString("tsd.core.storage_exception_handler.plugin")); + } + try { + storage_exception_handler.initialize(this); + } catch (Exception e) { + throw new RuntimeException( + "Failed to initialize storage exception handler plugin", e); + } + LOG.info("Successfully initialized storage exception handler plugin [" + + storage_exception_handler.getClass().getCanonicalName() + "] version: " + + storage_exception_handler.version()); + } + + // Writeable Data Point Filter + if (config.getBoolean("tsd.timeseriesfilter.enable")) { + ts_filter = PluginLoader.loadSpecificPlugin( + config.getString("tsd.timeseriesfilter.plugin"), + WriteableDataPointFilterPlugin.class); + if (ts_filter == null) { + throw new IllegalArgumentException( + "Unable to locate time series filter plugin plugin: " + + config.getString("tsd.timeseriesfilter.plugin")); + } + try { + ts_filter.initialize(this); + } catch (Exception e) { + throw new RuntimeException( + "Failed to initialize time series filter plugin", e); + } + LOG.info("Successfully initialized time series filter plugin [" + + ts_filter.getClass().getCanonicalName() + "] version: " + + ts_filter.version()); + } + + // UID Filter + if (config.getBoolean("tsd.uidfilter.enable")) { + uid_filter = PluginLoader.loadSpecificPlugin( + config.getString("tsd.uidfilter.plugin"), + UniqueIdFilterPlugin.class); + if (uid_filter == null) { + throw new IllegalArgumentException( + "Unable to locate UID filter plugin plugin: " + + config.getString("tsd.uidfilter.plugin")); + } + try { + uid_filter.initialize(this); + } catch (Exception e) { + throw new RuntimeException( + "Failed to initialize UID filter plugin", e); } + LOG.info("Successfully initialized UID filter plugin [" + + uid_filter.getClass().getCanonicalName() + "] version: " + + uid_filter.version()); + } + + // finally load the histo manager after plugins have been loaded. + if (config.hasProperty("tsd.core.histograms.config")) { + histogram_manager = new HistogramCodecManager(this); + } else { + histogram_manager = null; } } + + /** + * Returns the configured Authentication Plugin + * @return The Authentication Plugin + * @since 2.4 + */ + public final Authentication getAuth() { + return this.authentication; + } /** - * Returns the configured HBase client + * Returns the configured HBase client * @return The HBase client - * @since 2.0 + * @since 2.0 */ public final HBaseClient getClient() { return this.client; } - - /** + + /** + * Sets the startup plugin so that it can be shutdown properly. + * Note that this method will not initialize or call any other methods + * belonging to the plugin's implementation. + * @param plugin The startup plugin that was used. + * @since 2.3 + */ + public final void setStartupPlugin(final StartupPlugin plugin) { + startup = plugin; + } + + /** + * Getter that returns the startup plugin object. + * @return The StartupPlugin object or null if the plugin was not set. + * @since 2.3 + */ + public final StartupPlugin getStartupPlugin() { + return startup; + } + + /** * Getter that returns the configuration object * @return The configuration object - * @since 2.0 + * @since 2.0 */ public final Config getConfig() { return this.config; } + /** + * Returns the storage exception handler. May be null if not enabled + * @return The storage exception handler + * @since 2.2 + */ + public final StorageExceptionHandler getStorageExceptionHandler() { + return storage_exception_handler; + } + + /** + * @return the TS filter object, may be null + * @since 2.3 + */ + public WriteableDataPointFilterPlugin getTSfilter() { + return ts_filter; + } + + /** + * @return The UID filter object, may be null. + * @since 2.3 + */ + public UniqueIdFilterPlugin getUidFilter() { + return uid_filter; + } + /** * Attempts to find the name for a unique identifier given a type * @param type The type of UID @@ -302,7 +700,7 @@ public Deferred getUidName(final UniqueIdType type, final byte[] uid) { throw new IllegalArgumentException("Unrecognized UID type"); } } - + /** * Attempts to find the UID matching a given name * @param type The type of UID @@ -312,21 +710,42 @@ public Deferred getUidName(final UniqueIdType type, final byte[] uid) { * @since 2.0 */ public byte[] getUID(final UniqueIdType type, final String name) { + try { + return getUIDAsync(type, name).join(); + } catch (NoSuchUniqueName e) { + throw e; + } catch (IllegalArgumentException e) { + throw e; + } catch (Exception e) { + LOG.error("Unexpected exception", e); + throw new RuntimeException(e); + } + } + + /** + * Attempts to find the UID matching a given name + * @param type The type of UID + * @param name The name to search for + * @throws IllegalArgumentException if the type is not valid + * @throws NoSuchUniqueName if the name was not found + * @since 2.1 + */ + public Deferred getUIDAsync(final UniqueIdType type, final String name) { if (name == null || name.isEmpty()) { throw new IllegalArgumentException("Missing UID name"); } switch (type) { case METRIC: - return this.metrics.getId(name); + return metrics.getIdAsync(name); case TAGK: - return this.tag_names.getId(name); + return tag_names.getIdAsync(name); case TAGV: - return this.tag_values.getId(name); + return tag_values.getIdAsync(name); default: throw new IllegalArgumentException("Unrecognized UID type"); } } - + /** * Verifies that the data and UID tables exist in HBase and optionally the * tree and meta data tables if the user has enabled meta tracking or tree @@ -336,7 +755,7 @@ public byte[] getUID(final UniqueIdType type, final String name) { * @since 2.0 */ public Deferred> checkNecessaryTablesExist() { - final ArrayList> checks = + final ArrayList> checks = new ArrayList>(2); checks.add(client.ensureTableExists( config.getString("tsd.storage.hbase.data_table"))); @@ -346,28 +765,167 @@ public Deferred> checkNecessaryTablesExist() { checks.add(client.ensureTableExists( config.getString("tsd.storage.hbase.tree_table"))); } - if (config.enable_realtime_ts() || config.enable_realtime_uid() || + if (config.enable_realtime_ts() || config.enable_realtime_uid() || config.enable_tsuid_incrementing()) { checks.add(client.ensureTableExists( config.getString("tsd.storage.hbase.meta_table"))); } return Deferred.group(checks); } - + + /* Suffix for queries from availability check below. */ + private static byte[] PROBE_SUFFIX = { + ':', 'A', 's', 'y', 'n', 'c', 'H', 'B', 'a', 's', 'e', + '~', 'p', 'r', 'o', 'b', 'e', '~', '<', ';', '_', '<', + }; + + /** + * Get availability status of regions for a table. + * + * Implemented as separate method so we can override it in unit tests. + * + * @return Per-region availability. + * + * @since 2.5 + */ + Deferred> getTableRegionAvailability(String table) { + final String table_id = config.getString(table); + + /** Convert result to true. */ + final class SuccessToBoolCallback implements Callback> { + @Override + public Boolean call(final ArrayList o) { + LOG.info("Check HBase availability, got success."); + return true; + } + } + + /** Convert error result to false. */ + final class FailureToBoolCallback implements Callback { + @Override + public Boolean call(final Exception e) { + LOG.error("Check HBase availability, got error:", e); + return false; + } + } + + final SuccessToBoolCallback successCB = new SuccessToBoolCallback(); + final FailureToBoolCallback failureCB = new FailureToBoolCallback(); + + /** Lookup availability of each region. */ + final class RegionInfoCallback implements Callback>,List> { + @Override + public Deferred> call(final List regions) { + LOG.info("Availability check got this many regions: " + regions.size()); + ArrayList> available = new ArrayList>(); + for (RegionLocation region : regions) { + // Use suffix so we don't hit real data: + final byte[] key = region.startKey(); + final byte[] testKey = new byte[key.length + 64]; + System.arraycopy(key, 0, testKey, 0, key.length); + System.arraycopy(PROBE_SUFFIX, 0, + testKey, testKey.length - PROBE_SUFFIX.length, + PROBE_SUFFIX.length); + LOG.debug("Checking region with start key " + testKey + " end key " + region.stopKey()); + GetRequest probe = new GetRequest(table_id, testKey); + // If we don't get a response within 1 second, assume the region is + // unavailable. + probe.setTimeout(1000); + probe.setFailfast(true); + available.add(client.get(probe).addCallbacks(successCB, failureCB)); + } + return Deferred.group(available); + } + } + + return client.locateRegions(config.getString(table)) + .addCallbackDeferring(new RegionInfoCallback()); + } + + /** + * Check for full or partial data and UID tables availability in HBase. + * + * @return Status of table availability. + * + * @since 2.5 + */ + public Deferred checkNecessaryTablesAvailability() { + /** Convert list of booleans (indicating a region being available) into full + * (all were available), partial (some were available), none (none were + * available). + */ + final class TableAvailabilityCB implements Callback> { + @Override + public TableAvailability call(final ArrayList available) { + if (available.size() == 0) { + return TableAvailability.NONE; + } + boolean hasAvailable = false; + boolean hasUnavailable = false; + for (Boolean regionAvailable : available) { + if (regionAvailable) { + hasAvailable = true; + } else { + hasUnavailable = true; + } + } + if (hasAvailable && hasUnavailable) { + return TableAvailability.PARTIAL; + } else if (hasAvailable) { + return TableAvailability.FULL; + } else { + return TableAvailability.NONE; + } + } + } + + /** If getting regions fails, availability is NONE. */ + final class FailedRegionInfoCallback implements Callback { + @Override + public TableAvailability call(final Exception e) { + LOG.error("Failed to get regions during table availability check", e); + return TableAvailability.NONE; + } + } + + ArrayList> tables = new ArrayList>(); + tables.add(getTableRegionAvailability("tsd.storage.hbase.uid_table") + .addCallbacks(new TableAvailabilityCB(), new FailedRegionInfoCallback())); + tables.add(getTableRegionAvailability("tsd.storage.hbase.data_table") + .addCallbacks(new TableAvailabilityCB(), new FailedRegionInfoCallback())); + + /** Combine availability for two tables by picking the lower of the two. */ + final class CombineAvailabilityCB implements Callback> { + @Override + public TableAvailability call(final ArrayList availabilities) { + assert availabilities.size() == 2; + TableAvailability result = TableAvailability.FULL; + for (TableAvailability availability: availabilities) { + if (availability.ordinal() < result.ordinal()) { + result = availability; + } + } + return result; + } + } + + return Deferred.group(tables).addCallback(new CombineAvailabilityCB()); + } + /** Number of cache hits during lookups involving UIDs. */ - public int uidCacheHits() { + public long uidCacheHits() { return (metrics.cacheHits() + tag_names.cacheHits() + tag_values.cacheHits()); } /** Number of cache misses during lookups involving UIDs. */ - public int uidCacheMisses() { + public long uidCacheMisses() { return (metrics.cacheMisses() + tag_names.cacheMisses() + tag_values.cacheMisses()); } /** Number of cache entries currently in RAM for lookups involving UIDs. */ - public int uidCacheSize() { + public long uidCacheSize() { return (metrics.cacheSize() + tag_names.cacheSize() + tag_values.cacheSize()); } @@ -377,40 +935,50 @@ public int uidCacheSize() { * @param collector The collector to use. */ public void collectStats(final StatsCollector collector) { - final byte[][] kinds = { - METRICS_QUAL.getBytes(CHARSET), - TAG_NAME_QUAL.getBytes(CHARSET), - TAG_VALUE_QUAL.getBytes(CHARSET) + final byte[][] kinds = { + METRICS_QUAL.getBytes(CHARSET), + TAG_NAME_QUAL.getBytes(CHARSET), + TAG_VALUE_QUAL.getBytes(CHARSET) }; try { final Map used_uids = UniqueId.getUsedUIDs(this, kinds) .joinUninterruptibly(); - + collectUidStats(metrics, collector); - collector.record("uid.ids-used", used_uids.get(METRICS_QUAL), - "kind=" + METRICS_QUAL); - collector.record("uid.ids-available", - (metrics.maxPossibleId() - used_uids.get(METRICS_QUAL)), - "kind=" + METRICS_QUAL); - + if (config.getBoolean("tsd.core.uid.random_metrics")) { + collector.record("uid.ids-used", 0, "kind=" + METRICS_QUAL); + collector.record("uid.ids-available", 0, "kind=" + METRICS_QUAL); + } else { + collector.record("uid.ids-used", used_uids.get(METRICS_QUAL), + "kind=" + METRICS_QUAL); + collector.record("uid.ids-available", + (Internal.getMaxUnsignedValueOnBytes(metrics.width()) - + used_uids.get(METRICS_QUAL)), "kind=" + METRICS_QUAL); + } + collectUidStats(tag_names, collector); - collector.record("uid.ids-used", used_uids.get(TAG_NAME_QUAL), + collector.record("uid.ids-used", used_uids.get(TAG_NAME_QUAL), "kind=" + TAG_NAME_QUAL); - collector.record("uid.ids-available", - (tag_names.maxPossibleId() - used_uids.get(TAG_NAME_QUAL)), + collector.record("uid.ids-available", + (Internal.getMaxUnsignedValueOnBytes(tag_names.width()) - + used_uids.get(TAG_NAME_QUAL)), "kind=" + TAG_NAME_QUAL); - + collectUidStats(tag_values, collector); - collector.record("uid.ids-used", used_uids.get(TAG_VALUE_QUAL), + collector.record("uid.ids-used", used_uids.get(TAG_VALUE_QUAL), "kind=" + TAG_VALUE_QUAL); - collector.record("uid.ids-available", - (tag_values.maxPossibleId() - used_uids.get(TAG_VALUE_QUAL)), - "kind=" + TAG_VALUE_QUAL); - + collector.record("uid.ids-available", + (Internal.getMaxUnsignedValueOnBytes(tag_values.width()) - + used_uids.get(TAG_VALUE_QUAL)), "kind=" + TAG_VALUE_QUAL); + } catch (Exception e) { throw new RuntimeException("Shouldn't be here", e); } + collector.record("uid.filter.rejected", rejected_dps.get(), "kind=raw"); + collector.record("uid.filter.rejected", rejected_aggregate_dps.get(), + "kind=aggregate"); + { final Runtime runtime = Runtime.getRuntime(); collector.record("jvm.ramfree", runtime.freeMemory()); @@ -424,6 +992,20 @@ public void collectStats(final StatsCollector collector) { collector.clearExtraTag("class"); } + collector.addExtraTag("class", "IncomingDataPoints"); + try { + collector.record("uid.autometric.rejections", IncomingDataPoints.auto_metric_rejection_count, "method=put"); + } finally { + collector.clearExtraTag("class"); + } + + collector.addExtraTag("class", "TSDB"); + try { + collector.record("datapoints.added", datapoints_added, "type=all"); + } finally { + collector.clearExtraTag("class"); + } + collector.addExtraTag("class", "TsdbQuery"); try { collector.record("hbase.latency", TsdbQuery.scanlatency, "method=scan"); @@ -441,44 +1023,88 @@ public void collectStats(final StatsCollector collector) { collector.record("hbase.rpcs", stats.deletes(), "type=delete"); collector.record("hbase.rpcs", stats.gets(), "type=get"); collector.record("hbase.rpcs", stats.puts(), "type=put"); + collector.record("hbase.rpcs", stats.appends(), "type=append"); collector.record("hbase.rpcs", stats.rowLocks(), "type=rowLock"); collector.record("hbase.rpcs", stats.scannersOpened(), "type=openScanner"); collector.record("hbase.rpcs", stats.scans(), "type=scan"); collector.record("hbase.rpcs.batched", stats.numBatchedRpcSent()); collector.record("hbase.flushes", stats.flushes()); collector.record("hbase.connections.created", stats.connectionsCreated()); + collector.record("hbase.connections.idle_closed", stats.idleConnectionsClosed()); collector.record("hbase.nsre", stats.noSuchRegionExceptions()); collector.record("hbase.nsre.rpcs_delayed", stats.numRpcDelayedDueToNSRE()); + collector.record("hbase.region_clients.open", + stats.regionClients()); + collector.record("hbase.region_clients.idle_closed", + stats.idleConnectionsClosed()); compactionq.collectStats(collector); // Collect Stats from Plugins + if (startup != null) { + try { + collector.addExtraTag("plugin", "startup"); + startup.collectStats(collector); + } finally { + collector.clearExtraTag("plugin"); + } + } if (rt_publisher != null) { try { - collector.addExtraTag("plugin", "publish"); + collector.addExtraTag("plugin", "publish"); rt_publisher.collectStats(collector); } finally { - collector.clearExtraTag("plugin"); - } + collector.clearExtraTag("plugin"); + } + } + if (authentication != null) { + try { + collector.addExtraTag("plugin", "authentication"); + authentication.collectStats(collector); + } finally { + collector.clearExtraTag("plugin"); + } } if (search != null) { try { - collector.addExtraTag("plugin", "search"); - search.collectStats(collector); + collector.addExtraTag("plugin", "search"); + search.collectStats(collector); + } finally { + collector.clearExtraTag("plugin"); + } + } + if (storage_exception_handler != null) { + try { + collector.addExtraTag("plugin", "storageExceptionHandler"); + storage_exception_handler.collectStats(collector); } finally { - collector.clearExtraTag("plugin"); - } + collector.clearExtraTag("plugin"); + } } - if (rpc_plugins != null) { + if (ts_filter != null) { try { - collector.addExtraTag("plugin", "rpc"); - for(RpcPlugin rpc: rpc_plugins) { - rpc.collectStats(collector); - } + collector.addExtraTag("plugin", "timeseriesFilter"); + ts_filter.collectStats(collector); } finally { - collector.clearExtraTag("plugin"); - } - } + collector.clearExtraTag("plugin"); + } + } + if (uid_filter != null) { + try { + collector.addExtraTag("plugin", "uidFilter"); + uid_filter.collectStats(collector); + } finally { + collector.clearExtraTag("plugin"); + } + } + if (meta_cache != null) { + try { + collector.addExtraTag("plugin", "metaCache"); + meta_cache.collectStats(collector); + } finally { + collector.clearExtraTag("plugin"); + } + } } /** Returns a latency histogram for Put RPCs used to store data points. */ @@ -501,23 +1127,27 @@ private static void collectUidStats(final UniqueId uid, collector.record("uid.cache-hit", uid.cacheHits(), "kind=" + uid.kind()); collector.record("uid.cache-miss", uid.cacheMisses(), "kind=" + uid.kind()); collector.record("uid.cache-size", uid.cacheSize(), "kind=" + uid.kind()); + collector.record("uid.random-collisions", uid.randomIdCollisions(), + "kind=" + uid.kind()); + collector.record("uid.rejected-assignments", uid.rejectedAssignments(), + "kind=" + uid.kind()); } /** @return the width, in bytes, of metric UIDs */ public static short metrics_width() { return METRICS_WIDTH; } - + /** @return the width, in bytes, of tagk UIDs */ public static short tagk_width() { return TAG_NAME_WIDTH; } - + /** @return the width, in bytes, of tagv UIDs */ public static short tagv_width() { return TAG_VALUE_WIDTH; } - + /** * Returns a new {@link Query} instance suitable for this TSDB. */ @@ -536,15 +1166,337 @@ public WritableDataPoints newDataPoints() { } /** - * Adds a single integer value data point in the TSDB. + * Returns a new {@link BatchedDataPoints} instance suitable for this TSDB. + * + * @param metric Every data point that gets appended must be associated to this metric. + * @param tags The associated tags for all data points being added. + * @return data structure which can have data points appended. + */ + public WritableDataPoints newBatch(String metric, Map tags) { + return new BatchedDataPoints(this, metric, tags); + } + + /** + * Adds a single integer value data point in the TSDB. + *

    + * WARNING: The tags map may be modified by this method without a lock. Give + * the method a copy if you plan to use it elsewhere. + *

    + * @param metric A non-empty string. + * @param timestamp The timestamp associated with the value. + * @param value The value of the data point. + * @param tags The tags on this series. This map must be non-empty. + * @return A deferred object that indicates the completion of the request. + * The {@link Object} has not special meaning and can be {@code null} (think + * of it as {@code Deferred}). But you probably want to attach at + * least an errback to this {@code Deferred} to handle failures. + * @throws IllegalArgumentException if the timestamp is less than or equal + * to the previous timestamp added or 0 for the first timestamp, or if the + * difference with the previous timestamp is too large. + * @throws IllegalArgumentException if the metric name is empty or contains + * illegal characters. + * @throws IllegalArgumentException if the tags list is empty or one of the + * elements contains illegal characters. + * @throws HBaseException (deferred) if there was a problem while persisting + * data. + */ + public Deferred addPoint(final String metric, + final long timestamp, + final long value, + final Map tags) { + final byte[] v; + if (Byte.MIN_VALUE <= value && value <= Byte.MAX_VALUE) { + v = new byte[] { (byte) value }; + } else if (Short.MIN_VALUE <= value && value <= Short.MAX_VALUE) { + v = Bytes.fromShort((short) value); + } else if (Integer.MIN_VALUE <= value && value <= Integer.MAX_VALUE) { + v = Bytes.fromInt((int) value); + } else { + v = Bytes.fromLong(value); + } + + final short flags = (short) (v.length - 1); // Just the length. + return addPointInternal(metric, timestamp, v, tags, flags); + } + + /** + * Adds a double precision floating-point value data point in the TSDB. + *

    + * WARNING: The tags map may be modified by this method without a lock. Give + * the method a copy if you plan to use it elsewhere. + *

    + * @param metric A non-empty string. + * @param timestamp The timestamp associated with the value. + * @param value The value of the data point. + * @param tags The tags on this series. This map must be non-empty. + * @return A deferred object that indicates the completion of the request. + * The {@link Object} has not special meaning and can be {@code null} (think + * of it as {@code Deferred}). But you probably want to attach at + * least an errback to this {@code Deferred} to handle failures. + * @throws IllegalArgumentException if the timestamp is less than or equal + * to the previous timestamp added or 0 for the first timestamp, or if the + * difference with the previous timestamp is too large. + * @throws IllegalArgumentException if the metric name is empty or contains + * illegal characters. + * @throws IllegalArgumentException if the value is NaN or infinite. + * @throws IllegalArgumentException if the tags list is empty or one of the + * elements contains illegal characters. + * @throws HBaseException (deferred) if there was a problem while persisting + * data. + * @since 1.2 + */ + public Deferred addPoint(final String metric, + final long timestamp, + final double value, + final Map tags) { + if (Double.isNaN(value) || Double.isInfinite(value)) { + throw new IllegalArgumentException("value is NaN or Infinite: " + value + + " for metric=" + metric + + " timestamp=" + timestamp); + } + final short flags = Const.FLAG_FLOAT | 0x7; // A float stored on 8 bytes. + return addPointInternal(metric, timestamp, + Bytes.fromLong(Double.doubleToRawLongBits(value)), + tags, flags); + } + + /** + * Adds a single floating-point value data point in the TSDB. + *

    + * WARNING: The tags map may be modified by this method without a lock. Give + * the method a copy if you plan to use it elsewhere. + *

    + * @param metric A non-empty string. + * @param timestamp The timestamp associated with the value. + * @param value The value of the data point. + * @param tags The tags on this series. This map must be non-empty. + * @return A deferred object that indicates the completion of the request. + * The {@link Object} has not special meaning and can be {@code null} (think + * of it as {@code Deferred}). But you probably want to attach at + * least an errback to this {@code Deferred} to handle failures. + * @throws IllegalArgumentException if the timestamp is less than or equal + * to the previous timestamp added or 0 for the first timestamp, or if the + * difference with the previous timestamp is too large. + * @throws IllegalArgumentException if the metric name is empty or contains + * illegal characters. + * @throws IllegalArgumentException if the value is NaN or infinite. + * @throws IllegalArgumentException if the tags list is empty or one of the + * elements contains illegal characters. + * @throws HBaseException (deferred) if there was a problem while persisting + * data. + */ + public Deferred addPoint(final String metric, + final long timestamp, + final float value, + final Map tags) { + if (Float.isNaN(value) || Float.isInfinite(value)) { + throw new IllegalArgumentException("value is NaN or Infinite: " + value + + " for metric=" + metric + + " timestamp=" + timestamp); + } + final short flags = Const.FLAG_FLOAT | 0x3; // A float stored on 4 bytes. + return addPointInternal(metric, timestamp, + Bytes.fromInt(Float.floatToRawIntBits(value)), + tags, flags); + } + + /** + * Adds an encoded Histogram data point in the TSDB. + * @param metric A non-empty string. + * @param timestamp The timestamp associated with the value. + * @param raw_data The encoded data blob of the Histogram point. + * @param tags The tags on this series. This map must be non-empty. + * @return A deferred object that indicates the completion of the request. + * The {@link Object} has not special meaning and can be {@code null} (think + * of it as {@code Deferred}). But you probably want to attach at + * least an errback to this {@code Deferred} to handle failures. + * @throws IllegalArgumentException if the timestamp is less than or equal + * to the previous timestamp added or 0 for the first timestamp, or if the + * difference with the previous timestamp is too large. + * @throws IllegalArgumentException if the metric name is empty or contains + * illegal characters. + * @throws IllegalArgumentException if the tags list is empty or one of the + * elements contains illegal characters. + * @throws HBaseException (deferred) if there was a problem while persisting + * data. + */ + public Deferred addHistogramPoint(final String metric, + final long timestamp, + final byte[] raw_data, + final Map tags) { + if (raw_data == null || raw_data.length < MIN_HISTOGRAM_BYTES) { + return Deferred.fromError(new IllegalArgumentException( + "The histogram raw data is invalid: " + Bytes.pretty(raw_data))); + } + + checkTimestampAndTags(metric, timestamp, raw_data, tags, (short) 0); + final byte[] row = IncomingDataPoints.rowKeyTemplate(this, metric, tags); + + final byte[] qualifier = Internal.getQualifier(timestamp, + HistogramDataPoint.PREFIX); + + return storeIntoDB(metric, timestamp, raw_data, tags, (short) 0, row, qualifier); + } + + final Deferred addPointInternal(final String metric, + final long timestamp, + final byte[] value, + final Map tags, + final short flags) { + + checkTimestampAndTags(metric, timestamp, value, tags, flags); + final byte[] row = IncomingDataPoints.rowKeyTemplate(this, metric, tags); + + final byte[] qualifier = Internal.buildQualifier(timestamp, flags); + + return storeIntoDB(metric, timestamp, value, tags, flags, row, qualifier); + } + + private final Deferred storeIntoDB(final String metric, + final long timestamp, + final byte[] value, + final Map tags, + final short flags, + final byte[] row, + final byte[] qualifier) { + final long base_time; + + if ((timestamp & Const.SECOND_MASK) != 0) { + // drop the ms timestamp to seconds to calculate the base timestamp + base_time = ((timestamp / 1000) - + ((timestamp / 1000) % Const.MAX_TIMESPAN)); + } else { + base_time = (timestamp - (timestamp % Const.MAX_TIMESPAN)); + } + + /** Callback executed for chaining filter calls to see if the value + * should be written or not. */ + final class WriteCB implements Callback, Boolean> { + @Override + public Deferred call(final Boolean allowed) throws Exception { + if (!allowed) { + rejected_dps.incrementAndGet(); + return Deferred.fromResult(null); + } + + Bytes.setInt(row, (int) base_time, metrics.width() + Const.SALT_WIDTH()); + RowKey.prefixKeyWithSalt(row); + + Deferred result = null; + if (!isHistogram(qualifier) && config.enable_appends()) { + if(config.use_otsdb_timestamp()) { + LOG.error("Cannot use Date Tiered Compaction with AppendPoints. Please turn off either of them."); + } + final AppendDataPoints kv = new AppendDataPoints(qualifier, value); + final AppendRequest point = new AppendRequest(table, row, FAMILY, + AppendDataPoints.APPEND_COLUMN_QUALIFIER, kv.getBytes()); + result = client.append(point); + } else if (!isHistogram(qualifier)) { + scheduleForCompaction(row, (int) base_time); + final PutRequest point = RequestBuilder.buildPutRequest(config, table, row, FAMILY, qualifier, value, timestamp); + result = client.put(point); + } else { + scheduleForCompaction(row, (int) base_time); + final PutRequest histo_point = new PutRequest(table, row, FAMILY, qualifier, value); + result = client.put(histo_point); + } + + // Count all added datapoints, not just those that came in through PUT rpc + // Will there be others? Well, something could call addPoint programatically right? + datapoints_added.incrementAndGet(); + + // TODO(tsuna): Add a callback to time the latency of HBase and store the + // timing in a moving Histogram (once we have a class for this). + + if (!config.enable_realtime_ts() && !config.enable_tsuid_incrementing() && + !config.enable_tsuid_tracking() && rt_publisher == null) { + return result; + } + + final byte[] tsuid = UniqueId.getTSUIDFromKey(row, METRICS_WIDTH, + Const.TIMESTAMP_BYTES); + + // if the meta cache plugin is instantiated then tracking goes through it + if (meta_cache != null) { + meta_cache.increment(tsuid); + } else { + if (config.enable_tsuid_tracking()) { + if (config.enable_realtime_ts()) { + if (config.enable_tsuid_incrementing()) { + TSMeta.incrementAndGetCounter(TSDB.this, tsuid); + } else { + TSMeta.storeIfNecessary(TSDB.this, tsuid); + } + } else { + final PutRequest tracking = new PutRequest(meta_table, tsuid, + TSMeta.FAMILY(), TSMeta.COUNTER_QUALIFIER(), Bytes.fromLong(1)); + client.put(tracking); + } + } + } + + if (rt_publisher != null) { + if (isHistogram(qualifier)) { + rt_publisher.publishHistogramPoint(metric, timestamp, value, tags, tsuid); + } else { + rt_publisher.sinkDataPoint(metric, timestamp, value, tags, tsuid, flags); + } + } + return result; + } + @Override + public String toString() { + return "addPointInternal Write Callback"; + } + } + + if (ts_filter != null && ts_filter.filterDataPoints()) { + if (isHistogram(qualifier)) { + return ts_filter.allowHistogramPoint(metric, timestamp, value, tags) + .addCallbackDeferring(new WriteCB()); + } else { + return ts_filter.allowDataPoint(metric, timestamp, value, tags, flags) + .addCallbackDeferring(new WriteCB()); + } + } + return Deferred.fromResult(true).addCallbackDeferring(new WriteCB()); + } + + private final void checkTimestampAndTags(final String metric, final long timestamp, + final byte[] value, + final Map tags, final short flags) { + // we only accept positive unix epoch timestamps in seconds or milliseconds + if (timestamp < 0 || ((timestamp & Const.SECOND_MASK) != 0 && + timestamp > 9999999999999L)) { + throw new IllegalArgumentException((timestamp < 0 ? "negative " : "bad") + + " timestamp=" + timestamp + + " when trying to add value=" + Arrays.toString(value) + '/' + flags + + " to metric=" + metric + ", tags=" + tags); + } + + IncomingDataPoints.checkMetricAndTags(metric, tags); + } + + /** + * Adds a rolled up and/or groupby/pre-agged data point to the proper table. + *

    + * WARNING: The tags map may be modified by this method without a lock. Give + * the method a copy if you plan to use it elsewhere. + *

    + * If {@code interval} is null then the value will be directed to the + * pre-agg table. + * If the {@code is_groupby} flag is set, then the aggregate tag, defined in + * "tsd.rollups.agg_tag", will be added or overwritten with the {@code aggregator} + * value in uppercase as the value. * @param metric A non-empty string. * @param timestamp The timestamp associated with the value. * @param value The value of the data point. * @param tags The tags on this series. This map must be non-empty. - * @return A deferred object that indicates the completion of the request. - * The {@link Object} has not special meaning and can be {@code null} (think - * of it as {@code Deferred}). But you probably want to attach at - * least an errback to this {@code Deferred} to handle failures. + * @param is_groupby Whether or not the value is a pre-aggregate + * @param interval The interval the data reflects (may be null) + * @param rollup_aggregator The aggregator used to generate the data + * @param groupby_aggregator = The aggregator used for pre-aggregated data. + * @return A deferred to optionally wait on to be sure the value was stored * @throws IllegalArgumentException if the timestamp is less than or equal * to the previous timestamp added or 0 for the first timestamp, or if the * difference with the previous timestamp is too large. @@ -554,158 +1506,280 @@ public WritableDataPoints newDataPoints() { * elements contains illegal characters. * @throws HBaseException (deferred) if there was a problem while persisting * data. + * @since 2.4 */ - public Deferred addPoint(final String metric, + public Deferred addAggregatePoint(final String metric, final long timestamp, final long value, - final Map tags) { - final byte[] v; - if (Byte.MIN_VALUE <= value && value <= Byte.MAX_VALUE) { - v = new byte[] { (byte) value }; - } else if (Short.MIN_VALUE <= value && value <= Short.MAX_VALUE) { - v = Bytes.fromShort((short) value); - } else if (Integer.MIN_VALUE <= value && value <= Integer.MAX_VALUE) { - v = Bytes.fromInt((int) value); - } else { - v = Bytes.fromLong(value); - } - final short flags = (short) (v.length - 1); // Just the length. - return addPointInternal(metric, timestamp, v, tags, flags); - } + final Map tags, + final boolean is_groupby, + final String interval, + final String rollup_aggregator, + final String groupby_aggregator) { + final byte[] val = Internal.vleEncodeLong(value); + final short flags = (short) (val.length - 1); // Just the length. + + return addAggregatePointInternal(metric, timestamp, + val, tags, flags, is_groupby, interval, rollup_aggregator, + groupby_aggregator); + } + /** - * Adds a double precision floating-point value data point in the TSDB. + * Adds a rolled up and/or groupby/pre-agged data point to the proper table. + *

    + * WARNING: The tags map may be modified by this method without a lock. Give + * the method a copy if you plan to use it elsewhere. + *

    + * If {@code interval} is null then the value will be directed to the + * pre-agg table. + * If the {@code is_groupby} flag is set, then the aggregate tag, defined in + * "tsd.rollups.agg_tag", will be added or overwritten with the {@code aggregator} + * value in uppercase as the value. * @param metric A non-empty string. * @param timestamp The timestamp associated with the value. * @param value The value of the data point. * @param tags The tags on this series. This map must be non-empty. - * @return A deferred object that indicates the completion of the request. - * The {@link Object} has not special meaning and can be {@code null} (think - * of it as {@code Deferred}). But you probably want to attach at - * least an errback to this {@code Deferred} to handle failures. + * @param is_groupby Whether or not the value is a pre-aggregate + * @param interval The interval the data reflects (may be null) + * @param rollup_aggregator The aggregator used to generate the data + * @param groupby_aggregator = The aggregator used for pre-aggregated data. + * @return A deferred to optionally wait on to be sure the value was stored * @throws IllegalArgumentException if the timestamp is less than or equal * to the previous timestamp added or 0 for the first timestamp, or if the * difference with the previous timestamp is too large. * @throws IllegalArgumentException if the metric name is empty or contains * illegal characters. - * @throws IllegalArgumentException if the value is NaN or infinite. * @throws IllegalArgumentException if the tags list is empty or one of the * elements contains illegal characters. * @throws HBaseException (deferred) if there was a problem while persisting * data. - * @since 1.2 + * @since 2.4 */ - public Deferred addPoint(final String metric, + public Deferred addAggregatePoint(final String metric, final long timestamp, - final double value, - final Map tags) { - if (Double.isNaN(value) || Double.isInfinite(value)) { + final float value, + final Map tags, + final boolean is_groupby, + final String interval, + final String rollup_aggregator, + final String groupby_aggregator) { + if (Float.isNaN(value) || Float.isInfinite(value)) { throw new IllegalArgumentException("value is NaN or Infinite: " + value - + " for metric=" + metric - + " timestamp=" + timestamp); + + " for metric=" + metric + + " timestamp=" + timestamp); } - final short flags = Const.FLAG_FLOAT | 0x7; // A float stored on 8 bytes. - return addPointInternal(metric, timestamp, - Bytes.fromLong(Double.doubleToRawLongBits(value)), - tags, flags); - } + final short flags = Const.FLAG_FLOAT | 0x3; // A float stored on 4 bytes. + + final byte[] val = Bytes.fromInt(Float.floatToRawIntBits(value)); + + return addAggregatePointInternal(metric, timestamp, + val, tags, flags, is_groupby, interval, rollup_aggregator, + groupby_aggregator); + } + /** - * Adds a single floating-point value data point in the TSDB. + * Adds a rolled up and/or groupby/pre-agged data point to the proper table. + * If {@code interval} is null then the value will be directed to the + * pre-agg table. + * If the {@code is_groupby} flag is set, then the aggregate tag, defined in + * "tsd.rollups.agg_tag", will be added or overwritten with the {@code aggregator} + * value in uppercase as the value. * @param metric A non-empty string. * @param timestamp The timestamp associated with the value. * @param value The value of the data point. * @param tags The tags on this series. This map must be non-empty. - * @return A deferred object that indicates the completion of the request. - * The {@link Object} has not special meaning and can be {@code null} (think - * of it as {@code Deferred}). But you probably want to attach at - * least an errback to this {@code Deferred} to handle failures. + * @param is_groupby Whether or not the value is a pre-aggregate + * @param interval The interval the data reflects (may be null) + * @param rollup_aggregator The aggregator used to generate the data + * @param groupby_aggregator = The aggregator used for pre-aggregated data. + * @return A deferred to optionally wait on to be sure the value was stored * @throws IllegalArgumentException if the timestamp is less than or equal * to the previous timestamp added or 0 for the first timestamp, or if the * difference with the previous timestamp is too large. * @throws IllegalArgumentException if the metric name is empty or contains * illegal characters. - * @throws IllegalArgumentException if the value is NaN or infinite. * @throws IllegalArgumentException if the tags list is empty or one of the * elements contains illegal characters. * @throws HBaseException (deferred) if there was a problem while persisting * data. + * @since 2.4 */ - public Deferred addPoint(final String metric, + public Deferred addAggregatePoint(final String metric, final long timestamp, - final float value, - final Map tags) { - if (Float.isNaN(value) || Float.isInfinite(value)) { + final double value, + final Map tags, + final boolean is_groupby, + final String interval, + final String rollup_aggregator, + final String groupby_aggregator) { + if (Double.isNaN(value) || Double.isInfinite(value)) { throw new IllegalArgumentException("value is NaN or Infinite: " + value - + " for metric=" + metric - + " timestamp=" + timestamp); + + " for metric=" + metric + + " timestamp=" + timestamp); } - final short flags = Const.FLAG_FLOAT | 0x3; // A float stored on 4 bytes. - return addPointInternal(metric, timestamp, - Bytes.fromInt(Float.floatToRawIntBits(value)), - tags, flags); - } - private Deferred addPointInternal(final String metric, - final long timestamp, - final byte[] value, - final Map tags, - final short flags) { - // we only accept positive unix epoch timestamps in seconds or milliseconds - if (timestamp < 0 || ((timestamp & Const.SECOND_MASK) != 0 && - timestamp > 9999999999999L)) { + final short flags = Const.FLAG_FLOAT | 0x7; // A float stored on 4 bytes. + + final byte[] val = Bytes.fromLong(Double.doubleToRawLongBits(value)); + + return addAggregatePointInternal(metric, timestamp, + val, tags, flags, is_groupby, interval, rollup_aggregator, + groupby_aggregator); + } + + Deferred addAggregatePointInternal(final String metric, + final long timestamp, + final byte[] value, + final Map tags, + final short flags, + final boolean is_groupby, + final String interval, + final String rollup_aggregator, + final String groupby_aggregator) { + + if (interval != null && !interval.isEmpty() && rollup_config == null) { + throw new IllegalArgumentException( + "No rollup or aggregations were configured"); + } + if (is_groupby && + (groupby_aggregator == null || groupby_aggregator.isEmpty())) { + throw new IllegalArgumentException("Cannot write a group by data point " + + "without specifying the aggregation function. Metric=" + metric + + " tags=" + tags); + } + + // we only accept positive unix epoch timestamps in seconds for rollups + // and allow milliseconds for pre-aggregates + if (timestamp < 0 || ((timestamp & Const.SECOND_MASK) != 0)) { throw new IllegalArgumentException((timestamp < 0 ? "negative " : "bad") - + " timestamp=" + timestamp - + " when trying to add value=" + Arrays.toString(value) + '/' + flags - + " to metric=" + metric + ", tags=" + tags); + + " timestamp=" + timestamp + + " when trying to add value=" + Arrays.toString(value) + '/' + flags + + " to metric=" + metric + ", tags=" + tags); } - IncomingDataPoints.checkMetricAndTags(metric, tags); - final byte[] row = IncomingDataPoints.rowKeyTemplate(this, metric, tags); - final long base_time; - final byte[] qualifier = Internal.buildQualifier(timestamp, flags); - if ((timestamp & Const.SECOND_MASK) != 0) { - // drop the ms timestamp to seconds to calculate the base timestamp - base_time = ((timestamp / 1000) - - ((timestamp / 1000) % Const.MAX_TIMESPAN)); + String agg_tag_value = tags.get(agg_tag_key); + if (agg_tag_value == null) { + if (!is_groupby) { + // it's a rollup on "raw" data. + if (tag_raw_data) { + tags.put(agg_tag_key, raw_agg_tag_value); + } + agg_tag_value = raw_agg_tag_value; + } else { + // pre-agged so use the aggregator as the tag + agg_tag_value = groupby_aggregator.toUpperCase(); + tags.put(agg_tag_key, agg_tag_value); + } } else { - base_time = (timestamp - (timestamp % Const.MAX_TIMESPAN)); + // sanity check + if (!agg_tag_value.equalsIgnoreCase(groupby_aggregator)) { + throw new IllegalArgumentException("Given tag value for " + agg_tag_key + + " of " + agg_tag_value + " did not match the group by " + + "aggregator of " + groupby_aggregator + " for " + metric + + " " + tags); + } + // force upper case + agg_tag_value = groupby_aggregator.toUpperCase(); + tags.put(agg_tag_key, agg_tag_value); } - Bytes.setInt(row, (int) base_time, metrics.width()); - scheduleForCompaction(row, (int) base_time); - final PutRequest point = new PutRequest(table, row, FAMILY, qualifier, value); - - // TODO(tsuna): Add a callback to time the latency of HBase and store the - // timing in a moving Histogram (once we have a class for this). - Deferred result = client.put(point); - if (!config.enable_realtime_ts() && !config.enable_tsuid_incrementing() && - !config.enable_tsuid_tracking() && rt_publisher == null) { - return result; + if (is_groupby) { + try { + Aggregators.get(groupby_aggregator.toLowerCase()); + } catch (NoSuchElementException e) { + throw new IllegalArgumentException("Invalid group by aggregator " + + groupby_aggregator + " with metric " + metric + " " + tags); + } + if (rollups_block_derived && + // TODO - create a better list of aggs to block + (agg_tag_value.equals("AVG") || + agg_tag_value.equals("DEV"))) { + throw new IllegalArgumentException("Derived group by aggregations " + + "are not allowed " + groupby_aggregator + " with metric " + + metric + " " + tags); + } } - final byte[] tsuid = UniqueId.getTSUIDFromKey(row, METRICS_WIDTH, - Const.TIMESTAMP_BYTES); + IncomingDataPoints.checkMetricAndTags(metric, tags); - // for busy TSDs we may only enable TSUID tracking, storing a 1 in the - // counter field for a TSUID with the proper timestamp. If the user would - // rather have TSUID incrementing enabled, that will trump the PUT - if (config.enable_tsuid_tracking() && !config.enable_tsuid_incrementing()) { - final PutRequest tracking = new PutRequest(meta_table, tsuid, - TSMeta.FAMILY(), TSMeta.COUNTER_QUALIFIER(), Bytes.fromLong(1)); - client.put(tracking); - } else if (config.enable_tsuid_incrementing() || config.enable_realtime_ts()) { - TSMeta.incrementAndGetCounter(TSDB.this, tsuid); + final RollupInterval rollup_interval = (interval == null || interval.isEmpty() + ? null : rollup_config.getRollupInterval(interval)); + final int aggregator_id = rollup_interval == null ? -1 : + rollup_config.getIdForAggregator(rollup_aggregator); + final byte[] row = IncomingDataPoints.rowKeyTemplate(this, metric, tags); + final String rollup_agg = rollup_aggregator != null ? + rollup_aggregator.toUpperCase() : null; + if (rollup_agg!= null && rollups_block_derived && + // TODO - create a better list of aggs to block + (rollup_agg.equals("AVG") || + rollup_agg.equals("DEV"))) { + throw new IllegalArgumentException("Derived rollup aggregations " + + "are not allowed " + rollup_agg + " with metric " + + metric + " " + tags); } + final int base_time = interval == null || interval.isEmpty() ? + (int)(timestamp - (timestamp % Const.MAX_TIMESPAN)) + : RollupUtils.getRollupBasetime(timestamp, rollup_interval); + final byte[] qualifier = interval == null || interval.isEmpty() ? + Internal.buildQualifier(timestamp, flags) + : RollupUtils.buildRollupQualifier( + timestamp, base_time, flags, aggregator_id, rollup_interval); - if (rt_publisher != null) { - rt_publisher.sinkDataPoint(metric, timestamp, value, tags, tsuid, flags); + /** Callback executed for chaining filter calls to see if the value + * should be written or not. */ + final class WriteCB implements Callback, Boolean> { + @Override + public Deferred call(final Boolean allowed) throws Exception { + if (!allowed) { + rejected_aggregate_dps.incrementAndGet(); + return Deferred.fromResult(null); + } + Internal.setBaseTime(row, base_time); + // NOTE: Do not modify the row key after calculating and applying the salt + RowKey.prefixKeyWithSalt(row); + + Deferred result; + + final PutRequest point; + if (interval == null || interval.isEmpty()) { + if (!is_groupby) { + throw new IllegalArgumentException("Interval cannot be null " + + "for a non-group by point"); + } + point = new PutRequest(default_interval.getGroupbyTable(), row, + FAMILY, qualifier, value); + } else { + point = new PutRequest( + is_groupby ? rollup_interval.getGroupbyTable() : + rollup_interval.getTemporalTable(), + row, FAMILY, qualifier, value); + } + + // TODO: Add a callback to time the latency of HBase and store the + // timing in a moving Histogram (once we have a class for this). + result = client.put(point); + + // TODO - figure out what we want to do with the real time publisher and + // the meta tracking. + return result; + } + } + + if (ts_filter != null) { + return ts_filter.allowDataPoint(metric, timestamp, value, tags, flags) + .addCallbackDeferring(new WriteCB()); + } + try { + return new WriteCB().call(true); + } catch (Exception e) { + return Deferred.fromError(e); } - return result; } - + /** - * Forces a flush of any un-committed in memory data including left over + * Forces a flush of any un-committed in memory data including left over * compactions. *

    * For instance, any data point not persisted will be sent to HBase. @@ -719,10 +1793,10 @@ private Deferred addPointInternal(final String metric, */ public Deferred flush() throws HBaseException { final class HClientFlush implements Callback> { - public Object call(final ArrayList args) { + public Object call(final ArrayList args) { return client.flush(); } - public String toString() { + public String toString() { return "flush HBase client"; } } @@ -747,18 +1821,76 @@ public String toString() { * recoverable by retrying, some are not. */ public Deferred shutdown() { - final ArrayList> deferreds = + final ArrayList> deferreds = new ArrayList>(); - - final class HClientShutdown implements Callback> { - public Object call(final ArrayList args) { - return client.shutdown(); + + final class FinalShutdown implements Callback { + @Override + public Object call(Object result) throws Exception { + if (result instanceof Exception) { + LOG.error("A previous shutdown failed", (Exception)result); + } + final Set timeouts = timer.stop(); + // TODO - at some point we should clean these up. + if (timeouts.size() > 0) { + LOG.warn("There were " + timeouts.size() + " timer tasks queued"); + } + LOG.info("Completed shutting down the TSDB"); + return Deferred.fromResult(null); + } + } + + final class SEHShutdown implements Callback { + @Override + public Object call(Object result) throws Exception { + if (result instanceof Exception) { + LOG.error("Shutdown of the HBase client failed", (Exception)result); + } + LOG.info("Shutting down storage exception handler plugin: " + + storage_exception_handler.getClass().getCanonicalName()); + return storage_exception_handler.shutdown().addBoth(new FinalShutdown()); + } + @Override + public String toString() { + return "SEHShutdown"; + } + } + + final class RpcResponsderShutdown implements Callback { + @Override + public Object call(Object arg) throws Exception { + try { + TSDB.this.rpcResponder.close(); + } catch (Exception e) { + LOG.error( + "Run into unknown exception while closing RpcResponder.", e); + } finally { + return arg; + } + } + } + + final class HClientShutdown implements Callback, ArrayList> { + public Deferred call(final ArrayList args) { + Callback nextCallback; + if (storage_exception_handler != null) { + nextCallback = new SEHShutdown(); + } else { + nextCallback = new FinalShutdown(); + } + + if (TSDB.this.rpcResponder.isAsync()) { + client.shutdown().addBoth(new RpcResponsderShutdown()); + } + + return client.shutdown().addBoth(nextCallback); } + public String toString() { return "shutdown HBase client"; } } - + final class ShutdownErrback implements Callback { public Object call(final Exception e) { final Logger LOG = LoggerFactory.getLogger(ShutdownErrback.class); @@ -772,47 +1904,70 @@ public Object call(final Exception e) { } else { LOG.error("Failed to shutdown the TSD", e); } - return client.shutdown(); + return new HClientShutdown().call(null); } + public String toString() { return "shutdown HBase client after error"; } } - + final class CompactCB implements Callback> { public Object call(ArrayList compactions) throws Exception { return null; } } - + if (config.enable_compactions()) { LOG.info("Flushing compaction queue"); deferreds.add(compactionq.flush().addCallback(new CompactCB())); } + if (startup != null) { + LOG.info("Shutting down startup plugin: " + + startup.getClass().getCanonicalName()); + deferreds.add(startup.shutdown()); + } + if (authentication != null) { + LOG.info("Shutting down authentication plugin: " + + authentication.getClass().getCanonicalName()); + deferreds.add(authentication.shutdown()); + } if (search != null) { - LOG.info("Shutting down search plugin: " + + LOG.info("Shutting down search plugin: " + search.getClass().getCanonicalName()); deferreds.add(search.shutdown()); } if (rt_publisher != null) { - LOG.info("Shutting down RT plugin: " + + LOG.info("Shutting down RT plugin: " + rt_publisher.getClass().getCanonicalName()); deferreds.add(rt_publisher.shutdown()); } - - if (rpc_plugins != null && !rpc_plugins.isEmpty()) { - for (final RpcPlugin rpc : rpc_plugins) { - LOG.info("Shutting down RPC plugin: " + - rpc.getClass().getCanonicalName()); - deferreds.add(rpc.shutdown()); - } + if (meta_cache != null) { + LOG.info("Shutting down meta cache plugin: " + + meta_cache.getClass().getCanonicalName()); + deferreds.add(meta_cache.shutdown()); } - + if (storage_exception_handler != null) { + LOG.info("Shutting down storage exception handler plugin: " + + storage_exception_handler.getClass().getCanonicalName()); + deferreds.add(storage_exception_handler.shutdown()); + } + if (ts_filter != null) { + LOG.info("Shutting down time series filter plugin: " + + ts_filter.getClass().getCanonicalName()); + deferreds.add(ts_filter.shutdown()); + } + if (uid_filter != null) { + LOG.info("Shutting down UID filter plugin: " + + uid_filter.getClass().getCanonicalName()); + deferreds.add(uid_filter.shutdown()); + } + // wait for plugins to shutdown before we close the client return deferreds.size() > 0 - ? Deferred.group(deferreds).addCallbacks(new HClientShutdown(), - new ShutdownErrback()) - : client.shutdown(); + ? Deferred.group(deferreds).addCallbackDeferring(new HClientShutdown()) + .addErrback(new ShutdownErrback()) + : new HClientShutdown().call(null); } /** @@ -822,14 +1977,14 @@ public Object call(ArrayList compactions) throws Exception { public List suggestMetrics(final String search) { return metrics.suggest(search); } - + /** * Given a prefix search, returns matching metric names. * @param search A prefix to search. * @param max_results Maximum number of results to return. * @since 2.0 */ - public List suggestMetrics(final String search, + public List suggestMetrics(final String search, final int max_results) { return metrics.suggest(search, max_results); } @@ -841,14 +1996,14 @@ public List suggestMetrics(final String search, public List suggestTagNames(final String search) { return tag_names.suggest(search); } - + /** * Given a prefix search, returns matching tagk names. * @param search A prefix to search. * @param max_results Maximum number of results to return. * @since 2.0 */ - public List suggestTagNames(final String search, + public List suggestTagNames(final String search, final int max_results) { return tag_names.suggest(search, max_results); } @@ -860,14 +2015,14 @@ public List suggestTagNames(final String search, public List suggestTagValues(final String search) { return tag_values.suggest(search); } - + /** * Given a prefix search, returns matching tag values. * @param search A prefix to search. * @param max_results Maximum number of results to return. * @since 2.0 */ - public List suggestTagValues(final String search, + public List suggestTagValues(final String search, final int max_results) { return tag_values.suggest(search, max_results); } @@ -884,14 +2039,14 @@ public void dropCaches() { /** * Attempts to assign a UID to a name for the given type - * Used by the UniqueIdRpc call to generate IDs for new metrics, tagks or + * Used by the UniqueIdRpc call to generate IDs for new metrics, tagks or * tagvs. The name must pass validation and if it's already assigned a UID, * this method will throw an error with the proper UID. Otherwise if it can * create the UID, it will be returned * @param type The type of uid to assign, metric, tagk or tagv * @param name The name of the uid object * @return A byte array with the UID if the assignment was successful - * @throws IllegalArgumentException if the name is invalid or it already + * @throws IllegalArgumentException if the name is invalid or it already * exists * @since 2.0 */ @@ -926,22 +2081,91 @@ public byte[] assignUid(final String type, final String name) { throw new IllegalArgumentException("Unknown type name"); } } - + + /** + * Attempts to delete the given UID name mapping from the storage table as + * well as the local cache. + * @param type The type of UID to delete. Must be "metrics", "tagk" or "tagv" + * @param name The name of the UID to delete + * @return A deferred to wait on for completion, or an exception if thrown + * @throws IllegalArgumentException if the type is invalid + * @since 2.2 + */ + public Deferred deleteUidAsync(final String type, final String name) { + final UniqueIdType uid_type = UniqueId.stringToUniqueIdType(type); + switch (uid_type) { + case METRIC: + return metrics.deleteAsync(name); + case TAGK: + return tag_names.deleteAsync(name); + case TAGV: + return tag_values.deleteAsync(name); + default: + throw new IllegalArgumentException("Unrecognized UID type: " + uid_type); + } + } + + /** + * Attempts to rename a UID from existing name to the given name + * Used by the UniqueIdRpc call to rename name of existing metrics, tagks or + * tagvs. The name must pass validation. If the UID doesn't exist, the method + * will throw an error. Chained IllegalArgumentException is directly exposed + * to caller. If the rename was successful, this method returns. + * @param type The type of uid to rename, one of metric, tagk and tagv + * @param oldname The existing name of the uid object + * @param newname The new name to be used on the uid object + * @throws IllegalArgumentException if error happened + * @since 2.2 + */ + public void renameUid(final String type, final String oldname, + final String newname) { + Tags.validateString(type, oldname); + Tags.validateString(type, newname); + if (type.toLowerCase().equals("metric")) { + try { + this.metrics.getId(oldname); + this.metrics.rename(oldname, newname); + } catch (NoSuchUniqueName nsue) { + throw new IllegalArgumentException("Name(\"" + oldname + + "\") does not exist"); + } + } else if (type.toLowerCase().equals("tagk")) { + try { + this.tag_names.getId(oldname); + this.tag_names.rename(oldname, newname); + } catch (NoSuchUniqueName nsue) { + throw new IllegalArgumentException("Name(\"" + oldname + + "\") does not exist"); + } + } else if (type.toLowerCase().equals("tagv")) { + try { + this.tag_values.getId(oldname); + this.tag_values.rename(oldname, newname); + } catch (NoSuchUniqueName nsue) { + throw new IllegalArgumentException("Name(\"" + oldname + + "\") does not exist"); + } + } else { + LOG.warn("Unknown type name: " + type); + throw new IllegalArgumentException("Unknown type name"); + } + } + /** @return the name of the UID table as a byte array for client requests */ public byte[] uidTable() { return this.uidtable; } - + /** @return the name of the data table as a byte array for client requests */ public byte[] dataTable() { return this.table; } - + /** @return the name of the tree table as a byte array for client requests */ public byte[] treeTable() { return this.treetable; } - + /** @return the name of the meta table as a byte array for client requests */ public byte[] metaTable() { return this.meta_table; @@ -957,7 +2181,7 @@ public void indexTSMeta(final TSMeta meta) { search.indexTSMeta(meta).addErrback(new PluginError()); } } - + /** * Delete the timeseries meta object from the search index * @param tsuid The TSUID to delete @@ -968,7 +2192,7 @@ public void deleteTSMeta(final String tsuid) { search.deleteTSMeta(tsuid).addErrback(new PluginError()); } } - + /** * Index the given UID meta object via the configured search plugin * @param meta The meta data object to index @@ -979,7 +2203,7 @@ public void indexUIDMeta(final UIDMeta meta) { search.indexUIDMeta(meta).addErrback(new PluginError()); } } - + /** * Delete the UID meta object from the search index * @param meta The UID meta object to delete @@ -990,7 +2214,7 @@ public void deleteUIDMeta(final UIDMeta meta) { search.deleteUIDMeta(meta).addErrback(new PluginError()); } } - + /** * Index the given Annotation object via the configured search plugin * @param note The annotation object to index @@ -1004,7 +2228,7 @@ public void indexAnnotation(final Annotation note) { rt_publisher.publishAnnotation(note); } } - + /** * Delete the annotation object from the search index * @param note The annotation object to delete @@ -1015,7 +2239,7 @@ public void deleteAnnotation(final Annotation note) { search.deleteAnnotation(note).addErrback(new PluginError()); } } - + /** * Processes the TSMeta through all of the trees if configured to do so * @param meta The meta data to process @@ -1027,7 +2251,7 @@ public Deferred processTSMetaThroughTrees(final TSMeta meta) { } return Deferred.fromResult(false); } - + /** * Executes a search query using the search plugin * @param query The query to execute @@ -1041,13 +2265,13 @@ public Deferred executeSearch(final SearchQuery query) { throw new IllegalStateException( "Searching has not been enabled on this TSD"); } - + return search.executeQuery(query); } - + /** - * Simply logs plugin errors when they're thrown by attaching as an errorback. - * Without this, exceptions will just disappear (unless logged by the plugin) + * Simply logs plugin errors when they're thrown by attaching as an errorback. + * Without this, exceptions will just disappear (unless logged by the plugin) * since we don't wait for a result. */ final class PluginError implements Callback { @@ -1057,14 +2281,110 @@ public Object call(final Exception e) throws Exception { return null; } } + + /** @return the rollup config object. May be null + * @since 2.4 */ + public RollupConfig getRollupConfig() { + return rollup_config; + } + + /** @return The default rollup interval config. May be null. + * @since 2.4 */ + public RollupInterval getDefaultInterval() { + return default_interval; + } + + /** + * Blocks while pre-fetching meta data from the data and uid tables + * so that performance improves, particularly with a large number of + * regions and region servers. + * @since 2.2 + */ + public void preFetchHBaseMeta() { + LOG.info("Pre-fetching meta data for all tables"); + final long start = System.currentTimeMillis(); + final ArrayList> deferreds = new ArrayList>(); + deferreds.add(client.prefetchMeta(table)); + deferreds.add(client.prefetchMeta(uidtable)); + + // TODO(cl) - meta, tree, etc + + try { + Deferred.group(deferreds).join(); + LOG.info("Fetched meta data for tables in " + + (System.currentTimeMillis() - start) + "ms"); + } catch (InterruptedException e) { + LOG.error("Interrupted", e); + Thread.currentThread().interrupt(); + return; + } catch (Exception e) { + LOG.error("Failed to prefetch meta for our tables", e); + } + } + + /** @return the timer used for various house keeping functions */ + public Timer getTimer() { + return timer; + } + + /** @return The aggregate tag key if set. May be null. + * @since 2.4 */ + public String getAggTagKey() { + return agg_tag_key; + } + + /** @return The raw tag value if set. May be null. + * @since 2.4 */ + public String getRawTagValue() { + return raw_agg_tag_value; + } + + /** + * Returns whether the global config setting allows splitting rollups queries + * if the rollups table to be hit is lagging. + * + * @return Whether or not splitting rollup queries is enabled + * @since 2.4 + */ + public boolean isRollupsSplittingEnabled() { + return rollups_split_queries; + } + + /** @return The optional histogram manager registered to this TSD. + * @since 2.4 */ + public HistogramCodecManager histogramManager() { + return histogram_manager; + } + + /** @return The search plugin if configured and loaded. May be null. + * @since 2.4 */ + public SearchPlugin getSearchPlugin() { + return this.search; + } + + /** @return The byte limit class for queries */ + public QueryLimitOverride getQueryByteLimits() { + return query_limits; + } + + /** @return The mode of operation for this TSD. + * @since 2.4 */ + public OperationMode getMode() { + return mode; + } + + private final boolean isHistogram(final byte[] qualifier) { + return (qualifier.length & 0x1) == 1; + } // ------------------ // // Compaction helpers // // ------------------ // - final KeyValue compact(final ArrayList row, - List annotations) { - return compactionq.compact(row, annotations); + final KeyValue compact(final ArrayList row, + List annotations, + List histograms) { + return compactionq.compact(row, annotations, histograms); } /** @@ -1087,14 +2407,15 @@ final void scheduleForCompaction(final byte[] row, final int base_time) { /** Gets the entire given row from the data table. */ final Deferred> get(final byte[] key) { - return client.get(new GetRequest(table, key)); + return client.get(new GetRequest(table, key, FAMILY)); } /** Puts the given value into the data table. */ final Deferred put(final byte[] key, final byte[] qualifier, - final byte[] value) { - return client.put(new PutRequest(table, key, FAMILY, qualifier, value)); + final byte[] value, + long timestamp) { + return client.put(RequestBuilder.buildPutRequest(config, table, key, FAMILY, qualifier, value, timestamp)); } /** Deletes the given cells from the data table. */ @@ -1102,4 +2423,9 @@ final Deferred delete(final byte[] key, final byte[][] qualifiers) { return client.delete(new DeleteRequest(table, key, FAMILY, qualifiers)); } + /** Do response by RpcResponder */ + public void response(Runnable run) { + rpcResponder.response(run); + } + } diff --git a/src/core/TSQuery.java b/src/core/TSQuery.java index 9ba5248ae0..9e61ffeedf 100644 --- a/src/core/TSQuery.java +++ b/src/core/TSQuery.java @@ -16,7 +16,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.TimeZone; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.google.common.base.Objects; +import com.stumbleupon.async.Callback; +import com.stumbleupon.async.Deferred; + +import net.opentsdb.stats.QueryStats; import net.opentsdb.utils.DateTime; /** @@ -32,6 +40,7 @@ * {@code start_time} and {@code end_time} fields. * @since 2.0 */ +@JsonIgnoreProperties(ignoreUnknown = true) public final class TSQuery { /** User given start date/time, could be relative or absolute */ @@ -62,7 +71,7 @@ public final class TSQuery { private boolean show_tsuids; /** A list of parsed sub queries, must have one or more to fetch data */ - private ArrayList queries; + private List queries; /** The parsed start time value * Do not set directly */ @@ -75,6 +84,30 @@ public final class TSQuery { /** Whether or not the user wasn't millisecond resolution */ private boolean ms_resolution; + /** Whether or not to show the sub query with the results */ + private boolean show_query; + + /** Whether or not to include stats in the output */ + private boolean show_stats; + + /** Whether or not to include stats summary in the output */ + private boolean show_summary; + + /** Whether or not to delete the queried data */ + private boolean delete = false; + + /** A flag denoting whether or not to align intervals based on the calendar */ + private boolean use_calendar; + + /** The query status for tracking over all performance of this query */ + private QueryStats query_stats; + + /** Override default max byte limit */ + private boolean override_byte_limit; + + /** Override default max row count limit */ + private boolean override_data_point_limit; + /** * Default constructor necessary for POJO de/serialization */ @@ -82,6 +115,45 @@ public TSQuery() { } + @Override + public int hashCode() { + // NOTE: Do not add any non-user submitted variables to the hash. We don't + // want the hash to change after validation. + // We also don't care about stats or summary + return Objects.hashCode(start, end, timezone, use_calendar, options, padding, + no_annotations, with_global_annotations, show_tsuids, queries, + ms_resolution); + } + + @Override + public boolean equals(final Object obj) { + if (obj == null) { + return false; + } + if (!(obj instanceof TSQuery)) { + return false; + } + if (obj == this) { + return true; + } + + // NOTE: Do not add any non-user submitted variables to the comparator. We + // don't want the value to change after validation. + // We also don't care about stats or summary + final TSQuery query = (TSQuery)obj; + return Objects.equal(start, query.start) + && Objects.equal(end, query.end) + && Objects.equal(timezone, query.timezone) + && Objects.equal(use_calendar,query.use_calendar) + && Objects.equal(options, query.options) + && Objects.equal(padding, query.padding) + && Objects.equal(no_annotations, query.no_annotations) + && Objects.equal(with_global_annotations, query.with_global_annotations) + && Objects.equal(show_tsuids, query.show_tsuids) + && Objects.equal(queries, query.queries) + && Objects.equal(ms_resolution, query.ms_resolution); + } + /** * Runs through query parameters to make sure it's a valid request. * This includes parsing relative timestamps, verifying that the end time is @@ -104,9 +176,9 @@ public void validateAndSetQuery() { } else { end_time = System.currentTimeMillis(); } - if (end_time <= start_time) { + if (end_time < start_time) { throw new IllegalArgumentException( - "End time [" + end_time + "] must be greater than the start time [" + "End time [" + end_time + "] must be greater than or equal to the start time [" + start_time +"]"); } @@ -115,8 +187,25 @@ public void validateAndSetQuery() { } // validate queries + int i = 0; for (TSSubQuery sub : queries) { sub.validateAndSetQuery(); + final DownsamplingSpecification ds = sub.downsamplingSpecification(); + if (ds != null && timezone != null && !timezone.isEmpty() && + ds != DownsamplingSpecification.NO_DOWNSAMPLER) { + final TimeZone tz = DateTime.timezones.get(timezone); + if (tz == null) { + throw new IllegalArgumentException( + "The timezone specification could not be found"); + } + ds.setTimezone(tz); + } + if (ds != null && use_calendar && + ds != DownsamplingSpecification.NO_DOWNSAMPLER) { + ds.setUseCalendar(true); + } + + sub.setIndex(i++); } } @@ -129,37 +218,52 @@ public void validateAndSetQuery() { * @return An array of queries */ public Query[] buildQueries(final TSDB tsdb) { - final Query[] queries = new Query[this.queries.size()]; - int i = 0; - for (TSSubQuery sub : this.queries) { - final Query query = tsdb.newQuery(); - query.setStartTime(start_time); - query.setEndTime(end_time); - if (sub.downsampler() != null) { - query.downsample(sub.downsampleInterval(), sub.downsampler()); - } else if (!ms_resolution) { - // we *may* have multiple millisecond data points in the set so we have - // to downsample. use the sub query's aggregator - query.downsample(1000, sub.aggregator()); + try { + return buildQueriesAsync(tsdb).joinUninterruptibly(); + } catch (final Exception e) { + throw new RuntimeException("Unexpected exception", e); + } + } + + /** + * Compiles the TSQuery into an array of Query objects for execution. + * If the user has not set a down sampler explicitly, and they don't want + * millisecond resolution, then we set the down sampler to 1 second to handle + * situations where storage may have multiple data points per second. + * @param tsdb The tsdb to use for {@link TSDB#newQuery} + * @return A deferred array of queries to wait on for compilation. + * @since 2.2 + */ + public Deferred buildQueriesAsync(final TSDB tsdb) { + final Query[] tsdb_queries = new Query[queries.size()]; + + final List> deferreds = + new ArrayList>(queries.size()); + for (int i = 0; i < queries.size(); i++) { + Query query = tsdb.newQuery(); + Deferred resolution = query.configureFromQuery(this, i); + + if (query.needsSplitting() && (query instanceof TsdbQuery)) { + query = new SplitRollupQuery(tsdb, (TsdbQuery) query, resolution); + resolution = query.configureFromQuery(this, i); } - if (sub.getTsuids() != null && !sub.getTsuids().isEmpty()) { - if (sub.getRateOptions() != null) { - query.setTimeSeries(sub.getTsuids(), sub.aggregator(), sub.getRate(), - sub.getRateOptions()); - } else { - query.setTimeSeries(sub.getTsuids(), sub.aggregator(), sub.getRate()); - } - } else if (sub.getRateOptions() != null) { - query.setTimeSeries(sub.getMetric(), sub.getTags(), sub.aggregator(), - sub.getRate(), sub.getRateOptions()); - } else { - query.setTimeSeries(sub.getMetric(), sub.getTags(), sub.aggregator(), - sub.getRate()); + deferreds.add(resolution); + + tsdb_queries[i] = query; + } + + class GroupFinished implements Callback> { + @Override + public Query[] call(final ArrayList deferreds) { + return tsdb_queries; + } + @Override + public String toString() { + return "Query compile group callback"; } - queries[i] = query; - i++; } - return queries; + + return Deferred.group(deferreds).addCallback(new GroupFinished()); } public String toString() { @@ -271,6 +375,39 @@ public boolean getMsResolution() { return ms_resolution; } + /** @return whether or not to show the query with the results */ + public boolean getShowQuery() { + return show_query; + } + + /** @return whether or not to return stats per query */ + public boolean getShowStats() { + return show_stats; + } + + /** @return Whether or not to show the query summary */ + public boolean getShowSummary() { + return this.show_summary; + } + + /** @return Whether or not to delete the queried data @since 2.2 */ + public boolean getDelete() { + return this.delete; + } + + /** @return the flag denoting whether intervals should be aligned based on + * the calendar + * @since 2.3 */ + public boolean getUseCalendar() { + return use_calendar; + } + + /** @return the query stats object. Ignored during JSON serialization */ + @JsonIgnore + public QueryStats getQueryStats() { + return query_stats; + } + /** * Sets the start time for further parsing. This can be an absolute or * relative value. See {@link DateTime#parseDateTimeString} for details. @@ -321,7 +458,7 @@ public void setShowTSUIDs(boolean show_tsuids) { } /** @param queries a list of {@link TSSubQuery} objects to store*/ - public void setQueries(ArrayList queries) { + public void setQueries(final List queries) { this.queries = queries; } @@ -329,4 +466,58 @@ public void setQueries(ArrayList queries) { public void setMsResolution(boolean ms_resolution) { this.ms_resolution = ms_resolution; } + + /** @param show_query whether or not to show the query with the serialization */ + public void setShowQuery(boolean show_query) { + this.show_query = show_query; + } + + /** @param show_stats whether or not to show stats in the serialization */ + public void setShowStats(boolean show_stats) { + this.show_stats = show_stats; + } + + /** @param show_summary whether or not to show the query summary */ + public void setShowSummary(boolean show_summary) { + this.show_summary = show_summary; + } + + /** @param delete whether or not to delete the queried data @since 2.2 */ + public void setDelete(boolean delete) { + this.delete = delete; + } + + /** @param use_calendar a flag denoting whether or not to align intervals + * based on the calendar @since 2.3 */ + public void setUseCalendar(boolean use_calendar) { + this.use_calendar = use_calendar; + } + + /** @param query_stats the query stats object to associate with this query */ + public void setQueryStats(final QueryStats query_stats) { + this.query_stats = query_stats; + } + + /** @return Whether or not the query would like to override the byte limiter. */ + public boolean overrideByteLimit() { + return override_byte_limit; + } + + /** @param override_byte_limit Whether or not the query would like to override + * the byte limiter. */ + public void setOverrideByteLimit(boolean override_byte_limit) { + this.override_byte_limit = override_byte_limit; + } + + /** @return Whether or not the query would like to override the data point limit. */ + public boolean overrideDataPointLimit() { + return override_data_point_limit; + } + + /** @param override_data_point_limit Whether or not the query would like to + * override the data point limit. */ + public void setOverrideDataPointLimit(boolean override_data_point_limit) { + this.override_data_point_limit = override_data_point_limit; + } + } diff --git a/src/core/TSSubQuery.java b/src/core/TSSubQuery.java index de28b45cec..1803364142 100644 --- a/src/core/TSSubQuery.java +++ b/src/core/TSSubQuery.java @@ -12,13 +12,20 @@ // see . package net.opentsdb.core; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; -import net.opentsdb.utils.DateTime; +import net.opentsdb.query.filter.TagVFilter; +import net.opentsdb.utils.ByteSet; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.google.common.base.Objects; +import com.google.common.collect.ImmutableMap; /** * Represents the parameters for an individual sub query on a metric or specific @@ -34,9 +41,10 @@ * the {@link TSQuery} object will call this for you when the entire set of * queries has been compiled. * Note: If using POJO deserialization, make sure to avoid setting the - * {@code agg}, {@code downsampler} and {@code downsample_interval} fields. + * {@code agg} and {@code downsample_specifier} fields. * @since 2.0 */ +@JsonIgnoreProperties(ignoreUnknown = true) public final class TSSubQuery { /** User given name of an aggregation function to use */ private String aggregator; @@ -46,11 +54,7 @@ public final class TSSubQuery { /** User provided list of timeseries UIDs */ private List tsuids; - - /** User supplied list of tags for specificity or grouping. May be null or - * empty */ - private HashMap tags; - + /** User given downsampler */ private String downsample; @@ -63,34 +67,104 @@ public final class TSSubQuery { /** Parsed aggregation function */ private Aggregator agg; - /** Parsed downsampler function */ - private Aggregator downsampler; + /** Parsed downsampling specification. */ + private DownsamplingSpecification downsample_specifier; + + /** Search the query on pre-aggregated table directly instead of post fetch + * aggregation. */ + private boolean pre_aggregate; + + /** Do not use rollup tables for down sampling */ + private TsdbQuery.ROLLUP_USAGE rollup_usage; + + /** Pointer to the related TSDB Query */ + @JsonIgnore + private TsdbQuery tsdb_query; + + /** A list of filters for this query. For now these are pulled out of the + * tags map. In the future we'll have special JSON objects for them. */ + private List filters; + + /** Whether or not to match series with ONLY the given tags */ + private boolean explicit_tags; + + /** Whether or not to enable fuzzy scanning if explicit tags is set */ + private boolean use_fuzzy_filter; + + /** List of percentiles if fetching histogram data */ + private List percentiles; + + /** Whether or not to return the raw histogram buckets for a histo query. */ + private boolean show_histogram_buckets; + + /** Whether or not to override multi-gets for explicit tag queries */ + private boolean use_multi_gets; - /** Parsed downsample interval */ - private long downsample_interval; + /** Index of the sub query */ + private int index; /** * Default constructor necessary for POJO de/serialization */ public TSSubQuery() { + // Assume no downsampling until told otherwise. + downsample_specifier = DownsamplingSpecification.NO_DOWNSAMPLER; + use_fuzzy_filter = true; + use_multi_gets = true; + } + + @Override + public int hashCode() { + // NOTE: Do not add any non-user submitted variables to the hash. We don't + // want the hash to change after validation. + return Objects.hashCode(aggregator, metric, tsuids, downsample, rate, + rate_options, filters, explicit_tags); + } + + @Override + public boolean equals(final Object obj) { + if (obj == null) { + return false; + } + if (!(obj instanceof TSSubQuery)) { + return false; + } + if (obj == this) { + return true; + } + // NOTE: Do not add any non-user submitted variables to the comparator. We + // don't want the value to change after validation. + final TSSubQuery query = (TSSubQuery)obj; + return Objects.equal(aggregator, query.aggregator) + && Objects.equal(metric, query.metric) + && Objects.equal(tsuids, query.tsuids) + && Objects.equal(downsample, query.downsample) + && Objects.equal(rate, query.rate) + && Objects.equal(rate_options, query.rate_options) + && Objects.equal(filters, query.filters) + && Objects.equal(explicit_tags, query.explicit_tags) + && Objects.equal(pre_aggregate, query.pre_aggregate) + && Objects.equal(use_fuzzy_filter, query.use_fuzzy_filter) + && Objects.equal(percentiles, query.percentiles) + && Objects.equal(show_histogram_buckets, query.show_histogram_buckets) + && Objects.equal(use_fuzzy_filter, query.use_fuzzy_filter) + && Objects.equal(use_multi_gets, query.use_multi_gets); } public String toString() { final StringBuilder buf = new StringBuilder(); buf.append("TSSubQuery(metric=") .append(metric == null || metric.isEmpty() ? "" : metric); - buf.append(", tags=["); - if (tags != null && !tags.isEmpty()) { + buf.append(", filters=["); + if (filters != null && !filters.isEmpty()) { int counter = 0; - for (Map.Entry entry : tags.entrySet()) { + for (final TagVFilter filter : filters) { if (counter > 0) { buf.append(", "); } - buf.append(entry.getKey()) - .append("=") - .append(entry.getValue()); - counter++; + buf.append(filter); + ++counter; } } buf.append("], tsuids=["); @@ -109,12 +183,20 @@ public String toString() { .append(", downsample=") .append(downsample) .append(", ds_interval=") - .append(downsample_interval) + .append(downsample_specifier.getInterval()) .append(", rate=") .append(rate) .append(", rate_options=") - .append(rate_options); - buf.append(")"); + .append(rate_options) + .append(", explicit_tags=") + .append("explicit_tags") + .append(", index=") + .append(index) + .append(", percentiles=") + .append(percentiles) + .append(", show_histogram_buckets=") + .append(show_histogram_buckets) + .append(")"); return buf.toString(); } @@ -145,38 +227,85 @@ public void validateAndSetQuery() { "Missing the metric or tsuids, provide at least one"); } + // Make sure we have a filter list + if (filters == null) { + filters = new ArrayList(); + } + // parse the downsampler if we have one if (downsample != null && !downsample.isEmpty()) { - final int dash = downsample.indexOf('-', 1); // 1st char can't be - // `-'. - if (dash < 0) { - throw new IllegalArgumentException("Invalid downsampling specifier '" - + downsample + "' in [" + downsample + "]"); - } - try { - downsampler = Aggregators.get(downsample.substring(dash + 1)); - } catch (NoSuchElementException e) { - throw new IllegalArgumentException("No such downsampling function: " - + downsample.substring(dash + 1)); - } - downsample_interval = DateTime.parseDuration( - downsample.substring(0, dash)); + // downsampler given, so parse it + downsample_specifier = new DownsamplingSpecification(downsample); + } else { + // no downsampler + downsample_specifier = DownsamplingSpecification.NO_DOWNSAMPLER; } + checkHistogramQuery(); } + /** + * Make sure the parameters for histogram query are valid. + *
      + *
    • aggregation function: only NONE and SUM supported
      • + *
    • aggregation function in downsampling: only SUM supported
      • + *
    • percentile: only in rage (0,100)
      • + *
    + */ + private void checkHistogramQuery() { + if (!isHistogramQuery()) { + return; + } + + // only support NONE and SUM + if (agg != null && agg != Aggregators.NONE && agg != Aggregators.SUM) { + throw new IllegalArgumentException("Only NONE or SUM aggregation function supported for histogram query"); + } + + // only support SUM in downsampling + if (DownsamplingSpecification.NO_DOWNSAMPLER != downsample_specifier && + downsample_specifier.getHistogramAggregation() != HistogramAggregation.SUM) { + throw new IllegalArgumentException("Only SUM downsampling aggregation supported for histogram query"); + } + + + if (null != percentiles && percentiles.size() > 0) { + for (Float parameter : percentiles) { + if (parameter < 0 || parameter > 100) { + throw new IllegalArgumentException("Invalid percentile parameters: " + parameter); + } + } + } + } + /** @return the parsed aggregation function */ public Aggregator aggregator() { return this.agg; } - /** @return the parsed downsampler aggregation function */ + /** @return the parsed downsampler aggregation function + * @deprecated use {@link #downsamplingSpecification()} instead */ public Aggregator downsampler() { - return this.downsampler; + return downsample_specifier.getFunction(); } - /** @return the parsed downsample interval in seconds */ + /** @return the parsed downsample interval in seconds + * @deprecated use {@link #downsamplingSpecification()} instead */ public long downsampleInterval() { - return this.downsample_interval; + return downsample_specifier.getInterval(); + } + + /** @return The downsampling specification for more options + * @since 2.3 */ + public DownsamplingSpecification downsamplingSpecification() { + return downsample_specifier; + } + + /** + * @return the downsampling fill policy + * @since 2.2 + */ + public FillPolicy fillPolicy() { + return downsample_specifier.getFillPolicy(); } /** @return the user supplied aggregator */ @@ -194,16 +323,26 @@ public List getTsuids() { return tsuids; } - /** @return the user supplied list of query tags, may be empty */ + /** @return the user supplied list of group by query tags, may be empty. + * Note that as of version 2.2 this is an immutable list of tags built from + * the filter list. + * @deprecated */ public Map getTags() { - if (tags == null) { + if (filters == null) { return Collections.emptyMap(); } - return tags; + final Map tags = new HashMap(filters.size()); + for (final TagVFilter filter : filters) { + if (filter.isGroupBy()) { + tags.put(filter.getTagk(), filter.getType() + + "(" + filter.getFilter() + ")"); + } + } + return ImmutableMap.copyOf(tags); } /** @return the raw downsampling function request from the user, - * e.g. "1h-avg" */ + * e.g. "1h-avg" or "15m-sum-nan" */ public String getDownsample() { return downsample; } @@ -218,6 +357,45 @@ public RateOptions getRateOptions() { return rate_options; } + /** @return the filters pulled from the tags object + * @since 2.2 */ + public List getFilters() { + if (filters == null) { + filters = new ArrayList(); + } + // send a copy so ordering doesn't mess up the hash code + return new ArrayList(filters); + } + + /** @return the unique set of tagks from the filters. May be null if no filters + * were set. Must make sure to resolve the string tag to UIDs in the filter first. + * @since 2.3 + */ + public ByteSet getFilterTagKs() { + if (filters == null || filters.isEmpty()) { + return null; + } + final ByteSet tagks = new ByteSet(); + for (final TagVFilter filter : filters) { + if (filter != null && filter.getTagkBytes() != null) { + tagks.add(filter.getTagkBytes()); + } + } + return tagks; + } + + /** @return whether or not to match series with ONLY the given tags + * @since 2.3 */ + public boolean getExplicitTags() { + return explicit_tags; + } + + /** @return the index of the sub query + * @since 2.3 */ + public int getIndex() { + return index; + } + /** @param aggregator the name of an aggregation function */ public void setAggregator(String aggregator) { this.aggregator = aggregator; @@ -233,9 +411,29 @@ public void setTsuids(List tsuids) { this.tsuids = tsuids; } - /** @param tags an optional list of tags for specificity or grouping */ - public void setTags(HashMap tags) { - this.tags = tags; + /** @return The percentile parameters */ + public List getPercentiles() { + return percentiles; + } + + /** @param percentiles The percentile parameters*/ + public void setPercentiles(List percentiles) { + this.percentiles = percentiles; + if (this.percentiles != null && !this.percentiles.isEmpty()) { + Collections.sort(this.percentiles); + } + } + + /** @param tags an optional list of tags for specificity or grouping + * As of 2.2 this will convert the existing tags to filter + * @deprecated */ + public void setTags(Map tags) { + if (filters == null) { + filters = new ArrayList(tags.size()); + } else { + filters.clear(); + } + TagVFilter.tagsToFilters(tags, filters); } /** @param downsample the downsampling function to use, e.g. "2h-avg" */ @@ -252,4 +450,109 @@ public void setRate(boolean rate) { public void setRateOptions(RateOptions options) { this.rate_options = options; } + + /** @param filters A list of filters to use when querying + * @since 2.2 */ + public void setFilters(List filters) { + this.filters = filters; + } + + /** @param explicit_tags whether or not to match series with ONLY the given tags + * @since 2.3 */ + public void setExplicitTags(final boolean explicit_tags) { + this.explicit_tags = explicit_tags; + } + + /** @return Whether or not the fuzzy filter is enabled. */ + public boolean getUseFuzzyFilter() { + return use_fuzzy_filter; + } + + /** @param use_fuzzy_filter Whether or not to enable the fuzzy filter. */ + public void setUseFuzzyFilter(final boolean use_fuzzy_filter) { + this.use_fuzzy_filter = use_fuzzy_filter; + } + + /** @param index the index of the sub query + * @since 2.3 */ + public void setIndex(final int index) { + this.index = index; + } + + /** Search the query on pre-aggregated table directly instead of post fetch + * aggregation. + * @return Whether or not to fetch data on pre-aggregates + * @since 2.4 + */ + public boolean isPreAggregate() { + return pre_aggregate; + } + + /** Search the query on pre-aggregated table directly instead of post fetch + * aggregation. + * @param pre_aggregate Whether or not to fetch data on pre-aggregated tables. + * @since 2.4 + */ + public void setPreAggregate(boolean pre_aggregate) { + this.pre_aggregate = pre_aggregate; + } + + /** @return Rollup data usage type. + * @since 2.4 */ + public TsdbQuery.ROLLUP_USAGE getRollupUsage() { + return rollup_usage; + } + + /** @param rollup_usage Rollup data usage. + * @since 2.4 */ + public void setRollupUsage(String rollup_usage) { + this.rollup_usage = TsdbQuery.ROLLUP_USAGE.parse(rollup_usage); + } + + /** Which rollup table it scanned to get the final result. + * @return The rollup table to use. + * @since 2.4 + */ + public String getRollupTable() { + if (tsdb_query != null) { + return tsdb_query.getRollupTable(); + } + else { + return "raw"; + } + } + + /** @param tsdb_query Parent TsdbQuery, which will tell the which rollup + * table it scanned to get the final result + * @since 2.4 + */ + void setTsdbQuery(TsdbQuery tsdb_query) { + this.tsdb_query = tsdb_query; + } + + /** + * Whether this query is towards histogram data points. + * @return true if this query is toward histogram data points, false otherwise. + */ + public boolean isHistogramQuery() { + return ((percentiles != null && percentiles.size() > 0) || this.show_histogram_buckets); + } + + public boolean getShowHistogramBuckets() { + return this.show_histogram_buckets; + } + + public void setShowHistogramBuckets(final boolean show_histogram_buckets) { + this.show_histogram_buckets = show_histogram_buckets; + } + + /** @return Whether or not to use multi gets for explicit tag queries */ + public boolean getUseMultiGets() { + return use_multi_gets; + } + + /** @param use_multi_gets Whether or not to use multi gets for explicit tag queries */ + public void setUseMultiGets(final boolean use_multi_gets) { + this.use_multi_gets = use_multi_gets; + } } diff --git a/src/core/Tags.java b/src/core/Tags.java index 5422baeaab..2e3167b16a 100644 --- a/src/core/Tags.java +++ b/src/core/Tags.java @@ -19,6 +19,8 @@ import java.util.List; import java.util.Map; +import com.stumbleupon.async.DeferredGroupException; +import net.opentsdb.utils.Exceptions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,7 +28,9 @@ import com.stumbleupon.async.Deferred; import org.hbase.async.Bytes; +import org.hbase.async.Bytes.ByteMap; +import net.opentsdb.query.filter.TagVFilter; import net.opentsdb.uid.NoSuchUniqueId; import net.opentsdb.uid.NoSuchUniqueName; import net.opentsdb.utils.Pair; @@ -35,6 +39,7 @@ public final class Tags { private static final Logger LOG = LoggerFactory.getLogger(Tags.class); + private static String allowSpecialChars = ""; private Tags() { // Can't create instances of this utility class. @@ -202,6 +207,72 @@ public static String parseWithMetric(final String metric, return metric.substring(0, curly); } + /** + * Parses the metric and tags out of the given string. + * @param metric A string of the form "metric" or "metric{tag=value,...}" or + * now "metric{groupby=filter}{filter=filter}". + * @param filters A list of filters to write the results to. May not be null + * @return The name of the metric. + * @throws IllegalArgumentException if the metric is malformed or the filter + * list is null. + * @since 2.2 + */ + public static String parseWithMetricAndFilters(final String metric, + final List filters) { + if (metric == null || metric.isEmpty()) { + throw new IllegalArgumentException("Metric cannot be null or empty"); + } + if (filters == null) { + throw new IllegalArgumentException("Filters cannot be null"); + } + final int curly = metric.indexOf('{'); + if (curly < 0) { + return metric; + } + final int len = metric.length(); + if (metric.charAt(len - 1) != '}') { // "foo{" + throw new IllegalArgumentException("Missing '}' at the end of: " + metric); + } else if (curly == len - 2) { // "foo{}" + return metric.substring(0, len - 2); + } + final int close = metric.indexOf('}'); + final HashMap filter_map = new HashMap(); + if (close != metric.length() - 1) { // "foo{...}{tagk=filter}" + final int filter_bracket = metric.lastIndexOf('{'); + for (final String filter : splitString(metric.substring(filter_bracket + 1, + metric.length() - 1), ',')) { + if (filter.isEmpty()) { + break; + } + filter_map.clear(); + try { + parse(filter_map, filter); + TagVFilter.mapToFilters(filter_map, filters, false); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("When parsing filter '" + filter + + "': " + e.getMessage(), e); + } + } + } + + // substring the tags out of "foo{a=b,...,x=y}" and parse them. + for (final String tag : splitString(metric.substring(curly + 1, close), ',')) { + try { + if (tag.isEmpty() && close != metric.length() - 1){ + break; + } + filter_map.clear(); + parse(filter_map, tag); + TagVFilter.tagsToFilters(filter_map, filters); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("When parsing tag '" + tag + + "': " + e.getMessage(), e); + } + } + // Return the "foo" part of "foo{a=b,...,x=y}" + return metric.substring(0, curly); + } + /** * Parses an integer value as a long from the given character sequence. *

    @@ -284,7 +355,7 @@ static String getValue(final TSDB tsdb, final byte[] row, * Extracts the value ID of the given tag UD name from the given row key. * @param tsdb The TSDB instance to use for UniqueId lookups. * @param row The row key in which to search the tag name. - * @param name The name of the tag to search in the row key. + * @param tag_id The name of the tag to search in the row key. * @return The value ID associated with the given tag ID, or null if this * tag ID isn't present in this row key. */ @@ -293,7 +364,8 @@ static byte[] getValueId(final TSDB tsdb, final byte[] row, final short name_width = tsdb.tag_names.width(); final short value_width = tsdb.tag_values.width(); // TODO(tsuna): Can do a binary search. - for (short pos = (short) (tsdb.metrics.width() + Const.TIMESTAMP_BYTES); + for (short pos = (short) (Const.SALT_WIDTH() + + tsdb.metrics.width() + Const.TIMESTAMP_BYTES); pos < row.length; pos += name_width + value_width) { if (rowContains(row, pos, tag_id)) { @@ -334,7 +406,14 @@ static Map getTags(final TSDB tsdb, final byte[] row) throws NoSuchUniqueId { try { return getTagsAsync(tsdb, row).joinUninterruptibly(); - } catch (RuntimeException e) { + } catch (DeferredGroupException e) { + final Throwable ex = Exceptions.getCause(e); + if (ex instanceof NoSuchUniqueId) { + throw (NoSuchUniqueId)ex; + } + + throw new RuntimeException("Should never be here", e); + } catch (RuntimeException e) { throw e; } catch (Exception e) { throw new RuntimeException("Should never be here", e); @@ -349,12 +428,13 @@ static Map getTags(final TSDB tsdb, * @throws NoSuchUniqueId if the row key contained an invalid ID (unlikely). * @since 1.2 */ - static Deferred> getTagsAsync(final TSDB tsdb, + public static Deferred> getTagsAsync(final TSDB tsdb, final byte[] row) throws NoSuchUniqueId { final short name_width = tsdb.tag_names.width(); final short value_width = tsdb.tag_values.width(); final short tag_bytes = (short) (name_width + value_width); - final short metric_ts_bytes = (short) (tsdb.metrics.width() + final short metric_ts_bytes = (short) (Const.SALT_WIDTH() + + tsdb.metrics.width() + Const.TIMESTAMP_BYTES); final ArrayList> deferreds = @@ -392,6 +472,74 @@ public Map call(final ArrayList names) return Deferred.groupInOrder(deferreds).addCallback(new NameCB()); } + /** + * Returns the names mapped to tag key/value UIDs + * @param tsdb The TSDB instance to use for Unique ID lookups. + * @param tags The map of tag key to tag value pairs + * @return A map of tag names (keys), tag values (values). If the tags list + * was null or empty, the result will be an empty map + * @throws NoSuchUniqueId if the row key contained an invalid ID. + * @since 2.3 + */ + public static Deferred> getTagsAsync(final TSDB tsdb, + final ByteMap tags) { + if (tags == null || tags.isEmpty()) { + return Deferred.fromResult(Collections.emptyMap()); + } + + final ArrayList> deferreds = + new ArrayList>(); + + for (final Map.Entry pair : tags) { + deferreds.add(tsdb.tag_names.getNameAsync(pair.getKey())); + deferreds.add(tsdb.tag_values.getNameAsync(pair.getValue())); + } + + class NameCB implements Callback, ArrayList> { + public Map call(final ArrayList names) + throws Exception { + final HashMap result = new HashMap(); + String tagk = ""; + for (String name : names) { + if (tagk.isEmpty()) { + tagk = name; + } else { + result.put(tagk, name); + tagk = ""; + } + } + return result; + } + } + + return Deferred.groupInOrder(deferreds).addCallback(new NameCB()); + } + + /** + * Returns the tag key and value pairs as a byte map given a row key + * @param row The row key to parse the UIDs from + * @return A byte map with tagk and tagv pairs as raw UIDs + * @since 2.2 + */ + public static ByteMap getTagUids(final byte[] row) { + final ByteMap uids = new ByteMap(); + final short name_width = TSDB.tagk_width(); + final short value_width = TSDB.tagv_width(); + final short tag_bytes = (short) (name_width + value_width); + final short metric_ts_bytes = (short) (TSDB.metrics_width() + + Const.TIMESTAMP_BYTES + + Const.SALT_WIDTH()); + + for (short pos = metric_ts_bytes; pos < row.length; pos += tag_bytes) { + final byte[] tmp_name = new byte[name_width]; + final byte[] tmp_value = new byte[value_width]; + System.arraycopy(row, pos, tmp_name, 0, name_width); + System.arraycopy(row, pos + name_width, tmp_value, 0, value_width); + uids.put(tmp_name, tmp_value); + } + return uids; + } + /** * Ensures that a given string is a valid metric name or tag name/value. * @param what A human readable description of what's being validated. @@ -409,7 +557,7 @@ public static void validateString(final String what, final String s) { final char c = s.charAt(i); if (!(('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || ('0' <= c && c <= '9') || c == '-' || c == '_' || c == '.' - || c == '/' || Character.isLetter(c))) { + || c == '/' || Character.isLetter(c) || isAllowSpecialChars(c))) { throw new IllegalArgumentException("Invalid " + what + " (\"" + s + "\"): illegal character: " + c); } @@ -436,7 +584,22 @@ public static ArrayList resolveAll(final TSDB tsdb, throw new RuntimeException("Should never happen!", e); } } - + + /** + * Resolves a set of tag strings to their UIDs asynchronously + * @param tsdb the TSDB to use for access + * @param tags The tags to resolve + * @return A deferred with the list of UIDs in tagk1, tagv1, .. tagkn, tagvn + * order + * @throws NoSuchUniqueName if one of the elements in the map contained an + * unknown tag name or tag value. + * @since 2.1 + */ + public static Deferred> resolveAllAsync(final TSDB tsdb, + final Map tags) { + return resolveAllInternalAsync(tsdb, null, tags, false); + } + /** * Resolves (and creates, if necessary) all the tags (name=value) into the a * sorted byte arrays. @@ -473,7 +636,6 @@ static ArrayList resolveAllInternal(final TSDB tsdb, return tag_ids; } - /** * Resolves (and creates, if necessary) all the tags (name=value) into the a * sorted byte arrays. @@ -485,11 +647,28 @@ static ArrayList resolveAllInternal(final TSDB tsdb, */ static Deferred> resolveOrCreateAllAsync(final TSDB tsdb, final Map tags) { - return resolveAllInternalAsync(tsdb, tags, true); + return resolveAllInternalAsync(tsdb, null, tags, true); + } + + /** + * Resolves (and creates, if necessary) all the tags (name=value) into the a + * sorted byte arrays. + * @param tsdb The TSDB to use for UniqueId lookups. + * @param metric The metric associated with this tag set for filtering. + * @param tags The tags to resolve. If a new tag name or tag value is + * seen, it will be assigned an ID. + * @return an array of sorted tags (tag id, tag name). + * @since 2.3 + */ + static Deferred> + resolveOrCreateAllAsync(final TSDB tsdb, final String metric, + final Map tags) { + return resolveAllInternalAsync(tsdb, metric, tags, true); } private static Deferred> resolveAllInternalAsync(final TSDB tsdb, + final String metric, final Map tags, final boolean create) { final ArrayList> tag_ids = @@ -498,10 +677,10 @@ static ArrayList resolveAllInternal(final TSDB tsdb, // For each tag, start resolving the tag name and the tag value. for (final Map.Entry entry : tags.entrySet()) { final Deferred name_id = create - ? tsdb.tag_names.getOrCreateIdAsync(entry.getKey()) + ? tsdb.tag_names.getOrCreateIdAsync(entry.getKey(), metric, tags) : tsdb.tag_names.getIdAsync(entry.getKey()); final Deferred value_id = create - ? tsdb.tag_values.getOrCreateIdAsync(entry.getValue()) + ? tsdb.tag_values.getOrCreateIdAsync(entry.getValue(), metric, tags) : tsdb.tag_values.getIdAsync(entry.getValue()); // Then once the tag name is resolved, get the resolved tag value. @@ -562,6 +741,16 @@ public static HashMap resolveIds(final TSDB tsdb, throws NoSuchUniqueId { try { return resolveIdsAsync(tsdb, tags).joinUninterruptibly(); + } catch (NoSuchUniqueId e) { + throw e; + } catch (DeferredGroupException e) { + final Throwable ex = Exceptions.getCause(e); + if (ex instanceof NoSuchUniqueId) { + throw (NoSuchUniqueId)ex; + } + // TODO process e.results() + + throw new RuntimeException("Shouldn't be here", e); } catch (Exception e) { throw new RuntimeException("Shouldn't be here", e); } @@ -637,4 +826,34 @@ public static boolean looksLikeInteger(final String value) { return true; } + /** + * Set the special characters due to allowing for a key or a value of the tag. + * @param characters character sequences as a string + */ + public static void setAllowSpecialChars(String characters) { + allowSpecialChars = characters == null ? "" : characters; + } + + /** + * Returns true if the character can be used a tag name or a tag value. + * @param character + * @return + */ + static boolean isAllowSpecialChars(char character) { + return allowSpecialChars.indexOf(character) != -1; + } + + /** + * Returns true if the given string can fit into a float. + * @param value The String holding the float value. + * @return true if the value can fit into a float, false otherwise. + * @throws NumberFormatException if the value is not numeric. + * @since 2.4 + */ + public static boolean fitsInFloat(final String value) { + // TODO - probably still a better way to do this and we could save a lot + // of space by dropping useless precision, but for now this should help. + final double d = Double.parseDouble(value); + return ((float) d) == d; + } } diff --git a/src/core/TsdbQuery.java b/src/core/TsdbQuery.java index 914dbd69ff..17947c72d2 100644 --- a/src/core/TsdbQuery.java +++ b/src/core/TsdbQuery.java @@ -20,28 +20,48 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.SortedMap; import java.util.TreeMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.hbase.async.BinaryPrefixComparator; import org.hbase.async.Bytes; +import org.hbase.async.CompareFilter; +import org.hbase.async.FilterList; import org.hbase.async.HBaseException; -import org.hbase.async.KeyValue; +import org.hbase.async.QualifierFilter; +import org.hbase.async.ScanFilter; import org.hbase.async.Scanner; +import org.hbase.async.Bytes.ByteMap; +import org.hbase.async.FilterList.Operator; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; import com.stumbleupon.async.Callback; import com.stumbleupon.async.Deferred; +import com.stumbleupon.async.DeferredGroupException; -import static org.hbase.async.Bytes.ByteMap; +import net.opentsdb.query.QueryUtil; +import net.opentsdb.query.filter.TagVFilter; +import net.opentsdb.query.filter.TagVLiteralOrFilter; +import net.opentsdb.rollup.NoSuchRollupForIntervalException; +import net.opentsdb.rollup.RollupInterval; +import net.opentsdb.rollup.RollupQuery; +import net.opentsdb.rollup.RollupUtils; import net.opentsdb.stats.Histogram; +import net.opentsdb.stats.QueryStats; +import net.opentsdb.stats.QueryStats.QueryStat; +import net.opentsdb.uid.NoSuchUniqueId; import net.opentsdb.uid.NoSuchUniqueName; import net.opentsdb.uid.UniqueId; +import net.opentsdb.utils.ByteSet; +import net.opentsdb.utils.DateTime; /** * Non-synchronized implementation of {@link Query}. */ -final class TsdbQuery implements Query { +final class TsdbQuery extends AbstractQuery { private static final Logger LOG = LoggerFactory.getLogger(TsdbQuery.class); @@ -63,6 +83,12 @@ final class TsdbQuery implements Query { /** The TSDB we belong to. */ private final TSDB tsdb; + + /** The time, in ns, when we start scanning for data **/ + private long scan_start_time; + + /** Whether or not the query has any results. */ + private Boolean no_results; /** Value used for timestamps that are uninitialized. */ private static final int UNSET = -1; @@ -73,16 +99,20 @@ final class TsdbQuery implements Query { /** End time (UNIX timestamp in seconds) on 32 bits ("unsigned" int). */ private long end_time = UNSET; + /** Whether or not to delete the queried data */ + private boolean delete; + /** ID of the metric being looked up. */ private byte[] metric; - /** - * Tags of the metrics being looked up. - * Each tag is a byte array holding the ID of both the name and value - * of the tag. - * Invariant: an element cannot be both in this array and in group_bys. - */ - private ArrayList tags; + /** Row key regex to pass to HBase if we have tags or TSUIDs */ + private String regex; + + /** Whether or not to enable the fuzzy row filter for Hbase */ + private boolean enable_fuzzy_filter; + + /** Whether or not the user wants to use the fuzzy filter */ + private boolean override_fuzzy_filter; /** * Tags by which we must group the results. @@ -92,38 +122,140 @@ final class TsdbQuery implements Query { private ArrayList group_bys; /** - * Values we may be grouping on. - * For certain elements in {@code group_bys}, we may have a specific list of - * values IDs we're looking for. Those IDs are stored in this map. The key - * is an element of {@code group_bys} (so a tag name ID) and the values are - * tag value IDs (at least two). + * Tag key and values to use in the row key filter, all pre-sorted */ - private ByteMap group_by_values; + private ByteMap row_key_literals; + private List> row_key_literals_list; /** If true, use rate of change instead of actual values. */ private boolean rate; /** Specifies the various options for rate calculations */ private RateOptions rate_options; - + /** Aggregator function to use. */ private Aggregator aggregator; - /** - * Downsampling function to use, if any (can be {@code null}). - * If this is non-null, {@code sample_interval_ms} must be strictly positive. - */ - private Aggregator downsampler; + /** Downsampling specification to use, if any (can be {@code null}). */ + private DownsamplingSpecification downsampler; + + /** Rollup interval and aggregator, null if not applicable. */ + private RollupQuery rollup_query; + + /** Map of RollupInterval objects in the order of next best match + * like 1d, 1h, 10m, 1m, for rollup of 1d. */ + private List best_match_rollups; + + /** How to use the rollup data */ + private ROLLUP_USAGE rollup_usage = ROLLUP_USAGE.ROLLUP_NOFALLBACK; - /** Minimum time interval (in milliseconds) wanted between each data point. */ - private long sample_interval_ms; + /** Search the query on pre-aggregated table directly instead of post fetch + * aggregation. */ + private boolean pre_aggregate; /** Optional list of TSUIDs to fetch and aggregate instead of a metric */ private List tsuids; - + + /** An index that links this query to the original sub query */ + private int query_index; + + /** Tag value filters to apply post scan */ + private List filters; + + /** An object for storing stats in regarding the query. May be null */ + private QueryStats query_stats; + + /** Whether or not to match series with ONLY the given tags */ + private boolean explicit_tags; + + private List percentiles; + + private boolean show_histogram_buckets; + + /** Set at filter resolution time to determine if we can use multi-gets */ + private boolean use_multi_gets; + + /** Set by the user if they want to bypass multi-gets */ + private boolean override_multi_get; + + /** Whether or not to use the search plugin for multi-get resolution. */ + private boolean multiget_with_search; + + /** Whether or not to fall back on query failure. */ + private boolean search_query_failure; + + /** The maximum number of bytes allowed per query. */ + private long max_bytes = 0; + + /** The maximum number of data points allowed per query. */ + private long max_data_points = 0; + + /** + * Enum for rollup fallback control. + * @since 2.4 + */ + public static enum ROLLUP_USAGE { + ROLLUP_RAW, //Don't use rollup data, instead use raw data + ROLLUP_NOFALLBACK, //Use rollup data, and don't fallback on no data + ROLLUP_FALLBACK, //Use rollup data and fallback to next best match on data + ROLLUP_FALLBACK_RAW; //Use rollup data and fallback to raw on no data + + /** + * Parse and transform a string to ROLLUP_USAGE object + * @param str String to be parsed + * @return enum param tells how to use the rollup data + */ + public static ROLLUP_USAGE parse(String str) { + ROLLUP_USAGE def = ROLLUP_NOFALLBACK; + + if (str != null) { + try { + def = ROLLUP_USAGE.valueOf(str.toUpperCase()); + } + catch(IllegalArgumentException ex) { + LOG.warn("Unknown rollup usage, " + str + ", use default usage - which" + + "uses raw data but don't fallback on no data"); + } + } + + return def; + } + + /** + * Whether to fallback to next best match or raw + * @return true means fall back else false + */ + public boolean fallback() { + return this == ROLLUP_FALLBACK || this == ROLLUP_FALLBACK_RAW; + } + } + /** Constructor. */ public TsdbQuery(final TSDB tsdb) { this.tsdb = tsdb; + this.downsampler = DownsamplingSpecification.NO_DOWNSAMPLER; + enable_fuzzy_filter = tsdb.getConfig() + .getBoolean("tsd.query.enable_fuzzy_filter"); + use_multi_gets = tsdb.getConfig().getBoolean("tsd.query.multi_get.enable"); + } + + /** Which rollup table it scanned to get the final result. + * @since 2.4 */ + public String getRollupTable() { + if (RollupQuery.isValidQuery(rollup_query)) { + return rollup_query.getRollupInterval().getInterval(); + } + else { + return "raw"; + } + } + + /** Search the query on pre-aggregated table directly instead of post fetch + * aggregation. + * @since 2.4 + */ + public boolean isPreAggregate() { + return this.pre_aggregate; } /** @@ -134,12 +266,12 @@ public TsdbQuery(final TSDB tsdb) { */ @Override public void setStartTime(final long timestamp) { - if (timestamp < 0 || ((timestamp & Const.SECOND_MASK) != 0 && + if (timestamp < 0 || ((timestamp & Const.SECOND_MASK) != 0 && timestamp > 9999999999999L)) { throw new IllegalArgumentException("Invalid timestamp: " + timestamp); - } else if (end_time != UNSET && timestamp >= getEndTime()) { + } else if (end_time != UNSET && timestamp > getEndTime()) { throw new IllegalArgumentException("new start time (" + timestamp - + ") is greater than or equal to end time: " + getEndTime()); + + ") is greater than end time: " + getEndTime()); } start_time = timestamp; } @@ -168,9 +300,9 @@ public void setEndTime(final long timestamp) { if (timestamp < 0 || ((timestamp & Const.SECOND_MASK) != 0 && timestamp > 9999999999999L)) { throw new IllegalArgumentException("Invalid timestamp: " + timestamp); - } else if (start_time != UNSET && timestamp <= getStartTime()) { + } else if (start_time != UNSET && timestamp < getStartTime()) { throw new IllegalArgumentException("new end time (" + timestamp - + ") is less than or equal to start time: " + getStartTime()); + + ") is less than start time: " + getStartTime()); } end_time = timestamp; } @@ -181,11 +313,26 @@ public void setEndTime(final long timestamp) { @Override public long getEndTime() { if (end_time == UNSET) { - setEndTime(System.currentTimeMillis()); + setEndTime(DateTime.currentTimeMillis()); } return end_time; } - + + @Override + public void setDelete(boolean delete) { + this.delete = delete; + } + + @Override + public boolean getDelete() { + return delete; + } + + @Override + public void setPercentiles(List percentiles) { + this.percentiles = percentiles; + } + @Override public void setTimeSeries(final String metric, final Map tags, @@ -201,9 +348,37 @@ public void setTimeSeries(final String metric, final boolean rate, final RateOptions rate_options) throws NoSuchUniqueName { - findGroupBys(tags); + if (filters == null) { + filters = new ArrayList(tags.size()); + } + TagVFilter.tagsToFilters(tags, filters); + + try { + for (final TagVFilter filter : this.filters) { + filter.resolveTagkName(tsdb).join(); + } + } catch (final InterruptedException e) { + LOG.warn("Interrupted", e); + Thread.currentThread().interrupt(); + } catch (final NoSuchUniqueName e) { + throw e; + } catch (final Exception e) { + if (e instanceof DeferredGroupException) { + // rollback to the actual case. The DGE missdirects + Throwable ex = e.getCause(); + while(ex != null && ex instanceof DeferredGroupException) { + ex = ex.getCause(); + } + if (ex != null) { + throw (RuntimeException)ex; + } + } + LOG.error("Unexpected exception processing group bys", e); + throw new RuntimeException(e); + } + + findGroupBys(); this.metric = tsdb.metrics.getId(metric); - this.tags = Tags.resolveAll(tsdb, tags); aggregator = function; this.rate = rate; this.rate_options = rate_options; @@ -248,7 +423,237 @@ public void setTimeSeries(final List tsuids, } /** - * Sets an optional downsampling function on this query + * @param explicit_tags Whether or not to match only on the given tags + * @since 2.3 + */ + public void setExplicitTags(final boolean explicit_tags) { + this.explicit_tags = explicit_tags; + } + + /** + * Splits this query into one query for the part that is covered by the rollup + * table (as defined in its SLA) and one to get the data for the remaining time + * range from the raw table. + * @param query The original TSQuery as parsed + * @param index The index of the TSQuery + * @param rawQuery A new TsdbQuery instance that will be configured to hit the raw table + * for the correct time range + * @return the deferred analogous to {@link TsdbQuery#configureFromQuery(TSQuery, int)} + * @throws IllegalStateException if the query is not eligible or splitting is disabled + */ + public Deferred split(final TSQuery query, final int index, final TsdbQuery rawQuery) { + if (!needsSplitting()) { + throw new IllegalStateException("Query is not eligible for splitting" + this.toString()); + } + + Deferred rawResolutionDeferred = rawQuery.configureFromQuery(query, index, true); + + long lastRollupTimestampMillis = rollup_query.getLastRollupTimestampSeconds() * 1000L; + + boolean needsRawAndRollupData = QueryUtil.isTimestampAfter(lastRollupTimestampMillis, getStartTime()); + if (needsRawAndRollupData) { + updateRollupSplitTimes(rawQuery, lastRollupTimestampMillis); + } + + return rawResolutionDeferred; + } + + /** + * Updates the timestamp of this query and the corresponding raw part in the case of a split. + * + * Sets the start and end times for this query so that it hits the rollup table until the given timestamp. + * Also updates the passed {@param rawQuery} with the new start time so that it hits the raw table for points from the + * given timestamp onwards. + * + * Makes sure that all timestamps are in milliseconds. + * + * @param rawQuery The raw query part + * @param splitTimestamp The timestamp until when rollup data is guaranteed to be available + */ + private void updateRollupSplitTimes(final TsdbQuery rawQuery, long splitTimestamp) { + setEndTime(splitTimestamp); + + boolean isStartTimeInSeconds = (getStartTime() & Const.SECOND_MASK) == 0; + if (isStartTimeInSeconds) { + setStartTime(getStartTime() * 1000L); + } + + boolean isRawEndTimeInSeconds = (rawQuery.getEndTime() & Const.SECOND_MASK) == 0; + if (isRawEndTimeInSeconds) { + rawQuery.setEndTime(rawQuery.getEndTime() * 1000L); + } + + rawQuery.setStartTime(splitTimestamp); + } + + @Override + public Deferred configureFromQuery(final TSQuery query, + final int index) { + return configureFromQuery(query, index, false); + } + + + public Deferred configureFromQuery(final TSQuery query, + final int index, boolean force_raw) { + if (query.getQueries() == null || query.getQueries().isEmpty()) { + throw new IllegalArgumentException("Missing sub queries"); + } + if (index < 0 || index > query.getQueries().size()) { + throw new IllegalArgumentException("Query index was out of range"); + } + + final TSSubQuery sub_query = query.getQueries().get(index); + setStartTime(query.startTime()); + setEndTime(query.endTime()); + setDelete(query.getDelete()); + query_index = index; + query_stats = query.getQueryStats(); + + // set common options + aggregator = sub_query.aggregator(); + rate = sub_query.getRate(); + rate_options = sub_query.getRateOptions(); + if (rate_options == null) { + rate_options = new RateOptions(); + } + downsampler = sub_query.downsamplingSpecification(); + pre_aggregate = sub_query.isPreAggregate(); + rollup_usage = sub_query.getRollupUsage(); + filters = sub_query.getFilters(); + explicit_tags = sub_query.getExplicitTags(); + override_fuzzy_filter = sub_query.getUseFuzzyFilter(); + override_multi_get = sub_query.getUseMultiGets(); + + max_bytes = tsdb.getQueryByteLimits().getByteLimit(sub_query.getMetric()); + if (tsdb.getConfig().getBoolean("tsd.query.limits.bytes.allow_override") && + query.overrideByteLimit()) { + max_bytes = 0; + } + max_data_points = tsdb.getQueryByteLimits().getDataPointLimit(sub_query.getMetric()); + if (tsdb.getConfig().getBoolean("tsd.query.limits.data_points.allow_override") && + query.overrideDataPointLimit()) { + max_data_points = 0; + } + + // set percentile options + percentiles = sub_query.getPercentiles(); + show_histogram_buckets = sub_query.getShowHistogramBuckets(); + + if (!force_raw && rollup_usage != ROLLUP_USAGE.ROLLUP_RAW) { + //Check whether the down sampler is set and rollup is enabled + transformDownSamplerToRollupQuery(aggregator, sub_query.getDownsample()); + } + sub_query.setTsdbQuery(this); + + if (use_multi_gets && override_multi_get && multiget_with_search) { + row_key_literals_list = Lists.newArrayList(); + } + + // if we have tsuids set, that takes precedence + if (sub_query.getTsuids() != null && !sub_query.getTsuids().isEmpty()) { + tsuids = new ArrayList(sub_query.getTsuids()); + String first_metric = ""; + for (final String tsuid : tsuids) { + if (first_metric.isEmpty()) { + first_metric = tsuid.substring(0, TSDB.metrics_width() * 2) + .toUpperCase(); + continue; + } + + final String metric = tsuid.substring(0, TSDB.metrics_width() * 2) + .toUpperCase(); + if (!first_metric.equals(metric)) { + throw new IllegalArgumentException( + "One or more TSUIDs did not share the same metric [" + first_metric + + "] [" + metric + "]"); + } + } + return Deferred.fromResult(null); + } else { + /** Triggers the group by resolution if we had filters to resolve */ + class FilterCB implements Callback> { + @Override + public Object call(final ArrayList results) throws Exception { + findGroupBys(); + return null; + } + } + + /** Resolve and group by tags after resolving the metric */ + class MetricCB implements Callback, byte[]> { + @Override + public Deferred call(final byte[] uid) throws Exception { + metric = uid; + if (filters != null) { + if (use_multi_gets && override_multi_get && multiget_with_search) { + class ErrorCB implements Callback, Exception> { + @Override + public Deferred call(Exception arg) throws Exception { + LOG.info("Doing scans because meta query is failed", arg); + if (explicit_tags) { + search_query_failure = true; + } else { + override_multi_get = false; + use_multi_gets = false; + } + return Deferred.group(resolveTagFilters()).addCallback(new FilterCB()); + } + } + + class SuccessCB implements Callback, List>> { + @Override + public Deferred call(final List> results) throws Exception { + row_key_literals_list.addAll(results); + return Deferred.fromResult(null); + } + } + + tsdb.getSearchPlugin().resolveTSQuery(query, index) + .addCallbackDeferring(new SuccessCB()) + .addErrback(new ErrorCB()); + } + + return Deferred.group(resolveTagFilters()).addCallback(new FilterCB()); + } else { + return Deferred.fromResult(null); + } + } + + private List> resolveTagFilters() { + final List> deferreds = + new ArrayList>(filters.size()); + for (final TagVFilter filter : filters) { + // determine if the user is asking for pre-agg data + if (filter instanceof TagVLiteralOrFilter && tsdb.getAggTagKey() != null) { + if (filter.getTagk().equals(tsdb.getAggTagKey())) { + if (tsdb.getRawTagValue() != null && + !filter.getFilter().equals(tsdb.getRawTagValue())) { + pre_aggregate = true; + } + } + } + + deferreds.add(filter.resolveTagkName(tsdb)); + } + return deferreds; + } + } + + // fire off the callback chain by resolving the metric first + return tsdb.metrics.getIdAsync(sub_query.getMetric()) + .addCallbackDeferring(new MetricCB()); + } + } + + @Override + public void downsample(final long interval, final Aggregator downsampler, + final FillPolicy fill_policy) { + this.downsampler = new DownsamplingSpecification( + interval, downsampler, fill_policy); + } + + /** + * Sets an optional downsampling function with interpolation on this query. * @param interval The interval, in milliseconds to rollup data points * @param downsampler An aggregation function to use when rolling up data points * @throws NullPointerException if the aggregation function is null @@ -256,82 +661,189 @@ public void setTimeSeries(final List tsuids, */ @Override public void downsample(final long interval, final Aggregator downsampler) { - if (downsampler == null) { - throw new NullPointerException("downsampler"); - } else if (interval <= 0) { - throw new IllegalArgumentException("interval not > 0: " + interval); + if (downsampler == Aggregators.NONE) { + throw new IllegalArgumentException("cannot use the NONE " + + "aggregator for downsampling"); } - this.downsampler = downsampler; - this.sample_interval_ms = interval; + downsample(interval, downsampler, FillPolicy.NONE); } /** - * Extracts all the tags we must use to group results. - *
      - *
    • If a tag has the form {@code name=*} then we'll create one - * group per value we find for that tag.
      • - *
    • If a tag has the form {@code name={v1,v2,..,vN}} then we'll - * create {@code N} groups.
      • - *
    - * In the both cases above, {@code name} will be stored in the - * {@code group_bys} attribute. In the second case specifically, - * the {@code N} values would be stored in {@code group_by_values}, - * the key in this map being {@code name}. - * @param tags The tags from which to extract the 'GROUP BY's. - * Each tag that represents a 'GROUP BY' will be removed from the map - * passed in argument. + * Populates the {@link #group_bys} and {@link #row_key_literals}'s with + * values pulled from the filters. */ - private void findGroupBys(final Map tags) { - final Iterator> i = tags.entrySet().iterator(); - while (i.hasNext()) { - final Map.Entry tag = i.next(); - final String tagvalue = tag.getValue(); - if (tagvalue.equals("*") // 'GROUP BY' with any value. - || tagvalue.indexOf('|', 1) >= 0) { // Multiple possible values. + private void findGroupBys() { + if (filters == null || filters.isEmpty()) { + return; + } + + if ((use_multi_gets && override_multi_get) && !search_query_failure) { + + } + + row_key_literals = new ByteMap(); + final int expansion_limit = tsdb.getConfig().getInt( + "tsd.query.filter.expansion_limit"); + + Collections.sort(filters); + final Iterator current_iterator = filters.iterator(); + final Iterator look_ahead = filters.iterator(); + byte[] tagk = null; + TagVFilter next = look_ahead.hasNext() ? look_ahead.next() : null; + int row_key_literals_count = 0; + while (current_iterator.hasNext()) { + next = look_ahead.hasNext() ? look_ahead.next() : null; + int gbs = 0; + // sorted! + final ByteMap literals = new ByteMap(); + final List literal_filters = new ArrayList(); + TagVFilter current = null; + do { // yeah, I'm breakin out the do!!! + current = current_iterator.next(); + if (tagk == null) { + tagk = new byte[TSDB.tagk_width()]; + System.arraycopy(current.getTagkBytes(), 0, tagk, 0, TSDB.tagk_width()); + } + + if (current.isGroupBy()) { + gbs++; + } + if (!current.getTagVUids().isEmpty()) { + for (final byte[] uid : current.getTagVUids()) { + literals.put(uid, null); + } + literal_filters.add(current); + } + + if (next != null && Bytes.memcmp(tagk, next.getTagkBytes()) != 0) { + break; + } + next = look_ahead.hasNext() ? look_ahead.next() : null; + } while (current_iterator.hasNext() && + Bytes.memcmp(tagk, current.getTagkBytes()) == 0); + + if (gbs > 0) { if (group_bys == null) { group_bys = new ArrayList(); } - group_bys.add(tsdb.tag_names.getId(tag.getKey())); - i.remove(); - if (tagvalue.charAt(0) == '*') { - continue; // For a 'GROUP BY' with any value, we're done. + group_bys.add(current.getTagkBytes()); + } + + if (literals.size() > 0) { + if (literals.size() + row_key_literals_count > expansion_limit) { + LOG.debug("Skipping literals for " + current.getTagk() + + " as it exceedes the limit"); + //has_filter_cannot_use_get = true; + } else { + final byte[][] values = new byte[literals.size()][]; + literals.keySet().toArray(values); + row_key_literals.put(current.getTagkBytes(), values); + row_key_literals_count += values.length; + + for (final TagVFilter filter : literal_filters) { + filter.setPostScan(false); + } } - // 'GROUP BY' with specific values. Need to split the values - // to group on and store their IDs in group_by_values. - final String[] values = Tags.splitString(tagvalue, '|'); - if (group_by_values == null) { - group_by_values = new ByteMap(); + } else { + row_key_literals.put(current.getTagkBytes(), null); + // no literal values, just keys, so we can't multi-get + if (search_query_failure) { + use_multi_gets = false; } - final short value_width = tsdb.tag_values.width(); - final byte[][] value_ids = new byte[values.length][value_width]; - group_by_values.put(tsdb.tag_names.getId(tag.getKey()), - value_ids); - for (int j = 0; j < values.length; j++) { - final byte[] value_id = tsdb.tag_values.getId(values[j]); - System.arraycopy(value_id, 0, value_ids[j], 0, value_width); + } + + // make sure the multi-get cardinality doesn't exceed our limit (or disable + // multi-gets) + if ((use_multi_gets && override_multi_get)) { + int multi_get_limit = tsdb.getConfig().getInt("tsd.query.multi_get.limit"); + int cardinality = filters.size() * row_key_literals_count; + if (cardinality > multi_get_limit) { + use_multi_gets = false; + } else if (search_query_failure) { + row_key_literals_list.add(row_key_literals); } + // TODO - account for time as well } } } - /** - * Executes the query - * @return An array of data points with one time series per array value - */ @Override - public DataPoints[] run() throws HBaseException { - try { - return runAsync().joinUninterruptibly(); - } catch (RuntimeException e) { - throw e; - } catch (Exception e) { - throw new RuntimeException("Should never be here", e); + public Deferred runAsync() throws HBaseException { + Deferred result = null; + + if (use_multi_gets && override_multi_get) { + result = this.findSpansWithMultiGetter().addCallback(new GroupByAndAggregateCB()); + } else { + result = findSpans().addCallback(new GroupByAndAggregateCB()); + } + + if (rollup_usage != null && rollup_usage.fallback()) { + result.addCallback(new FallbackRollupOnEmptyResult()); + } + + return result; + } + + @Override + public Deferred runHistogramAsync() throws HBaseException { + if (!isHistogramQuery()) { + throw new RuntimeException("Should never be here"); + } + + Deferred result = null; + if (use_multi_gets && override_multi_get) { + result = findHistogramSpansWithMultiGetter() + .addCallback(new HistogramGroupByAndAggregateCB()); + } else { + result = findHistogramSpans() + .addCallback(new HistogramGroupByAndAggregateCB()); } + + return result; } @Override - public Deferred runAsync() throws HBaseException { - return findSpans().addCallback(new GroupByAndAggregateCB()); + public boolean isHistogramQuery() { + if ((this.percentiles != null && this.percentiles.size() > 0) || show_histogram_buckets) { + return true; + } + + return false; + } + + @Override + public boolean isRollupQuery() { + return RollupQuery.isValidQuery(rollup_query); + } + + /** + * Returns whether this query needs to be split. It does if + * - splitting of queries is enabled globally AND + * - it can be split (i.e. it's a valid rollups query) AND + * - the table it is hitting has an SLA configured that describes the blackout period AND + * - the query is actually looking at data from the time beyond the SLA + * + * @return whether this query needs to be split. + * @since 2.4 + */ + @Override + public boolean needsSplitting() { + if (!tsdb.isRollupsSplittingEnabled()) { + // Don't split if the global config doesn't allow it + return false; + } + + if (!isRollupQuery()) { + // Don't split if it's hitting the raw table anyway + return false; + } + + if (rollup_query.getRollupInterval().getMaximumLag() <= 0) { + // Don't split if the table doesn't have a maximum lag configured + return false; + } + + return rollup_query.isInBlackoutPeriod(getEndTime()); } /** @@ -345,106 +857,129 @@ public Deferred runAsync() throws HBaseException { * perform the search. * @throws IllegalArgumentException if bad data was retrieved from HBase. */ - private Deferred> findSpans() throws HBaseException { + private Deferred> findSpans() throws HBaseException { final short metric_width = tsdb.metrics.width(); final TreeMap spans = // The key is a row key from HBase. - new TreeMap(new SpanCmp(metric_width)); - final Scanner scanner = getScanner(); - final Deferred> results = - new Deferred>(); + new TreeMap(new SpanCmp( + (short)(Const.SALT_WIDTH() + metric_width))); - /** - * Scanner callback executed recursively each time we get a set of data - * from storage. This is responsible for determining what columns are - * returned and issuing requests to load leaf objects. - * When the scanner returns a null set of rows, the method initiates the - * final callback. - */ - final class ScannerCB implements Callback>> { - - int nrows = 0; - boolean seenAnnotation = false; - int hbase_time = 0; // milliseconds. - long starttime = System.nanoTime(); - - /** - * Starts the scanner and is called recursively to fetch the next set of - * rows from the scanner. - * @return The map of spans if loaded successfully, null if no data was - * found - */ - public Object scan() { - starttime = System.nanoTime(); - return scanner.nextRows().addCallback(this); - } + // Copy only the filters that should trigger a tag resolution. If this list + // is empty due to literals or a wildcard star, then we'll save a TON of + // UID lookups + final List scanner_filters; + if (filters != null) { + scanner_filters = new ArrayList(filters.size()); + for (final TagVFilter filter : filters) { + if (filter.postScan()) { + scanner_filters.add(filter); + } + } + } else { + scanner_filters = null; + } + + if (Const.SALT_WIDTH() > 0) { + final List scanners = new ArrayList(Const.SALT_BUCKETS()); + for (int i = 0; i < Const.SALT_BUCKETS(); i++) { + scanners.add(getScanner(i)); + } + scan_start_time = DateTime.nanoTime(); + return new SaltScanner(tsdb, metric, scanners, spans, scanner_filters, + delete, rollup_query, query_stats, query_index, null, + max_bytes, max_data_points).scan(); + } else { + final List scanners = new ArrayList(1); + scanners.add(getScanner(0)); + scan_start_time = DateTime.nanoTime(); + return new SaltScanner(tsdb, metric, scanners, spans, scanner_filters, + delete, rollup_query, query_stats, query_index, null, max_bytes, + max_data_points).scan(); + } + } - /** - * Loops through each row of the scanner results and parses out data - * points and optional meta data - * @return null if no rows were found, otherwise the TreeMap with spans - */ - @Override - public Object call(final ArrayList> rows) - throws Exception { - hbase_time += (System.nanoTime() - starttime) / 1000000; - try { - if (rows == null) { - hbase_time += (System.nanoTime() - starttime) / 1000000; - scanlatency.add(hbase_time); - LOG.info(TsdbQuery.this + " matched " + nrows + " rows in " + - spans.size() + " spans in " + hbase_time + "ms"); - if (nrows < 1 && !seenAnnotation) { - results.callback(null); - } else { - results.callback(spans); - } - scanner.close(); - return null; - } - - for (final ArrayList row : rows) { - final byte[] key = row.get(0).key(); - if (Bytes.memcmp(metric, key, 0, metric_width) != 0) { - scanner.close(); - throw new IllegalDataException( - "HBase returned a row that doesn't match" - + " our scanner (" + scanner + ")! " + row + " does not start" - + " with " + Arrays.toString(metric)); - } - Span datapoints = spans.get(key); - if (datapoints == null) { - datapoints = new Span(tsdb); - spans.put(key, datapoints); - } - final KeyValue compacted = - tsdb.compact(row, datapoints.getAnnotations()); - seenAnnotation |= !datapoints.getAnnotations().isEmpty(); - if (compacted != null) { // Can be null if we ignored all KVs. - datapoints.addRow(compacted); - nrows++; - } - } + private Deferred> findSpansWithMultiGetter() throws HBaseException { + final short metric_width = tsdb.metrics.width(); + final TreeMap spans = // The key is a row key from HBase. + new TreeMap(new SpanCmp(metric_width)); - return scan(); - } catch (Exception e) { - scanner.close(); - results.callback(e); - return null; - } - } - } + scan_start_time = System.nanoTime(); + + return new MultiGetQuery(tsdb, this, metric, row_key_literals_list, + getScanStartTimeSeconds(), getScanEndTimeSeconds(), + tableToBeScanned(), spans, null, 0, rollup_query, query_stats, query_index, 0, + false, search_query_failure).fetch(); + } + + /** + * Finds all the {@link HistogramSpan}s that match this query. + * This is what actually scans the HBase table and loads the data into + * {@link HistogramSpan}s. + * + * @return A map from HBase row key to the {@link HistogramSpan} for that row key. + * Since a {@link HistogramSpan} actually contains multiple HBase rows, the row key + * stored in the map has its timestamp zero'ed out. + * + * @throws HBaseException if there was a problem communicating with HBase to + * perform the search. + * @throws IllegalArgumentException if bad data was retreived from HBase. + */ + private Deferred> findHistogramSpans() throws HBaseException { + final short metric_width = tsdb.metrics.width(); + final TreeMap histSpans = new TreeMap(new SpanCmp(metric_width)); + + // Copy only the filters that should trigger a tag resolution. If this list + // is empty due to literals or a wildcard star, then we'll save a TON of + // UID lookups + final List scanner_filters; + if (filters != null) { + scanner_filters = new ArrayList(filters.size()); + for (final TagVFilter filter : filters) { + if (filter.postScan()) { + scanner_filters.add(filter); + } + } + } else { + scanner_filters = null; + } - new ScannerCB().scan(); - return results; + scan_start_time = System.nanoTime(); + final List scanners; + if (Const.SALT_WIDTH() > 0) { + scanners = new ArrayList(Const.SALT_BUCKETS()); + for (int i = 0; i < Const.SALT_BUCKETS(); i++) { + scanners.add(getScanner(i)); + } + scan_start_time = DateTime.nanoTime(); + return new SaltScanner(tsdb, metric, scanners, null, scanner_filters, + delete, rollup_query, query_stats, query_index, histSpans, + max_bytes, max_data_points).scanHistogram(); + } else { + scanners = Lists.newArrayList(getScanner()); + scan_start_time = DateTime.nanoTime(); + return new SaltScanner(tsdb, metric, scanners, null, scanner_filters, + delete, rollup_query, query_stats, query_index, histSpans, + max_bytes, max_data_points).scanHistogram(); + } } + + private Deferred> findHistogramSpansWithMultiGetter() throws HBaseException { + final short metric_width = tsdb.metrics.width(); + // The key is a row key from HBase + final TreeMap histSpans = new TreeMap(new SpanCmp(metric_width)); + scan_start_time = System.nanoTime(); + return new MultiGetQuery(tsdb, this, metric, row_key_literals_list, + getScanStartTimeSeconds(), getScanEndTimeSeconds(), + tableToBeScanned(), null, histSpans, 0, rollup_query, query_stats, query_index, 0, + false, search_query_failure).fetchHistogram(); + } + /** - * Callback that should be attached the the output of - * {@link TsdbQuery#findSpans} to group and sort the results. - */ + * Callback that should be attached the the output of + * {@link TsdbQuery#findSpans} to group and sort the results. + */ private class GroupByAndAggregateCB implements - Callback>{ + Callback>{ /** * Creates the {@link SpanGroup}s to form the final results of this query. @@ -454,10 +989,44 @@ private class GroupByAndAggregateCB implements * any 'GROUP BY' formulated in this query. */ @Override - public DataPoints[] call(final TreeMap spans) throws Exception { + public DataPoints[] call(final SortedMap spans) throws Exception { + if (query_stats != null) { + query_stats.addStat(query_index, QueryStat.QUERY_SCAN_TIME, + (System.nanoTime() - TsdbQuery.this.scan_start_time)); + } + if (spans == null || spans.size() <= 0) { + if (query_stats != null) { + query_stats.addStat(query_index, QueryStat.GROUP_BY_TIME, 0); + } return NO_RESULT; } + + // The raw aggregator skips group bys and ignores downsampling + if (aggregator == Aggregators.NONE) { + final SpanGroup[] groups = new SpanGroup[spans.size()]; + int i = 0; + for (final Span span : spans.values()) { + final SpanGroup group = new SpanGroup( + tsdb, + getScanStartTimeSeconds(), + getScanEndTimeSeconds(), + null, + rate, + rate_options, + aggregator, + downsampler, + getStartTime(), + getEndTime(), + query_index, + rollup_query, + null); + group.add(span); + groups[i++] = group; + } + return groups; + } + if (group_bys == null) { // We haven't been asked to find groups, so let's put all the spans // together in the same group. @@ -467,7 +1036,15 @@ public DataPoints[] call(final TreeMap spans) throws Exception { spans.values(), rate, rate_options, aggregator, - sample_interval_ms, downsampler); + downsampler, + getStartTime(), + getEndTime(), + query_index, + rollup_query, + new byte[0]); + if (query_stats != null) { + query_stats.addStat(query_index, QueryStat.GROUP_BY_TIME, 0); + } return new SpanGroup[] { group }; } @@ -508,14 +1085,20 @@ public DataPoints[] call(final TreeMap spans) throws Exception { //LOG.info("Span belongs to group " + Arrays.toString(group) + ": " + Arrays.toString(row)); SpanGroup thegroup = groups.get(group); if (thegroup == null) { - thegroup = new SpanGroup(tsdb, getScanStartTimeSeconds(), - getScanEndTimeSeconds(), - null, rate, rate_options, aggregator, - sample_interval_ms, downsampler); // Copy the array because we're going to keep `group' and overwrite // its contents. So we want the collection to have an immutable copy. final byte[] group_copy = new byte[group.length]; System.arraycopy(group, 0, group_copy, 0, group.length); + + thegroup = new SpanGroup(tsdb, getScanStartTimeSeconds(), + getScanEndTimeSeconds(), + null, rate, rate_options, aggregator, + downsampler, + getStartTime(), + getEndTime(), + query_index, + rollup_query, + group_copy); groups.put(group_copy, thegroup); } thegroup.add(entry.getValue()); @@ -523,10 +1106,322 @@ public DataPoints[] call(final TreeMap spans) throws Exception { //for (final Map.Entry entry : groups) { // LOG.info("group for " + Arrays.toString(entry.getKey()) + ": " + entry.getValue()); //} + if (query_stats != null) { + query_stats.addStat(query_index, QueryStat.GROUP_BY_TIME, 0); + } return groups.values().toArray(new SpanGroup[groups.size()]); } } + /** + * Callback that should be attached the the output of + * {@link TsdbQuery#findHistogramSpans} to group and sort the results. + */ + private class HistogramGroupByAndAggregateCB implements + Callback>{ + + /** + * Creates the {@link HistogramSpanGroup}s to form the final results of this query. + * @param spans The {@link HistogramSpan}s found for this query ({@link #findHistogramSpans}). + * Can be {@code null}, in which case the array returned will be empty. + * @return A possibly empty array of {@link HistogramSpanGroup}s built according to + * any 'GROUP BY' formulated in this query. + */ + public DataPoints[] call(final SortedMap spans) throws Exception { + if (query_stats != null) { + query_stats.addStat(query_index, QueryStat.QUERY_SCAN_TIME, + (System.nanoTime() - TsdbQuery.this.scan_start_time)); + } + + final long group_build = System.nanoTime(); + if (spans == null || spans.size() <= 0) { + if (query_stats != null) { + query_stats.addStat(query_index, QueryStat.GROUP_BY_TIME, 0); + } + return NO_RESULT; + } + final ByteSet query_tags = null; + // TODO +// if (agg_tag_promotion && group_bys != null && !group_bys.isEmpty()) { +// query_tags = new ByteSet(); +// query_tags.addAll(group_bys); +// } else { +// query_tags = null; +// } + + final ArrayList result_dp_groups = new ArrayList(); + // The raw aggregator skips group bys and ignores downsampling + if (aggregator == Aggregators.NONE) { + for (final HistogramSpan span : spans.values()) { + final HistogramSpanGroup group = new HistogramSpanGroup(tsdb, + getScanStartTimeSeconds(), + getScanEndTimeSeconds(), + null, + null, + downsampler, + getStartTime(), + getEndTime(), + query_index, + RollupQuery.isValidQuery(rollup_query), + query_tags); + group.add(span); + + // create histogram data points to data points adaptor for each percentile calculation + if (null != percentiles && percentiles.size() > 0) { + List percentile_datapoints_list = generateHistogramPercentileDataPoints(group); + if (null != percentile_datapoints_list && percentile_datapoints_list.size() > 0) + result_dp_groups.addAll(percentile_datapoints_list); + } + + + // create bucket metric + if (show_histogram_buckets) { + List bucket_datapoints_list = generateHistogramBucketDataPoints(group); + if (null != bucket_datapoints_list && bucket_datapoints_list.size() > 0) { + result_dp_groups.addAll(bucket_datapoints_list); + } + } + } // end for + + int i = 0; + DataPoints[] result = new DataPoints[result_dp_groups.size()]; + for (DataPoints item : result_dp_groups) { + result[i++] = item; + } + return result; + } + + if (group_bys == null) { + // We haven't been asked to find groups, so let's put all the spans + // together in the same group. + final HistogramSpanGroup group = new HistogramSpanGroup(tsdb, + getScanStartTimeSeconds(), + getScanEndTimeSeconds(), + spans.values(), + HistogramAggregation.SUM, // only SUM is applicable for histogram metric + downsampler, + getStartTime(), + getEndTime(), + query_index, + RollupQuery.isValidQuery(rollup_query), + query_tags); + if (query_stats != null) { + query_stats.addStat(query_index, QueryStat.GROUP_BY_TIME, + (System.nanoTime() - group_build)); + } + + // create histogram data points to data points adaptor for each percentile calculation + if (null != percentiles && percentiles.size() > 0) { + List percentile_datapoints_list = generateHistogramPercentileDataPoints(group); + if (null != percentile_datapoints_list && percentile_datapoints_list.size() > 0) + result_dp_groups.addAll(percentile_datapoints_list); + } + + // create bucket metric + if (show_histogram_buckets) { + List bucket_datapoints_list = generateHistogramBucketDataPoints(group); + if (null != bucket_datapoints_list && bucket_datapoints_list.size() > 0) { + result_dp_groups.addAll(bucket_datapoints_list); + } + } + + int i = 0; + DataPoints[] result = new DataPoints[result_dp_groups.size()]; + for (DataPoints item : result_dp_groups) { + result[i++] = item; + } + return result; + } + + // Maps group value IDs to the SpanGroup for those values. Say we've + // been asked to group by two things: foo=* bar=* Then the keys in this + // map will contain all the value IDs combinations we've seen. If the + // name IDs for `foo' and `bar' are respectively [0, 0, 7] and [0, 0, 2] + // then we'll have group_bys=[[0, 0, 2], [0, 0, 7]] (notice it's sorted + // by ID, so bar is first) and say we find foo=LOL bar=OMG as well as + // foo=LOL bar=WTF and that the IDs of the tag values are: + // LOL=[0, 0, 1] OMG=[0, 0, 4] WTF=[0, 0, 3] + // then the map will have two keys: + // - one for the LOL-OMG combination: [0, 0, 1, 0, 0, 4] and, + // - one for the LOL-WTF combination: [0, 0, 1, 0, 0, 3]. + final ByteMap groups = new ByteMap(); + final short value_width = tsdb.tag_values.width(); + final byte[] group = new byte[group_bys.size() * value_width]; + for (final Map.Entry entry : spans.entrySet()) { + final byte[] row = entry.getKey(); + byte[] value_id = null; + int i = 0; + // TODO(tsuna): The following loop has a quadratic behavior. We can + // make it much better since both the row key and group_bys are sorted. + for (final byte[] tag_id : group_bys) { + value_id = Tags.getValueId(tsdb, row, tag_id); + if (value_id == null) { + break; + } + System.arraycopy(value_id, 0, group, i, value_width); + i += value_width; + } + if (value_id == null) { + LOG.error("WTF? Dropping span for row " + Arrays.toString(row) + + " as it had no matching tag from the requested groups," + + " which is unexpected. Query=" + this); + continue; + } + + //LOG.info("Span belongs to group " + Arrays.toString(group) + ": " + Arrays.toString(row)); + HistogramSpanGroup thegroup = groups.get(group); + if (thegroup == null) { + thegroup = new HistogramSpanGroup(tsdb, + getScanStartTimeSeconds(), + getScanEndTimeSeconds(), + null, + HistogramAggregation.SUM, // only SUM is applicable for histogram metric + downsampler, + getStartTime(), + getEndTime(), + query_index, + RollupQuery.isValidQuery(rollup_query), + query_tags); + + // Copy the array because we're going to keep `group' and overwrite + // its contents. So we want the collection to have an immutable copy. + final byte[] group_copy = new byte[group.length]; + System.arraycopy(group, 0, group_copy, 0, group.length); + groups.put(group_copy, thegroup); + } + thegroup.add(entry.getValue()); + } + + if (query_stats != null) { + query_stats.addStat(query_index, QueryStat.GROUP_BY_TIME, + (System.nanoTime() - group_build)); + } + + + for (final Map.Entry entry : groups.entrySet()) { + // create histogram data points to data points adaptor for each percentile calculation + if (null != percentiles && percentiles.size() > 0) { + List percentile_datapoints_list = generateHistogramPercentileDataPoints(entry.getValue()); + if (null != percentile_datapoints_list && percentile_datapoints_list.size() > 0) + result_dp_groups.addAll(percentile_datapoints_list); + } + + // create bucket metric + if (show_histogram_buckets) { + List bucket_datapoints_list = generateHistogramBucketDataPoints(entry.getValue()); + if (null != bucket_datapoints_list && bucket_datapoints_list.size() > 0) { + result_dp_groups.addAll(bucket_datapoints_list); + } + } + } // end for + + int i = 0; + DataPoints[] result = new DataPoints[result_dp_groups.size()]; + for (DataPoints item : result_dp_groups) { + result[i++] = item; + } + return result; + } + + private List generateHistogramPercentileDataPoints(final HistogramSpanGroup group) { + ArrayList result_dp_groups = new ArrayList(); + for (final Float percentil : percentiles) { + final HistogramDataPointsToDataPointsAdaptor dp_adaptor = new HistogramDataPointsToDataPointsAdaptor(group, + percentil.floatValue()); + result_dp_groups.add(dp_adaptor); + } // end for + + return result_dp_groups; + } + + private List generateHistogramBucketDataPoints(final HistogramSpanGroup group) { + ArrayList result_dp_groups = new ArrayList(); + try { + HistogramSeekableView seek_view = group.iterator(); + if (seek_view.hasNext()) { + HistogramDataPoint hdp = seek_view.next(); + Map buckets = hdp.getHistogramBucketsIfHas(); + if (null != buckets) { + for (Map.Entry bucket : buckets.entrySet()) { + final HistogramBucketDataPointsAdaptor dp_bucket_adaptor = new HistogramBucketDataPointsAdaptor(group, bucket.getKey()); + result_dp_groups.add(dp_bucket_adaptor); + } // end for + } // end if + } + } catch (UnsupportedOperationException e) { + // Just Ignore + } + + return result_dp_groups; + } + } + + /** + * Scan the tables again with the next best rollup match, on empty result set + */ + private class FallbackRollupOnEmptyResult implements + Callback, DataPoints[]>{ + + /** + * Creates the {@link SpanGroup}s to form the final results of this query. + * @param spans The {@link Span}s found for this query ({@link #findSpans}). + * Can be {@code null}, in which case the array returned will be empty. + * @return A possibly empty array of {@link SpanGroup}s built according to + * any 'GROUP BY' formulated in this query. + */ + public Deferred call(final DataPoints[] datapoints) throws Exception { + //TODO review this logic during spatial aggregation implementation + + if (datapoints == NO_RESULT && RollupQuery.isValidQuery(rollup_query)) { + //There are no datapoints for this query and it is a rollup query + //but not the default interval (default interval means raw). + //This will prevent redundant scan on raw data on the presense of + //default rollup interval + + //If the rollup usage is to fallback directly to raw data + //then nullyfy the rollup query, so that the recursive scan will use + //raw data and this will not called again because of isValida check + //If the rollup usage is to fallback to next best match then pupup + //next best match and attach that to the rollup query + if (rollup_usage == ROLLUP_USAGE.ROLLUP_FALLBACK_RAW) { + transformRollupQueryToDownSampler(); + return runAsync(); + } + else if (best_match_rollups != null && best_match_rollups.size() > 0) { + RollupInterval interval = best_match_rollups.remove(0); + + if (interval.isDefaultInterval()) { + transformRollupQueryToDownSampler(); + } + else { + rollup_query = new RollupQuery(interval, + rollup_query.getRollupAgg(), + rollup_query.getSampleIntervalInMS(), + aggregator); + //Here the requested sampling rate will be higher than + //resulted result. So downsample it + if (!rollup_query.isLowerSamplingRate()) { +// sample_interval_ms = rollup_query.getSampleIntervalInMS(); +// downsampler = rollup_query.getRollupAgg(); + // TODO - default fill + downsampler = new DownsamplingSpecification( + rollup_query.getSampleIntervalInMS(), + rollup_query.getRollupAgg(), + (downsampler != null ? downsampler.getFillPolicy() : + FillPolicy.ZERO)); + } + } + + return runAsync(); + } + return Deferred.fromResult(NO_RESULT); + } + else { + return Deferred.fromResult(datapoints); + } + } + } + /** * Returns a scanner set for the given metric (from {@link #metric} or from * the first TSUID in the {@link #tsuids}s list. If one or more tags are @@ -536,83 +1431,249 @@ public DataPoints[] call(final TreeMap spans) throws Exception { * @return A scanner to use for fetching data points */ protected Scanner getScanner() throws HBaseException { + return getScanner(0); + } + + /** + * Returns a scanner set for the given metric (from {@link #metric} or from + * the first TSUID in the {@link #tsuids}s list. If one or more tags are + * provided, it calls into {@link #createAndSetFilter} to setup a row key + * filter. If one or more TSUIDs have been provided, it calls into + * {@link #createAndSetTSUIDFilter} to setup a row key filter. + * @param salt_bucket The salt bucket to scan over when salting is enabled. + * @return A scanner to use for fetching data points + */ + protected Scanner getScanner(final int salt_bucket) throws HBaseException { final short metric_width = tsdb.metrics.width(); - final byte[] start_row = new byte[metric_width + Const.TIMESTAMP_BYTES]; - final byte[] end_row = new byte[metric_width + Const.TIMESTAMP_BYTES]; + + // set the metric UID based on the TSUIDs if given, or the metric UID + if (tsuids != null && !tsuids.isEmpty()) { + final String tsuid = tsuids.get(0); + final String metric_uid = tsuid.substring(0, metric_width * 2); + metric = UniqueId.stringToUid(metric_uid); + } + + final boolean is_rollup = RollupQuery.isValidQuery(rollup_query); + // We search at least one row before and one row after the start & end // time we've been given as it's quite likely that the exact timestamp // we're looking for is in the middle of a row. Plus, a number of things // rely on having a few extra data points before & after the exact start // & end dates in order to do proper rate calculation or downsampling near // the "edges" of the graph. - Bytes.setInt(start_row, (int) getScanStartTimeSeconds(), metric_width); - Bytes.setInt(end_row, (end_time == UNSET - ? -1 // Will scan until the end (0xFFF...). - : (int) getScanEndTimeSeconds()), - metric_width); - - // set the metric UID based on the TSUIDs if given, or the metric UID - if (tsuids != null && !tsuids.isEmpty()) { - final String tsuid = tsuids.get(0); - final String metric_uid = tsuid.substring(0, TSDB.metrics_width() * 2); - metric = UniqueId.stringToUid(metric_uid); - System.arraycopy(metric, 0, start_row, 0, metric_width); - System.arraycopy(metric, 0, end_row, 0, metric_width); - } else { - System.arraycopy(metric, 0, start_row, 0, metric_width); - System.arraycopy(metric, 0, end_row, 0, metric_width); + final Scanner scanner = QueryUtil.getMetricScanner(tsdb, salt_bucket, metric, + (int) getScanStartTimeSeconds(), end_time == UNSET + ? -1 // Will scan until the end (0xFFF...). + : (int) getScanEndTimeSeconds(), + tableToBeScanned(), + TSDB.FAMILY()); + if(tsdb.getConfig().use_otsdb_timestamp()) { + long stTime = (getScanStartTimeSeconds() * 1000); + long endTime = end_time == UNSET ? -1 : (getScanEndTimeSeconds() * 1000); + if (tsdb.getConfig().get_date_tiered_compaction_start() <= stTime && + rollup_query == null) { + // TODO - we could set this for rollups but we also need to write + // the rollup columns at the proper time. + scanner.setTimeRange(stTime, endTime); + } } - - final Scanner scanner = tsdb.client.newScanner(tsdb.table); - scanner.setStartKey(start_row); - scanner.setStopKey(end_row); if (tsuids != null && !tsuids.isEmpty()) { createAndSetTSUIDFilter(scanner); - } else if (tags.size() > 0 || group_bys != null) { + } else if (filters.size() > 0) { createAndSetFilter(scanner); } - scanner.setFamily(TSDB.FAMILY); + + if (is_rollup) { + ScanFilter existing = scanner.getFilter(); + // TODO - need some UTs around this! + // Set the Scanners column qualifier pattern with rollup aggregator + // HBase allows only a single filter so if we have a row key filter, keep + // it. If not, then we can do this + if (!rollup_query.getRollupAgg().toString().equals("avg")) { + if (existing != null) { + final List filters = new ArrayList(2); + filters.add(existing); + final List rollup_filters = new ArrayList(2); + rollup_filters.add(new QualifierFilter(CompareFilter.CompareOp.EQUAL, + new BinaryPrefixComparator(rollup_query.getRollupAgg().toString() + .getBytes(Const.ASCII_CHARSET)))); + rollup_filters.add(new QualifierFilter(CompareFilter.CompareOp.EQUAL, + new BinaryPrefixComparator(new byte[] { + (byte) tsdb.getRollupConfig().getIdForAggregator( + rollup_query.getRollupAgg().toString()) + }))); + filters.add(new FilterList(rollup_filters, Operator.MUST_PASS_ONE)); + scanner.setFilter(new FilterList(filters, Operator.MUST_PASS_ALL)); + } else { + final List filters = new ArrayList(2); + filters.add(new QualifierFilter(CompareFilter.CompareOp.EQUAL, + new BinaryPrefixComparator(rollup_query.getRollupAgg().toString() + .getBytes(Const.ASCII_CHARSET)))); + filters.add(new QualifierFilter(CompareFilter.CompareOp.EQUAL, + new BinaryPrefixComparator(new byte[] { + (byte) tsdb.getRollupConfig().getIdForAggregator( + rollup_query.getRollupAgg().toString()) + }))); + scanner.setFilter(new FilterList(filters, Operator.MUST_PASS_ONE)); + } + } else { + final List filters = new ArrayList(4); + filters.add(new QualifierFilter(CompareFilter.CompareOp.EQUAL, + new BinaryPrefixComparator("sum".getBytes()))); + filters.add(new QualifierFilter(CompareFilter.CompareOp.EQUAL, + new BinaryPrefixComparator("count".getBytes()))); + filters.add(new QualifierFilter(CompareFilter.CompareOp.EQUAL, + new BinaryPrefixComparator(new byte[] { + (byte) tsdb.getRollupConfig().getIdForAggregator("sum") + }))); + filters.add(new QualifierFilter(CompareFilter.CompareOp.EQUAL, + new BinaryPrefixComparator(new byte[] { + (byte) tsdb.getRollupConfig().getIdForAggregator("count") + }))); + + if (existing != null) { + final List combined = new ArrayList(2); + combined.add(existing); + combined.add(new FilterList(filters, Operator.MUST_PASS_ONE)); + scanner.setFilter(new FilterList(combined, Operator.MUST_PASS_ALL)); + } else { + scanner.setFilter(new FilterList(filters, Operator.MUST_PASS_ONE)); + } + } + } return scanner; } + /** + * Identify the table to be scanned based on the roll up and pre-aggregate + * query parameters + * @return table name as byte array + * @since 2.4 + */ + private byte[] tableToBeScanned() { + final byte[] tableName; + + if (RollupQuery.isValidQuery(rollup_query)) { + if (pre_aggregate) { + tableName= rollup_query.getRollupInterval().getGroupbyTable(); + } + else { + tableName= rollup_query.getRollupInterval().getTemporalTable(); + } + } + else if (pre_aggregate) { + tableName = tsdb.getDefaultInterval().getGroupbyTable(); + } + else { + tableName = tsdb.dataTable(); + } + + return tableName; + } + /** Returns the UNIX timestamp from which we must start scanning. */ - private long getScanStartTimeSeconds() { - // The reason we look before by `MAX_TIMESPAN * 2' seconds is because of - // the following. Let's assume MAX_TIMESPAN = 600 (10 minutes) and the - // start_time = ... 12:31:00. If we initialize the scanner to look - // only 10 minutes before, we'll start scanning at time=12:21, which will - // give us the row that starts at 12:30 (remember: rows are always aligned - // on MAX_TIMESPAN boundaries -- so in this example, on 10m boundaries). - // But we need to start scanning at least 1 row before, so we actually - // look back by twice MAX_TIMESPAN. Only when start_time is aligned on a - // MAX_TIMESPAN boundary then we'll mistakenly scan back by an extra row, - // but this doesn't really matter. - // Additionally, in case our sample_interval_ms is large, we need to look - // even further before/after, so use that too. + long getScanStartTimeSeconds() { + // Begin with the raw query start time. long start = getStartTime(); - // down cast to seconds if we have a query in ms - if ((start & Const.SECOND_MASK) != 0) { - start /= 1000; + + // Convert to seconds if we have a query in ms. + if ((start & Const.SECOND_MASK) != 0L) { + start /= 1000L; + } + + // if we have a rollup query, we have different row key start times so find + // the base time from which we need to search + if (rollup_query != null) { + long base_time = RollupUtils.getRollupBasetime(start, + rollup_query.getRollupInterval()); + if (rate) { + // scan one row back so we can get the first rate value. + base_time = RollupUtils.getRollupBasetime(base_time - 1, + rollup_query.getRollupInterval()); + } + return base_time; + } + + // First, we align the start timestamp to its representative value for the + // interval in which it appears, if downsampling. + long interval_aligned_ts = start; + if (downsampler != null && downsampler.getInterval() > 0) { + // Downsampling enabled. + // TODO - calendar interval + final long interval_offset = (1000L * start) % downsampler.getInterval(); + interval_aligned_ts -= interval_offset / 1000L; } - final long ts = start - Const.MAX_TIMESPAN * 2 - sample_interval_ms / 1000; - return ts > 0 ? ts : 0; + + // Then snap that timestamp back to its representative value for the + // timespan in which it appears. + final long timespan_offset = interval_aligned_ts % Const.MAX_TIMESPAN; + final long timespan_aligned_ts = interval_aligned_ts - timespan_offset; + + // Don't return negative numbers. + return timespan_aligned_ts > 0L ? timespan_aligned_ts : 0L; } /** Returns the UNIX timestamp at which we must stop scanning. */ - private long getScanEndTimeSeconds() { - // For the end_time, we have a different problem. For instance if our - // end_time = ... 12:30:00, we'll stop scanning when we get to 12:40, but - // once again we wanna try to look ahead one more row, so to avoid this - // problem we always add 1 second to the end_time. Only when the end_time - // is of the form HH:59:59 then we will scan ahead an extra row, but once - // again that doesn't really matter. - // Additionally, in case our sample_interval_ms is large, we need to look - // even further before/after, so use that too. + @VisibleForTesting + protected long getScanEndTimeSeconds() { + // Begin with the raw query end time. long end = getEndTime(); - if ((end & Const.SECOND_MASK) != 0) { - end /= 1000; + + // Convert to seconds if we have a query in ms. + if ((end & Const.SECOND_MASK) != 0L) { + end /= 1000L; + if (end - (end * 1000) < 1) { + // handle an edge case where a user may request a ms time between + // 0 and 1 seconds. Just bump it a second. + end++; + } + } + + if (rollup_query != null) { + return RollupUtils.getRollupBasetime(end + + (rollup_query.getRollupInterval().getIntervalSeconds() * + rollup_query.getRollupInterval().getIntervals()), + rollup_query.getRollupInterval()); + } + + // The calculation depends on whether we're downsampling. + if (downsampler != null && downsampler.getInterval() > 0) { + // Downsampling enabled. + // + // First, we align the end timestamp to its representative value for the + // interval FOLLOWING the one in which it appears. + // + // OpenTSDB's query bounds are inclusive, but HBase scan bounds are half- + // open. The user may have provided an end bound that is already + // interval-aligned (i.e., its interval offset is zero). If so, the user + // wishes for that interval to appear in the output. In that case, we + // skip forward an entire extra interval. + // + // This can be accomplished by simply not testing for zero offset. + final long interval_offset = (1000L * end) % downsampler.getInterval(); + final long interval_aligned_ts = end + + (downsampler.getInterval() - interval_offset) / 1000L; + + // Then, if we're now aligned on a timespan boundary, then we need no + // further adjustment: we are guaranteed to have always moved the end time + // forward, so the scan will find the data we need. + // + // Otherwise, we need to align to the NEXT timespan to ensure that we scan + // the needed data. + final long timespan_offset = interval_aligned_ts % Const.MAX_TIMESPAN; + return (0L == timespan_offset) ? + interval_aligned_ts : + interval_aligned_ts + (Const.MAX_TIMESPAN - timespan_offset); + } else { + // Not downsampling. + // + // Regardless of the end timestamp's position within the current timespan, + // we must always align to the beginning of the next timespan. This is + // true even if it's already aligned on a timespan boundary. Again, the + // reason for this is OpenTSDB's closed interval vs. HBase's half-open. + final long timespan_offset = end % Const.MAX_TIMESPAN; + return end + (Const.MAX_TIMESPAN - timespan_offset); } - return end + Const.MAX_TIMESPAN + 1 + sample_interval_ms / 1000; } /** @@ -622,66 +1683,13 @@ private long getScanEndTimeSeconds() { * @param scanner The scanner on which to add the filter. */ private void createAndSetFilter(final Scanner scanner) { - if (group_bys != null) { - Collections.sort(group_bys, Bytes.MEMCMP); - } - final short name_width = tsdb.tag_names.width(); - final short value_width = tsdb.tag_values.width(); - final short tagsize = (short) (name_width + value_width); - // Generate a regexp for our tags. Say we have 2 tags: { 0 0 1 0 0 2 } - // and { 4 5 6 9 8 7 }, the regexp will be: - // "^.{7}(?:.{6})*\\Q\000\000\001\000\000\002\\E(?:.{6})*\\Q\004\005\006\011\010\007\\E(?:.{6})*$" - final StringBuilder buf = new StringBuilder( - 15 // "^.{N}" + "(?:.{M})*" + "$" - + ((13 + tagsize) // "(?:.{M})*\\Q" + tagsize bytes + "\\E" - * (tags.size() + (group_bys == null ? 0 : group_bys.size() * 3)))); - // In order to avoid re-allocations, reserve a bit more w/ groups ^^^ - - // Alright, let's build this regexp. From the beginning... - buf.append("(?s)" // Ensure we use the DOTALL flag. - + "^.{") - // ... start by skipping the metric ID and timestamp. - .append(tsdb.metrics.width() + Const.TIMESTAMP_BYTES) - .append("}"); - final Iterator tags = this.tags.iterator(); - final Iterator group_bys = (this.group_bys == null - ? new ArrayList(0).iterator() - : this.group_bys.iterator()); - byte[] tag = tags.hasNext() ? tags.next() : null; - byte[] group_by = group_bys.hasNext() ? group_bys.next() : null; - // Tags and group_bys are already sorted. We need to put them in the - // regexp in order by ID, which means we just merge two sorted lists. - do { - // Skip any number of tags. - buf.append("(?:.{").append(tagsize).append("})*\\Q"); - if (isTagNext(name_width, tag, group_by)) { - addId(buf, tag); - tag = tags.hasNext() ? tags.next() : null; - } else { // Add a group_by. - addId(buf, group_by); - final byte[][] value_ids = (group_by_values == null - ? null - : group_by_values.get(group_by)); - if (value_ids == null) { // We don't want any specific ID... - buf.append(".{").append(value_width).append('}'); // Any value ID. - } else { // We want specific IDs. List them: /(AAA|BBB|CCC|..)/ - buf.append("(?:"); - for (final byte[] value_id : value_ids) { - buf.append("\\Q"); - addId(buf, value_id); - buf.append('|'); - } - // Replace the pipe of the last iteration. - buf.setCharAt(buf.length() - 1, ')'); - } - group_by = group_bys.hasNext() ? group_bys.next() : null; - } - } while (tag != group_by); // Stop when they both become null. - // Skip any number of tags before the end. - buf.append("(?:.{").append(tagsize).append("})*$"); - scanner.setKeyRegexp(buf.toString(), CHARSET); - } - + QueryUtil.setDataTableScanFilter(scanner, group_bys, row_key_literals, + explicit_tags, enable_fuzzy_filter, + (end_time == UNSET + ? -1 // Will scan until the end (0xFFF...). + : (int) getScanEndTimeSeconds())); + } + /** * Sets the server-side regexp filter on the scanner. * This will compile a list of the tagk/v pairs for the TSUIDs to prevent @@ -690,92 +1698,92 @@ private void createAndSetFilter(final Scanner scanner) { * @since 2.0 */ private void createAndSetTSUIDFilter(final Scanner scanner) { - Collections.sort(tsuids); - - // first, convert the tags to byte arrays and count up the total length - // so we can allocate the string builder - final short metric_width = tsdb.metrics.width(); - int tags_length = 0; - final ArrayList uids = new ArrayList(tsuids.size()); - for (final String tsuid : tsuids) { - final String tags = tsuid.substring(metric_width * 2); - final byte[] tag_bytes = UniqueId.stringToUid(tags); - tags_length += tag_bytes.length; - uids.add(tag_bytes); - } - - // Generate a regexp for our tags based on any metric and timestamp (since - // those are handled by the row start/stop) and the list of TSUID tagk/v - // pairs. The generated regex will look like: ^.{7}(tags|tags|tags)$ - // where each "tags" is similar to \\Q\000\000\001\000\000\002\\E - final StringBuilder buf = new StringBuilder( - 13 // "(?s)^.{N}(" + ")$" - + (tsuids.size() * 11) // "\\Q" + "\\E|" - + tags_length); // total # of bytes in tsuids tagk/v pairs - - // Alright, let's build this regexp. From the beginning... - buf.append("(?s)" // Ensure we use the DOTALL flag. - + "^.{") - // ... start by skipping the metric ID and timestamp. - .append(tsdb.metrics.width() + Const.TIMESTAMP_BYTES) - .append("}("); - - for (final byte[] tags : uids) { - // quote the bytes - buf.append("\\Q"); - addId(buf, tags); - buf.append('|'); + if (regex == null) { + regex = QueryUtil.getRowKeyTSUIDRegex(tsuids); } - - // Replace the pipe of the last iteration, close and set - buf.setCharAt(buf.length() - 1, ')'); - buf.append("$"); - scanner.setKeyRegexp(buf.toString(), CHARSET); + scanner.setKeyRegexp(regex, CHARSET); } /** - * Helper comparison function to compare tag name IDs. - * @param name_width Number of bytes used by a tag name ID. - * @param tag A tag (array containing a tag name ID and a tag value ID). - * @param group_by A tag name ID. - * @return {@code true} number if {@code tag} should be used next (because - * it contains a smaller ID), {@code false} otherwise. + * Return the query index that maps this datapoints to the original subquery + * @return index of the query in the TSQuery class + * @since 2.4 */ - private boolean isTagNext(final short name_width, - final byte[] tag, - final byte[] group_by) { - if (tag == null) { - return false; - } else if (group_by == null) { - return true; - } - final int cmp = Bytes.memcmp(tag, group_by, 0, name_width); - if (cmp == 0) { - throw new AssertionError("invariant violation: tag ID " - + Arrays.toString(group_by) + " is both in 'tags' and" - + " 'group_bys' in " + this); + @Override + public int getQueryIdx() { + return query_index; + } + + /** + * set the index that link this query to the original index. + * @param idx query index idx + * @since 2.4 + */ + public void setQueryIdx(int idx) { + query_index = idx; + } + + /** + * Transform downsampler properties to rollup properties, if the rollup + * is enabled at configuration level and down sampler is set. + * It falls back to raw data and down sampling if there is no + * RollupInterval is configured against this down sample interval + * @param group_by The group by aggregator. + * @param str_interval String representation of the interval, for logging + * @since 2.4 + */ + public void transformDownSamplerToRollupQuery(final Aggregator group_by, + final String str_interval) { + + if (downsampler != null && downsampler.getInterval() > 0) { + if (tsdb.getRollupConfig() != null) { + try { + best_match_rollups = tsdb.getRollupConfig(). + getRollupInterval(downsampler.getInterval() / 1000, str_interval); + //It is thread safe as each thread will be working on unique + // TsdbQuery object + //RollupConfig.getRollupInterval guarantees that, + // it always return a non-empty list + // TODO + rollup_query = new RollupQuery(best_match_rollups.remove(0), + downsampler.getFunction(), downsampler.getInterval(), + group_by); + } + catch (NoSuchRollupForIntervalException nre) { + LOG.error("There is no such rollup for the downsample interval " + + str_interval + ". So fall back to the default tsdb down" + + " sampling approach and it requires raw data scan." ); + //nullify the rollup_query if this api is called explicitly + rollup_query = null; + return; + } + + if (rollup_query.getRollupInterval().isDefaultInterval()) { + //Anyways it is a scan on raw data + rollup_query = null; + } + } } - return cmp < 0; } - + /** - * Appends the given ID to the given buffer, followed by "\\E". + * Transform rollup query to downsampler + * It is mainly useful when it scan on raw data on fallback. + * @since 2.4 */ - private static void addId(final StringBuilder buf, final byte[] id) { - boolean backslash = false; - for (final byte b : id) { - buf.append((char) (b & 0xFF)); - if (b == 'E' && backslash) { // If we saw a `\' and now we have a `E'. - // So we just terminated the quoted section because we just added \E - // to `buf'. So let's put a litteral \E now and start quoting again. - buf.append("\\\\E\\Q"); - } else { - backslash = b == '\\'; - } + private void transformRollupQueryToDownSampler() { + + if (rollup_query != null) { + // TODO - clean up and handle fill + downsampler = new DownsamplingSpecification( + rollup_query.getRollupInterval().getIntervalSeconds() * 1000, + rollup_query.getRollupAgg(), + (downsampler != null ? downsampler.getFillPolicy() : + FillPolicy.ZERO)); + rollup_query = null; } - buf.append("\\E"); } - + @Override public String toString() { final StringBuilder buf = new StringBuilder(); @@ -790,9 +1798,9 @@ public String toString() { } } else { buf.append(", metric=").append(Arrays.toString(metric)); - buf.append(", tags=["); - for (final Iterator it = tags.iterator(); it.hasNext(); ) { - buf.append(Arrays.toString(it.next())); + buf.append(", filters=["); + for (final Iterator it = filters.iterator(); it.hasNext(); ) { + buf.append(it.next()); if (it.hasNext()) { buf.append(','); } @@ -802,18 +1810,32 @@ public String toString() { .append(", group_bys=("); if (group_bys != null) { for (final byte[] tag_id : group_bys) { - buf.append(Arrays.toString(tag_id)); - if (group_by_values != null) { - final byte[][] value_ids = group_by_values.get(tag_id); + try { + buf.append(tsdb.tag_names.getName(tag_id)); + } catch (NoSuchUniqueId e) { + buf.append('<').append(e.getMessage()).append('>'); + } + buf.append(' ') + .append(Arrays.toString(tag_id)); + if (row_key_literals != null) { + final byte[][] value_ids = row_key_literals.get(tag_id); if (value_ids == null) { continue; } buf.append("={"); - for (int i = 0; i < value_ids.length; i++) { - buf.append(Arrays.toString(value_ids[i])); - if (i < value_ids.length - 1) { - buf.append(','); + for (final byte[] value_id : value_ids) { + try { + if (value_id != null) { + buf.append(tsdb.tag_values.getName(value_id)); + } else { + buf.append("null"); + } + } catch (NoSuchUniqueId e) { + buf.append('<').append(e.getMessage()).append('>'); } + buf.append(' ') + .append(Arrays.toString(value_id)) + .append(", "); } buf.append('}'); } @@ -821,14 +1843,25 @@ public String toString() { } } } - buf.append("))"); + buf.append(")") + .append(", rollup=") + .append(RollupQuery.isValidQuery(rollup_query)) + .append("))"); return buf.toString(); } + + public Boolean getNoResults() { + return no_results; + } + public void setNoResults(Boolean noResults) { + this.no_results = noResults; + } + /** * Comparator that ignores timestamps in row keys. */ - private static final class SpanCmp implements Comparator { + static final class SpanCmp implements Comparator { private final short metric_width; @@ -860,23 +1893,60 @@ public int compare(final byte[] a, final byte[] b) { } + RateOptions getRateOptions() { return rate_options; } + boolean isRate() { return rate; } + Aggregator getAggregator() {return aggregator; } + DownsamplingSpecification getDownsampler() { return downsampler; } + RollupQuery getRollupQuery() { return rollup_query; } + int getQueryIndex() { return query_index; } + + /** Helps unit tests inspect private methods. */ @VisibleForTesting static class ForTesting { /** @return the start time of the HBase scan for unit tests. */ - static long getScanStartTimeSeconds(TsdbQuery query) { + static long getScanStartTimeSeconds(final TsdbQuery query) { return query.getScanStartTimeSeconds(); } /** @return the end time of the HBase scan for unit tests. */ - static long getScanEndTimeSeconds(TsdbQuery query) { + static long getScanEndTimeSeconds(final TsdbQuery query) { return query.getScanEndTimeSeconds(); } /** @return the downsampling interval for unit tests. */ - static long getDownsampleIntervalMs(TsdbQuery query) { - return query.sample_interval_ms; + static long getDownsampleIntervalMs(final TsdbQuery query) { + return query.downsampler.getInterval(); + } + + static byte[] getMetric(final TsdbQuery query) { + return query.metric; } + + static RateOptions getRateOptions(final TsdbQuery query) { + return query.rate_options; + } + + static List getFilters(final TsdbQuery query) { + return query.filters; + } + + static ArrayList getGroupBys(final TsdbQuery query) { + return query.group_bys; + } + + static ByteMap getRowKeyLiterals(final TsdbQuery query) { + return query.row_key_literals; + } + + static long maxBytes(final TsdbQuery query) { + return query.max_bytes; + } + + static long maxDataPoints(final TsdbQuery query) { + return query.max_data_points; + } + } } diff --git a/src/core/WritableDataPoints.java b/src/core/WritableDataPoints.java index 671405263b..2623d8aaff 100644 --- a/src/core/WritableDataPoints.java +++ b/src/core/WritableDataPoints.java @@ -25,6 +25,13 @@ */ public interface WritableDataPoints extends DataPoints { + /** + * Perform a put to the database to store writable points into the data table. + *

    + * @return A deferred object to wait on for the results to be fetched. + */ + Deferred persist(); + /** * Sets the metric name and tags of the series. *

    diff --git a/src/core/WriteableDataPointFilterPlugin.java b/src/core/WriteableDataPointFilterPlugin.java new file mode 100644 index 0000000000..3202951ede --- /dev/null +++ b/src/core/WriteableDataPointFilterPlugin.java @@ -0,0 +1,118 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import java.util.Map; + +import com.stumbleupon.async.Deferred; + +import net.opentsdb.stats.StatsCollector; + +/** + * A filter that can determine whether or not time series should be allowed + * assignment based on their metric and tags. This is useful for such + * situations as: + *

    • Enforcing naming standards
      • + *
    • Blacklisting certain names or properties
      • + *
    • Preventing cardinality explosions
    + * Note: Implementations must have a parameterless constructor. The + * {@link #initialize(TSDB)} method will be called immediately after the plugin is + * instantiated and before any other methods are called. + * @since 2.3 + */ +public abstract class WriteableDataPointFilterPlugin { + + /** + * Called by TSDB to initialize the plugin + * Implementations are responsible for setting up any IO they need as well + * as starting any required background threads. + * Note: Implementations should throw exceptions if they can't start + * up properly. The TSD will then shutdown so the operator can fix the + * problem. Please use IllegalArgumentException for configuration issues. + * @param tsdb The parent TSDB object + * @throws IllegalArgumentException if required configuration parameters are + * missing + * @throws RuntimeException if something else goes wrong + */ + public abstract void initialize(final TSDB tsdb); + + /** + * Called to gracefully shutdown the plugin. Implementations should close + * any IO they have open + * @return A deferred object that indicates the completion of the request. + * The {@link Object} has not special meaning and can be {@code null} + * (think of it as {@code Deferred}). + */ + public abstract Deferred shutdown(); + + /** + * Should return the version of this plugin in the format: + * MAJOR.MINOR.MAINT, e.g. "2.3.1". The MAJOR version should match the major + * version of OpenTSDB the plugin is meant to work with. + * @return A version string used to log the loaded version + */ + public abstract String version(); + + /** + * Called by the TSD when a request for statistics collection has come in. The + * implementation may provide one or more statistics. If no statistics are + * available for the implementation, simply stub the method. + * @param collector The collector used for emitting statistics + */ + public abstract void collectStats(final StatsCollector collector); + + /** + * Determine whether or not the data point should be stored. + * If the data should not be stored, the implementation can return false or an + * exception in the deferred object. Otherwise it should return true and the + * data point will be written to storage. + * @param metric The metric name for the data point + * @param timestamp The timestamp of the data + * @param value The value encoded as either an integer or floating point value + * @param tags The tags associated with the data point + * @param flags Encoding flags for the value + * @return True if the data should be written, false if it should be rejected. + */ + public abstract Deferred allowDataPoint( + final String metric, + final long timestamp, + final byte[] value, + final Map tags, + final short flags); + + /** + * Determine whether or not the data point should be stored. + * If the data should not be stored, the implementation can return false or an + * exception in the deferred object. Otherwise it should return true and the + * data point will be written to storage. + * @param metric The metric name for the data point + * @param timestamp The timestamp of the data + * @param value The value encoded as either an integer or floating point value + * @param tags The tags associated with the data point + * @return True if the data should be written, false if it should be rejected. + */ + public Deferred allowHistogramPoint( + final String metric, + final long timestamp, + final byte[] value, + final Map tags) { + throw new UnsupportedOperationException("Not yet implemented."); + } + + /** + * Whether or not the filter should process data points. + * @return False if {@link #allowDataPoint(String, long, byte[], Map, short)} + * should NOT be called, true if it should. + */ + public abstract boolean filterDataPoints(); +} diff --git a/src/core/iHistogramRowSeq.java b/src/core/iHistogramRowSeq.java new file mode 100644 index 0000000000..d61469189d --- /dev/null +++ b/src/core/iHistogramRowSeq.java @@ -0,0 +1,66 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import java.util.List; + +/** + * Clone of the {@link iRowSeq} interface but for histograms. + * + * @since 2.4 + */ +public interface iHistogramRowSeq extends HistogramDataPoints { + + /** + * Sets the initial column in the sequence. The key cannot be empty. + * @param key The row key. + * @param row A non-null list of histogram points. + * @throws IllegalStateException if {@link #setRow(byte[], List)} or + * {@link #addRow(List)} has already been called. + */ + public void setRow(final byte[] key, final List row); + + /** + * Adds a column in the proper sequence in the row. Must be called after + * {@link #setRow(byte[], List)} has been called. + * @param row A non-null list of histogram points. + * @throws IllegalStateException if {@link #setRow(byte[], List)} has not been + * called first. + */ + public void addRow(final List row); + + /** + * Returns the row key this sequence represents. May be null if + * {@link #setRow(byte[], List)} has not been called. + * @return The row key for this sequence. + */ + public byte[] key(); + + /** + * Returns the base time for the row in Unix epoch seconds. + * @return The base time for the row. + * @throws NullPointerException if {@link #setRow(byte[], List)} has not been + * called. + */ + public long baseTime(); + + /** @return an internal iterator for this row sequence. */ + public Iterator internalIterator(); + + /** + * An interface for an iterator that all row sequences must implement. + */ + public interface Iterator extends HistogramSeekableView, HistogramDataPoint { + + } +} diff --git a/src/core/iRowSeq.java b/src/core/iRowSeq.java new file mode 100644 index 0000000000..5e01d56dcc --- /dev/null +++ b/src/core/iRowSeq.java @@ -0,0 +1,66 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.core; + +import org.hbase.async.KeyValue; + +/** + * An interface that defines methods implemented by row sequence implementations + * that represent a row of data in storage. + * @since 2.4 + */ +public interface iRowSeq extends DataPoints { + + /** + * Sets the initial column in the sequence. The row must be empty and this + * must be called before {@link #addRow(KeyValue)}. + * @param row A non-null KeyValue representing a column in the row. + * @throws IllegalStateException if {@link #setRow(KeyValue)} or + * {@link #addRow(KeyValue)} has already been called. + */ + public void setRow(final KeyValue row); + + /** + * Adds a column in the proper sequence in the row. Must be called after + * {@link #setRow(KeyValue)} has been called. + * @param row A non-null KeyValue representing a column in the row. + * @throws IllegalStateException if {@link #setRow(KeyValue)} has not been + * called first. + */ + public void addRow(final KeyValue row); + + /** + * Returns the row key this sequence represents. May be null if + * {@link #setRow(KeyValue)} has not been called. + * @return The row key for this sequence. + */ + public byte[] key(); + + /** + * Returns the base time for the row in Unix epoch seconds. + * @return The base time for the row. + * @throws NullPointerException if {@link #setRow(KeyValue)} has not been + * called. + */ + public long baseTime(); + + /** @return an internal iterator for this row sequence. */ + public Iterator internalIterator(); + + /** + * An interface for an iterator that all row sequences must implement. + */ + public interface Iterator extends SeekableView, DataPoint { + + } +} diff --git a/src/create_table.sh b/src/create_table.sh index ad01f623c6..1cbe666319 100755 --- a/src/create_table.sh +++ b/src/create_table.sh @@ -19,6 +19,11 @@ BLOOMFILTER=${BLOOMFILTER-'ROW'} COMPRESSION=${COMPRESSION-'LZO'} # All compression codec names are upper case (NONE, LZO, SNAPPY, etc). COMPRESSION=`echo "$COMPRESSION" | tr a-z A-Z` +# DIFF encoding is very useful for OpenTSDB's case that many small KVs and common prefix. +# This can save a lot of storage space. +DATA_BLOCK_ENCODING=${DATA_BLOCK_ENCODING-'DIFF'} +DATA_BLOCK_ENCODING=`echo "$DATA_BLOCK_ENCODING" | tr a-z A-Z` +TSDB_TTL=${TSDB_TTL-'FOREVER'} case $COMPRESSION in (NONE|LZO|GZIP|SNAPPY) :;; # Known good. @@ -27,6 +32,13 @@ case $COMPRESSION in ;; esac +case $DATA_BLOCK_ENCODING in + (NONE|PREFIX|DIFF|FAST_DIFF|ROW_INDEX_V1) :;; # Know good + (*) + echo >&2 "warning: encoding '$DATA_BLOCK_ENCODING' might not be supported." + ;; +esac + # HBase scripts also use a variable named `HBASE_HOME', and having this # variable in the environment with a value somewhat different from what # they expect can confuse them in some cases. So rename the variable. @@ -34,15 +46,15 @@ hbh=$HBASE_HOME unset HBASE_HOME exec "$hbh/bin/hbase" shell < 'id', COMPRESSION => '$COMPRESSION', BLOOMFILTER => '$BLOOMFILTER'}, - {NAME => 'name', COMPRESSION => '$COMPRESSION', BLOOMFILTER => '$BLOOMFILTER'} + {NAME => 'id', COMPRESSION => '$COMPRESSION', BLOOMFILTER => '$BLOOMFILTER', DATA_BLOCK_ENCODING => '$DATA_BLOCK_ENCODING'}, + {NAME => 'name', COMPRESSION => '$COMPRESSION', BLOOMFILTER => '$BLOOMFILTER', DATA_BLOCK_ENCODING => '$DATA_BLOCK_ENCODING'} create '$TSDB_TABLE', - {NAME => 't', VERSIONS => 1, COMPRESSION => '$COMPRESSION', BLOOMFILTER => '$BLOOMFILTER'} + {NAME => 't', VERSIONS => 1, COMPRESSION => '$COMPRESSION', BLOOMFILTER => '$BLOOMFILTER', DATA_BLOCK_ENCODING => '$DATA_BLOCK_ENCODING', TTL => '$TSDB_TTL'} create '$TREE_TABLE', - {NAME => 't', VERSIONS => 1, COMPRESSION => '$COMPRESSION', BLOOMFILTER => '$BLOOMFILTER'} + {NAME => 't', VERSIONS => 1, COMPRESSION => '$COMPRESSION', BLOOMFILTER => '$BLOOMFILTER', DATA_BLOCK_ENCODING => '$DATA_BLOCK_ENCODING'} create '$META_TABLE', - {NAME => 'name', COMPRESSION => '$COMPRESSION', BLOOMFILTER => '$BLOOMFILTER'} + {NAME => 'name', COMPRESSION => '$COMPRESSION', BLOOMFILTER => '$BLOOMFILTER', DATA_BLOCK_ENCODING => '$DATA_BLOCK_ENCODING'} EOF diff --git a/src/examples/AddDataExample.java b/src/examples/AddDataExample.java new file mode 100644 index 0000000000..7c16c69ac3 --- /dev/null +++ b/src/examples/AddDataExample.java @@ -0,0 +1,141 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . + +package net.opentsdb.examples; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import com.stumbleupon.async.Callback; +import com.stumbleupon.async.Deferred; + +import net.opentsdb.core.TSDB; +import net.opentsdb.uid.NoSuchUniqueName; +import net.opentsdb.uid.UniqueId.UniqueIdType; +import net.opentsdb.utils.Config; + +/** + * Examples for how to add points to the tsdb. + * + */ +public class AddDataExample { + private static String pathToConfigFile; + + public static void processArgs(final String[] args) { + // Set these as arguments so you don't have to keep path information in + // source files + if (args != null && args.length > 0) { + pathToConfigFile = args[0]; + } + } + + public static void main(final String[] args) throws Exception { + processArgs(args); + + // Create a config object with a path to the file for parsing. Or manually + // override settings. + // e.g. config.overrideConfig("tsd.storage.hbase.zk_quorum", "localhost"); + final Config config; + if (pathToConfigFile != null && !pathToConfigFile.isEmpty()) { + config = new Config(pathToConfigFile); + } else { + // Search for a default config from /etc/opentsdb/opentsdb.conf, etc. + config = new Config(true); + } + final TSDB tsdb = new TSDB(config); + + // Declare new metric + String metricName = "my.tsdb.test.metric"; + // First check to see it doesn't already exist + byte[] byteMetricUID; // we don't actually need this for the first + // .addPoint() call below. + // TODO: Ideally we could just call a not-yet-implemented tsdb.uIdExists() + // function. + // Note, however, that this is optional. If auto metric is enabled + // (tsd.core.auto_create_metrics), the UID will be assigned in call to + // addPoint(). + try { + byteMetricUID = tsdb.getUID(UniqueIdType.METRIC, metricName); + } catch (IllegalArgumentException iae) { + System.out.println("Metric name not valid."); + iae.printStackTrace(); + System.exit(1); + } catch (NoSuchUniqueName nsune) { + // If not, great. Create it. + byteMetricUID = tsdb.assignUid("metric", metricName); + } + + // Make a single datum + long timestamp = System.currentTimeMillis() / 1000; + long value = 314159; + // Make key-val + Map tags = new HashMap(1); + tags.put("script", "example1"); + + // Start timer + long startTime1 = System.currentTimeMillis(); + + // Write a number of data points at 30 second intervals. Each write will + // return a deferred (similar to a Java Future or JS Promise) that will + // be called on completion with either a "null" value on success or an + // exception. + int n = 100; + ArrayList> deferreds = new ArrayList>(n); + for (int i = 0; i < n; i++) { + Deferred deferred = tsdb.addPoint(metricName, timestamp, value + i, tags); + deferreds.add(deferred); + timestamp += 30; + } + + // Add the callbacks to the deferred object. (They might have already + // returned, btw) + // This will cause the calling thread to wait until the add has completed. + System.out.println("Waiting for deferred result to return..."); + Deferred.groupInOrder(deferreds) + .addErrback(new AddDataExample().new errBack()) + .addCallback(new AddDataExample().new succBack()) + // Block the thread until the deferred returns it's result. + .join(); + // Alternatively you can add another callback here or use a join with a + // timeout argument. + + // End timer. + long elapsedTime1 = System.currentTimeMillis() - startTime1; + System.out.println("\nAdding " + n + " points took: " + elapsedTime1 + + " milliseconds.\n"); + + // Gracefully shutdown connection to TSDB. This is CRITICAL as it will + // flush any pending operations to HBase. + tsdb.shutdown().join(); + } + + // This is an optional errorback to handle when there is a failure. + class errBack implements Callback { + public String call(final Exception e) throws Exception { + String message = ">>>>>>>>>>>Failure!>>>>>>>>>>>"; + System.err.println(message + " " + e.getMessage()); + e.printStackTrace(); + return message; + } + }; + + // This is an optional success callback to handle when there is a success. + class succBack implements Callback> { + public Object call(final ArrayList results) { + System.out.println("Successfully wrote " + results.size() + " data points"); + return null; + } + }; + +} diff --git a/src/examples/QueryExample.java b/src/examples/QueryExample.java new file mode 100644 index 0000000000..d03b23ea7c --- /dev/null +++ b/src/examples/QueryExample.java @@ -0,0 +1,198 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.examples; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import com.stumbleupon.async.Callback; +import com.stumbleupon.async.Deferred; + +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.Query; +import net.opentsdb.core.SeekableView; +import net.opentsdb.core.TSDB; +import net.opentsdb.core.TSQuery; +import net.opentsdb.core.TSSubQuery; +import net.opentsdb.query.filter.TagVFilter; +import net.opentsdb.utils.Config; +import net.opentsdb.utils.DateTime; + +/** + * One example on how to query. + * Taken from + * + * this thread + * The metric and key query arguments assume that you've input data from the Quick Start tutorial + * here. + */ +public class QueryExample { + + public static void main(final String[] args) throws IOException { + + // Set these as arguments so you don't have to keep path information in + // source files + String pathToConfigFile = (args != null && args.length > 0 ? args[0] : null); + + // Create a config object with a path to the file for parsing. Or manually + // override settings. + // e.g. config.overrideConfig("tsd.storage.hbase.zk_quorum", "localhost"); + final Config config; + if (pathToConfigFile != null && !pathToConfigFile.isEmpty()) { + config = new Config(pathToConfigFile); + } else { + // Search for a default config from /etc/opentsdb/opentsdb.conf, etc. + config = new Config(true); + } + final TSDB tsdb = new TSDB(config); + + // main query + final TSQuery query = new TSQuery(); + + // use any string format from + // http://opentsdb.net/docs/build/html/user_guide/query/dates.html + query.setStart("1h-ago"); + // Optional: set other global query params + + // at least one sub query required. This is where you specify the metric and + // tags + final TSSubQuery subQuery = new TSSubQuery(); + subQuery.setMetric("my.tsdb.test.metric"); + + // filters are optional but useful. + final List filters = new ArrayList(1); + filters.add(new TagVFilter.Builder() + .setType("literal_or") + .setFilter("example1") + .setTagk("script") + .setGroupBy(true) + .build()); + subQuery.setFilters(filters); + + // you do have to set an aggregator. Just provide the name as a string + subQuery.setAggregator("sum"); + + // IMPORTANT: don't forget to add the subQuery + final ArrayList subQueries = new ArrayList(1); + subQueries.add(subQuery); + query.setQueries(subQueries); + query.setMsResolution(true); // otherwise we aggregate on the second. + + // make sure the query is valid. This will throw exceptions if something + // is missing + query.validateAndSetQuery(); + + // compile the queries into TsdbQuery objects behind the scenes + Query[] tsdbqueries = query.buildQueries(tsdb); + + // create some arrays for storing the results and the async calls + final int nqueries = tsdbqueries.length; + final ArrayList results = new ArrayList( + nqueries); + final ArrayList> deferreds = + new ArrayList>(nqueries); + + // this executes each of the sub queries asynchronously and puts the + // deferred in an array so we can wait for them to complete. + for (int i = 0; i < nqueries; i++) { + deferreds.add(tsdbqueries[i].runAsync()); + } + + // Start timer + long startTime = DateTime.nanoTime(); + + // This is a required callback class to store the results after each + // query has finished + class QueriesCB implements Callback> { + public Object call(final ArrayList queryResults) + throws Exception { + results.addAll(queryResults); + return null; + } + } + + // Make sure to handle any errors that might crop up + class QueriesEB implements Callback { + @Override + public Object call(final Exception e) throws Exception { + System.err.println("Queries failed"); + e.printStackTrace(); + return null; + } + } + + // this will cause the calling thread to wait until ALL of the queries + // have completed. + try { + Deferred.groupInOrder(deferreds) + .addCallback(new QueriesCB()) + .addErrback(new QueriesEB()) + .join(); + } catch (Exception e) { + e.printStackTrace(); + } + + // End timer. + double elapsedTime = DateTime.msFromNanoDiff(DateTime.nanoTime(), startTime); + System.out.println("Query returned in: " + elapsedTime + " milliseconds."); + + // now all of the results are in so we just iterate over each set of + // results and do any processing necessary. + for (final DataPoints[] dataSets : results) { + for (final DataPoints data : dataSets) { + System.out.print(data.metricName()); + Map resolvedTags = data.getTags(); + for (final Map.Entry pair : resolvedTags.entrySet()) { + System.out.print(" " + pair.getKey() + "=" + pair.getValue()); + } + System.out.print("\n"); + + final SeekableView it = data.iterator(); + /* + * An important point about SeekableView: + * Because no data is copied during iteration and no new object gets + * created, the DataPoint returned must not be stored and gets + * invalidated as soon as next is called on the iterator (actually it + * doesn't get invalidated but rather its contents changes). If you want + * to store individual data points, you need to copy the timestamp and + * value out of each DataPoint into your own data structures. + * + * In the vast majority of cases, the iterator will be used to go once + * through all the data points, which is why it's not a problem if the + * iterator acts just as a transient "view". Iterating will be very + * cheap since no memory allocation is required (except to instantiate + * the actual iterator at the beginning). + */ + while (it.hasNext()) { + final DataPoint dp = it.next(); + System.out.println(" " + dp.timestamp() + " " + + (dp.isInteger() ? dp.longValue() : dp.doubleValue())); + } + System.out.println(""); + } + } + + // Gracefully shutdown connection to TSDB + try { + tsdb.shutdown().join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (Exception e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file diff --git a/src/graph/Plot.java b/src/graph/Plot.java index c24eeed72d..9ca14a8a52 100644 --- a/src/graph/Plot.java +++ b/src/graph/Plot.java @@ -1,5 +1,5 @@ // This file is part of OpenTSDB. -// Copyright (C) 2010-2012 The OpenTSDB Authors. +// Copyright (C) 2010-2021 The OpenTSDB Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU Lesser General Public License as published by @@ -134,6 +134,13 @@ public Plot(final long start_time, final long end_time, TimeZone tz) { * */ public void setParams(final Map params) { + // check "format y" and "format y2" + String[] y_format_keys = {"format y", "format y2"}; + for(String k : y_format_keys){ + if(params.containsKey(k)){ + params.put(k, params.get(k)); + } + } this.params = params; } @@ -204,23 +211,33 @@ public int dumpToFiles(final String basepath) throws IOException { final PrintWriter datafile = new PrintWriter(datafiles[i]); try { for (final DataPoint d : datapoints.get(i)) { - final long ts = d.timestamp() / 1000; - if (ts >= (start_time & UNSIGNED) && ts <= (end_time & UNSIGNED)) { - npoints++; - } - datafile.print(ts + utc_offset); - datafile.print(' '); + final long ts = d.timestamp() / 1000; if (d.isInteger()) { + datafile.print(ts + utc_offset); + datafile.print(' '); datafile.print(d.longValue()); } else { final double value = d.doubleValue(); - if (value != value || Double.isInfinite(value)) { - throw new IllegalStateException("NaN or Infinity found in" + + if (Double.isInfinite(value)) { + // Infinity is invalid. + throw new IllegalStateException("Infinity found in" + " datapoints #" + i + ": " + value + " d=" + d); + } else if (Double.isNaN(value)) { + // NaNs should be skipped. + continue; } + + datafile.print(ts + utc_offset); + datafile.print(' '); datafile.print(value); } + datafile.print('\n'); + + if (ts >= (start_time & UNSIGNED) && ts <= (end_time & UNSIGNED)) { + npoints++; + } } } finally { datafile.close(); @@ -258,6 +275,7 @@ private void writeGnuplotScript(final String basepath, .append(Short.toString(height)); final String smooth = params.remove("smooth"); final String fgcolor = params.remove("fgcolor"); + final String style = params.remove("style"); String bgcolor = params.remove("bgcolor"); if (fgcolor != null && bgcolor == null) { // We can't specify a fgcolor without specifying a bgcolor. @@ -292,7 +310,8 @@ private void writeGnuplotScript(final String basepath, final int nseries = datapoints.size(); if (nseries > 0) { gp.write("set grid\n" - + "set style data linespoints\n"); + + "set style data "); + gp.append(style != null? style : "linespoint").append("\n"); if (!params.containsKey("key")) { gp.write("set key right box\n"); } diff --git a/src/logback.xml b/src/logback.xml index b06776504a..49eff6d58e 100644 --- a/src/logback.xml +++ b/src/logback.xml @@ -8,15 +8,70 @@ + + 1024 + + + + /var/log/opentsdb/opentsdb.log + true + + /home/y/logs/opentsdb2/opentsdb.log.%i + 1 + 4 + + + 512MB + + + + %date{ISO8601} [%thread] %-5level [%logger{0}.%M] - %msg%n + + + + + + /var/log/opentsdb/queries.log + true - - - - + + /var/log/opentsdb/queries.log.%i + 1 + 4 + + + + 128MB + + + %date{ISO8601} [%logger.%M] %msg%n + + + + + + + + + + + + + + + + + + diff --git a/src/meta/Annotation.java b/src/meta/Annotation.java index 07762f3c9f..d00988aece 100644 --- a/src/meta/Annotation.java +++ b/src/meta/Annotation.java @@ -19,6 +19,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; import org.hbase.async.Bytes; import org.hbase.async.DeleteRequest; @@ -32,6 +33,8 @@ import net.opentsdb.core.Const; import net.opentsdb.core.Internal; +import net.opentsdb.core.RequestBuilder; +import net.opentsdb.core.RowKey; import net.opentsdb.core.TSDB; import net.opentsdb.uid.UniqueId; import net.opentsdb.utils.JSON; @@ -43,6 +46,7 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.core.JsonGenerator; +import com.google.common.annotations.VisibleForTesting; import com.stumbleupon.async.Callback; import com.stumbleupon.async.Deferred; @@ -72,7 +76,7 @@ @JsonAutoDetect(fieldVisibility = Visibility.PUBLIC_ONLY) @JsonInclude(Include.NON_NULL) @JsonIgnoreProperties(ignoreUnknown = true) -public final class Annotation implements Comparable { +public class Annotation implements Comparable { private static final Logger LOG = LoggerFactory.getLogger(Annotation.class); /** Charset used to convert Strings to byte arrays and back. */ @@ -181,10 +185,10 @@ public Deferred call(final Annotation stored_note) final byte[] tsuid_byte = tsuid != null && !tsuid.isEmpty() ? UniqueId.stringToUid(tsuid) : null; - final PutRequest put = new PutRequest(tsdb.dataTable(), + final PutRequest put = RequestBuilder.buildPutRequest(tsdb.getConfig(), tsdb.dataTable(), getRowKey(start_time, tsuid_byte), FAMILY, getQualifier(start_time), - Annotation.this.getStorageJSON()); + Annotation.this.getStorageJSON(), start_time); return tsdb.getClient().compareAndSet(put, original_note); } @@ -272,7 +276,6 @@ public Deferred call(final ArrayList row) if (row == null || row.isEmpty()) { return Deferred.fromResult(null); } - Annotation note = JSON.parseToObject(row.get(0).value(), Annotation.class); return Deferred.fromResult(note); @@ -322,9 +325,11 @@ final class ScannerCB implements Callback>, * Initializes the scanner */ public ScannerCB() { - final byte[] start = new byte[TSDB.metrics_width() + + final byte[] start = new byte[Const.SALT_WIDTH() + + TSDB.metrics_width() + Const.TIMESTAMP_BYTES]; - final byte[] end = new byte[TSDB.metrics_width() + + final byte[] end = new byte[Const.SALT_WIDTH() + + TSDB.metrics_width() + Const.TIMESTAMP_BYTES]; final long normalized_start = (start_time - @@ -332,8 +337,10 @@ public ScannerCB() { final long normalized_end = (end_time - (end_time % Const.MAX_TIMESPAN) + Const.MAX_TIMESPAN); - Bytes.setInt(start, (int) normalized_start, TSDB.metrics_width()); - Bytes.setInt(end, (int) normalized_end, TSDB.metrics_width()); + Bytes.setInt(start, (int) normalized_start, + Const.SALT_WIDTH() + TSDB.metrics_width()); + Bytes.setInt(end, (int) normalized_end, + Const.SALT_WIDTH() + TSDB.metrics_width()); scanner = tsdb.getClient().newScanner(tsdb.dataTable()); scanner.setStartKey(start); @@ -396,8 +403,9 @@ public static Deferred deleteRange(final TSDB tsdb, } final List> delete_requests = new ArrayList>(); - int width = tsuid != null ? tsuid.length + Const.TIMESTAMP_BYTES : - TSDB.metrics_width() + Const.TIMESTAMP_BYTES; + int width = tsuid != null ? + Const.SALT_WIDTH() + tsuid.length + Const.TIMESTAMP_BYTES : + Const.SALT_WIDTH() + TSDB.metrics_width() + Const.TIMESTAMP_BYTES; final byte[] start_row = new byte[width]; final byte[] end_row = new byte[width]; @@ -406,14 +414,16 @@ public static Deferred deleteRange(final TSDB tsdb, final long end = end_time / 1000; final long normalized_start = (start - (start % Const.MAX_TIMESPAN)); final long normalized_end = (end - (end % Const.MAX_TIMESPAN) + Const.MAX_TIMESPAN); - Bytes.setInt(start_row, (int) normalized_start, TSDB.metrics_width()); - Bytes.setInt(end_row, (int) normalized_end, TSDB.metrics_width()); + Bytes.setInt(start_row, (int) normalized_start, + Const.SALT_WIDTH() + TSDB.metrics_width()); + Bytes.setInt(end_row, (int) normalized_end, + Const.SALT_WIDTH() + TSDB.metrics_width()); if (tsuid != null) { // first copy the metric UID then the tags - System.arraycopy(tsuid, 0, start_row, 0, TSDB.metrics_width()); - System.arraycopy(tsuid, 0, end_row, 0, TSDB.metrics_width()); - width = TSDB.metrics_width() + Const.TIMESTAMP_BYTES; + System.arraycopy(tsuid, 0, start_row, Const.SALT_WIDTH(), TSDB.metrics_width()); + System.arraycopy(tsuid, 0, end_row, Const.SALT_WIDTH(), TSDB.metrics_width()); + width = Const.SALT_WIDTH() + TSDB.metrics_width() + Const.TIMESTAMP_BYTES; final int remainder = tsuid.length - TSDB.metrics_width(); System.arraycopy(tsuid, TSDB.metrics_width(), start_row, width, remainder); System.arraycopy(tsuid, TSDB.metrics_width(), end_row, width, remainder); @@ -504,7 +514,8 @@ public static byte PREFIX() { * successful CAS calls * @return The serialized object as a byte array */ - private byte[] getStorageJSON() { + @VisibleForTesting + byte[] getStorageJSON() { // TODO - precalculate size final ByteArrayOutputStream output = new ByteArrayOutputStream(); try { @@ -520,11 +531,9 @@ private byte[] getStorageJSON() { if (custom == null) { json.writeNullField("custom"); } else { - json.writeObjectFieldStart("custom"); - for (Map.Entry entry : custom.entrySet()) { - json.writeStringField(entry.getKey(), entry.getValue()); - } - json.writeEndObject(); + final TreeMap sorted_custom = + new TreeMap(custom); + json.writeObjectField("custom", sorted_custom); } json.writeEndObject(); @@ -661,19 +670,24 @@ private static byte[] getRowKey(final long start_time, final byte[] tsuid) { } // if the TSUID is empty, then we're a global annotation. The row key will - // just be an empty byte array of metric width plus the timestamp + // just be an empty byte array of metric width plus the timestamp. We also + // don't salt the global row key (though it has space for salts) if (tsuid == null || tsuid.length < 1) { - final byte[] row = new byte[TSDB.metrics_width() + Const.TIMESTAMP_BYTES]; - Bytes.setInt(row, (int) base_time, TSDB.metrics_width()); + final byte[] row = new byte[Const.SALT_WIDTH() + + TSDB.metrics_width() + Const.TIMESTAMP_BYTES]; + Bytes.setInt(row, (int) base_time, Const.SALT_WIDTH() + TSDB.metrics_width()); return row; } // otherwise we need to build the row key from the TSUID and start time - final byte[] row = new byte[Const.TIMESTAMP_BYTES + tsuid.length]; - System.arraycopy(tsuid, 0, row, 0, TSDB.metrics_width()); - Bytes.setInt(row, (int) base_time, TSDB.metrics_width()); - System.arraycopy(tsuid, TSDB.metrics_width(), row, TSDB.metrics_width() + - Const.TIMESTAMP_BYTES, (tsuid.length - TSDB.metrics_width())); + final byte[] row = new byte[Const.SALT_WIDTH() + Const.TIMESTAMP_BYTES + + tsuid.length]; + System.arraycopy(tsuid, 0, row, Const.SALT_WIDTH(), TSDB.metrics_width()); + Bytes.setInt(row, (int) base_time, Const.SALT_WIDTH() + TSDB.metrics_width()); + System.arraycopy(tsuid, TSDB.metrics_width(), row, + Const.SALT_WIDTH() + TSDB.metrics_width() + Const.TIMESTAMP_BYTES, + (tsuid.length - TSDB.metrics_width())); + RowKey.prefixKeyWithSalt(row); return row; } diff --git a/src/meta/MetaDataCache.java b/src/meta/MetaDataCache.java new file mode 100644 index 0000000000..05efcc0d8e --- /dev/null +++ b/src/meta/MetaDataCache.java @@ -0,0 +1,73 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.meta; + +import com.stumbleupon.async.Deferred; + +import net.opentsdb.core.TSDB; +import net.opentsdb.stats.StatsCollector; + +/** + * This is a first stab at a meta data cache. Initially it only handles + * incrementing TSUID counters in a local database. The class will then + * periodically sync the local counter cache with HBase and generate TSMeta + * objects if necessary. This keeps us from having to maintain thousands or + * millions of callback objects in memory while we wait for individual atomic + * increments per data point. + * @since 2.3 + */ +public abstract class MetaDataCache { + + /** + * Called by TSDB to initialize the plugin + * Implementations are responsible for setting up any IO they need as well + * as starting any required background threads. + * Note: Implementations should throw exceptions if they can't start + * up properly. The TSD will then shutdown so the operator can fix the + * problem. Please use IllegalArgumentException for configuration issues. + * @param tsdb The parent TSDB object + * @throws IllegalArgumentException if required configuration parameters are + * missing + * @throws RuntimeException if something else goes wrong + */ + public abstract void initialize(final TSDB tsdb); + + /** + * Called when the TSD is shutting down to gracefully flush any buffers or + * close open connections. + */ + public abstract Deferred shutdown(); + + /** + * Should return the version of this plugin in the format: + * MAJOR.MINOR.MAINT, e.g. 2.0.1. The MAJOR version should match the major + * version of OpenTSDB the plugin is meant to work with. + * @return A version string used to log the loaded version + */ + public abstract String version(); + + /** + * Called by the TSD when a request for statistics collection has come in. The + * implementation may provide one or more statistics. If no statistics are + * available for the implementation, simply stub the method. + * @param collector The collector used for emitting statistics + */ + public abstract void collectStats(final StatsCollector collector); + + /** + * Increments the given TSUID in the cache by 1 + * @param tsuid The tsuid to increment + */ + public abstract void increment(final byte[] tsuid); + +} diff --git a/src/meta/TSMeta.java b/src/meta/TSMeta.java index 9abc89e463..517a3247c9 100644 --- a/src/meta/TSMeta.java +++ b/src/meta/TSMeta.java @@ -22,6 +22,8 @@ import java.util.Map; import net.opentsdb.core.TSDB; +import net.opentsdb.uid.NoSuchUniqueId; +import net.opentsdb.uid.NoSuchUniqueName; import net.opentsdb.uid.UniqueId; import net.opentsdb.uid.UniqueId.UniqueIdType; import net.opentsdb.utils.JSON; @@ -70,14 +72,14 @@ @JsonIgnoreProperties(ignoreUnknown = true) @JsonInclude(Include.NON_NULL) @JsonAutoDetect(fieldVisibility = Visibility.PUBLIC_ONLY) -public final class TSMeta { +public class TSMeta { private static final Logger LOG = LoggerFactory.getLogger(TSMeta.class); /** Charset used to convert Strings to byte arrays and back. */ private static final Charset CHARSET = Charset.forName("ISO-8859-1"); /** The single column family used by this class. */ - private static final byte[] FAMILY = "name".getBytes(CHARSET); + public static final byte[] FAMILY = "name".getBytes(CHARSET); /** The cell qualifier to use for timeseries meta */ private static final byte[] META_QUALIFIER = "ts_meta".getBytes(CHARSET); @@ -338,15 +340,14 @@ public Deferred call(ArrayList validated) } /** - * Attempts to store a new, blank timeseries meta object via a CompareAndSet + * Attempts to store a new, blank timeseries meta object * Note: This should not be called by user accessible methods as it will * overwrite any data already in the column. * Note: This call does not guarantee that the UIDs exist before * storing as it should only be called *after* a data point has been recorded * or during a meta sync. * @param tsdb The TSDB to use for storage access - * @return True if the CAS completed successfully (and no TSMeta existed - * previously), false if something was already stored in the TSMeta column. + * @return True if the TSMeta created(or updated) successfully * @throws HBaseException if there was an issue fetching * @throws IllegalArgumentException if parsing failed * @throws JSONException if the object could not be serialized @@ -525,7 +526,7 @@ final class TSMetaCB implements Callback, Long> { @Override public Deferred call(final Long incremented_value) throws Exception { -LOG.info("Value: " + incremented_value); + LOG.debug("Value: " + incremented_value); if (incremented_value > 1) { // TODO - maybe update the search index every X number of increments? // Otherwise the search engine would only get last_updated/count @@ -588,10 +589,8 @@ public Deferred call(Boolean success) throws Exception { } LOG.info("Successfullly created new TSUID entry for: " + meta); - final Deferred meta = getFromStorage(tsdb, tsuid) - .addCallbackDeferring( - new LoadUIDs(tsdb, UniqueId.uidToString(tsuid))); - return meta.addCallbackDeferring(new FetchNewCB()); + return new LoadUIDs(tsdb, UniqueId.uidToString(tsuid)).call(meta) + .addCallbackDeferring(new FetchNewCB()); } } @@ -613,6 +612,74 @@ public Deferred call(Boolean success) throws Exception { return tsdb.getClient().atomicIncrement(inc).addCallbackDeferring( new TSMetaCB()); } + + /** + * Attempts to fetch the meta column and if null, attempts to write a new + * column using {@link #storeNew}. + * @param tsdb The TSDB instance to use for access. + * @param tsuid The TSUID of the time series. + * @return A deferred with a true if the meta exists or was created, false + * if the meta did not exist and writing failed. + */ + public static Deferred storeIfNecessary(final TSDB tsdb, + final byte[] tsuid) { + final GetRequest get = new GetRequest(tsdb.metaTable(), tsuid); + get.family(FAMILY); + get.qualifier(META_QUALIFIER); + + final class CreateNewCB implements Callback, Object> { + + @Override + public Deferred call(Object arg0) throws Exception { + final TSMeta meta = new TSMeta(tsuid, System.currentTimeMillis() / 1000); + + final class FetchNewCB implements Callback, TSMeta> { + + @Override + public Deferred call(TSMeta stored_meta) throws Exception { + + // pass to the search plugin + tsdb.indexTSMeta(stored_meta); + + // pass through the trees + tsdb.processTSMetaThroughTrees(stored_meta); + + return Deferred.fromResult(true); + } + } + + final class StoreNewCB implements Callback, Boolean> { + + @Override + public Deferred call(Boolean success) throws Exception { + if (!success) { + LOG.warn("Unable to save metadata: " + meta); + return Deferred.fromResult(false); + } + + LOG.info("Successfullly created new TSUID entry for: " + meta); + return new LoadUIDs(tsdb, UniqueId.uidToString(tsuid)).call(meta) + .addCallbackDeferring(new FetchNewCB()); + } + } + + return meta.storeNew(tsdb).addCallbackDeferring(new StoreNewCB()); + } + } + + final class ExistsCB implements Callback, ArrayList> { + + @Override + public Deferred call(ArrayList row) throws Exception { + if (row == null || row.isEmpty() || row.get(0).value() == null) { + return new CreateNewCB().call(null); + } + return Deferred.fromResult(true); + } + } + + return tsdb.getClient().get(get).addCallbackDeferring(new ExistsCB()); + } /** * Attempts to fetch the timeseries meta data from storage. diff --git a/src/meta/TSUIDQuery.java b/src/meta/TSUIDQuery.java index b22031c5f7..51eb4667c0 100644 --- a/src/meta/TSUIDQuery.java +++ b/src/meta/TSUIDQuery.java @@ -18,6 +18,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import net.opentsdb.core.IncomingDataPoint; import net.opentsdb.core.Internal; @@ -28,6 +29,7 @@ import net.opentsdb.uid.NoSuchUniqueName; import net.opentsdb.uid.UniqueId; import net.opentsdb.uid.UniqueId.UniqueIdType; +import net.opentsdb.utils.DateTime; import org.hbase.async.Bytes.ByteMap; import org.hbase.async.GetRequest; @@ -38,10 +40,10 @@ import com.stumbleupon.async.Callback; import com.stumbleupon.async.Deferred; -import com.stumbleupon.async.DeferredGroupException; /** - * Methods for querying the tsdb-meta table. This can be used to figure out what + * Methods for querying the tsdb-meta table or finding the last data point of + * a particular time series. This can be used to figure out what * time series are actually stored as well as fetch TSMeta objects or optimize * queries against the data table. * @since 2.1 @@ -55,26 +57,163 @@ public class TSUIDQuery { */ private static final Charset CHARSET = Charset.forName("ISO-8859-1"); - /** ID of the metric being looked up. */ - private byte[] metric; - - /** - * Tags of the metrics being looked up. - * Each tag is a byte array holding the ID of both the name and value - * of the tag. - * Invariant: an element cannot be both in this array and in group_bys. - */ - private ArrayList tags; - + /** The TSUID that can be set by the caller or after processing the metric */ + private byte[] tsuid; + + /** The metric set by the caller */ + private String metric; + + /** The metric UID after lookup */ + private byte[] metric_uid; + + /** The tags set by the caller */ + private Map tags; + + /** The tag UID list after lookup */ + private ArrayList tag_uids; + + /** Whether or not to resolve names for last data point queries */ + private boolean resolve_names; + + /** How far back, in hours, to scan for last data point queries */ + private int back_scan; + + /** The last timestamp scanned for last data point queries */ + private long last_timestamp; + /** The TSDB we belong to. */ private final TSDB tsdb; /** - * Constructor. + * Default CTor just sets the TSDB reference * @param tsdb The TSDB to use for storage access + * @throws IllegalArgumentException if the TSDB reference is null + * @deprecated Please use one of the other constructors. Will be removed in 2.3 */ public TSUIDQuery(final TSDB tsdb) { + if (tsdb == null) { + throw new IllegalArgumentException("TSDB reference cannot be null"); + } + this.tsdb =tsdb; + } + + /** + * CTor used for a TSUID based query when we know exactly what we want + * @param tsdb The TSDB to use for storage access + * @param tsuid A TSUID to use for querying + * @throws IllegalArgumentException if the TSUID is invalid + */ + public TSUIDQuery(final TSDB tsdb, final byte[] tsuid) { + if (tsdb == null) { + throw new IllegalArgumentException("TSDB reference cannot be null"); + } + if (tsuid == null || tsuid.length < + TSDB.metrics_width() + TSDB.tagk_width() + TSDB.tagv_width()) { + throw new IllegalArgumentException("TSUID must not be null and must " + + "have a metric and at least one tag pair"); + } + this.tsdb = tsdb; + this.tsuid = tsuid; + } + + /** + * CTor used for a metric style query + * @param tsdb The TSDB to use for storage access + * @param metric The metric to look up + * @param tags The tags to lookup. This may be an empty map if you're scanning + * meta. + * @throws IllegalArgumentException if the metric is null, empty or the tag + * map is null. + */ + public TSUIDQuery(final TSDB tsdb, final String metric, + final Map tags) { + if (tsdb == null) { + throw new IllegalArgumentException("TSDB reference cannot be null"); + } + if (metric == null || metric.isEmpty()) { + throw new IllegalArgumentException("Metric cannot be null or empty"); + } + if (tags == null) { + throw new IllegalArgumentException("Tag map cannot be null. Empty is ok"); + } this.tsdb = tsdb; + this.metric = metric; + this.tags = tags; + } + + /** + * Attempts to fetch the last data point for the given metric or TSUID. + * If back_scan == 0 and meta is enabled via + * "tsd.core.meta.enable_tsuid_tracking" or + * "tsd.core.meta.enable_tsuid_incrementing" then we will look up the metric + * or TSUID in the meta table first and use the counter there to get the + * last write time. + *

    + * However if backscan is set, then we'll start with the current time and + * iterate back "back_scan" number of hours until we find a value. + *

    + * @param resolve_names Whether or not to resolve the UIDs back to their + * names when we find a value. + * @param back_scan The number of hours back in time to scan + * @return A data point if found, null if not. Or an exception if something + * went pear shaped. + */ + public Deferred getLastPoint(final boolean resolve_names, + final int back_scan) { + if (back_scan < 0) { + throw new IllegalArgumentException( + "Backscan must be zero or a positive number"); + } + + this.resolve_names = resolve_names; + this.back_scan = back_scan; + + final boolean meta_enabled = tsdb.getConfig().enable_tsuid_tracking() || + tsdb.getConfig().enable_tsuid_incrementing(); + + class TSUIDCB implements Callback, byte[]> { + @Override + public Deferred call(final byte[] incoming_tsuid) + throws Exception { + if (tsuid == null && incoming_tsuid == null) { + return Deferred.fromError(new RuntimeException("Both incoming and " + + "supplied TSUIDs were null for " + TSUIDQuery.this)); + } else if (incoming_tsuid != null) { + setTSUID(incoming_tsuid); + } + if (back_scan < 1 && meta_enabled) { + final GetRequest get = new GetRequest(tsdb.metaTable(), tsuid); + get.family(TSMeta.FAMILY()); + get.qualifier(TSMeta.COUNTER_QUALIFIER()); + return tsdb.getClient().get(get).addCallbackDeferring(new MetaCB()); + } + + if (last_timestamp > 0) { + last_timestamp = Internal.baseTime(last_timestamp); + } else { + last_timestamp = Internal.baseTime(DateTime.currentTimeMillis()); + } + final byte[] key = RowKey.rowKeyFromTSUID(tsdb, tsuid, last_timestamp); + final GetRequest get = new GetRequest(tsdb.dataTable(), key); + get.family(TSDB.FAMILY()); + return tsdb.getClient().get(get).addCallbackDeferring(new LastPointCB()); + } + @Override + public String toString() { + return "TSUID callback"; + } + } + + if (tsuid == null) { + return tsuidFromMetric(tsdb, metric, tags) + .addCallbackDeferring(new TSUIDCB()); + } + try { + // damn typed exceptions.... + return new TSUIDCB().call(null); + } catch (Exception e) { + return Deferred.fromError(e); + } } /** @@ -83,189 +222,309 @@ public TSUIDQuery(final TSDB tsdb) { * @param tags A map of tag value pairs or simply an empty map * @throws NoSuchUniqueName if the metric or any of the tag names/values did * not exist + * @deprecated Please use one of the constructors instead. Will be removed in 2.3 */ - public void setQuery(final String metric, final HashMap tags) { - this.metric = tsdb.getUID(UniqueIdType.METRIC, metric); - this.tags = Tags.resolveAll(tsdb, tags); + public void setQuery(final String metric, final Map tags) { + this.metric = metric; + this.tags = tags; + metric_uid = tsdb.getUID(UniqueIdType.METRIC, metric); + tag_uids = Tags.resolveAll(tsdb, tags); } /** * Fetches a list of TSUIDs given the metric and optional tag pairs. The query * format is similar to TsdbQuery but doesn't support grouping operators for * tags. Only TSUIDs that had "ts_counter" qualifiers will be returned. + *

    + * NOTE: If you called {@link #setQuery(String, Map)} successfully this will + * immediately scan the meta table. But if you used the CTOR to set the + * metric and tags it will attempt to resolve those and may return an exception. * @return A map of TSUIDs to the last timestamp (in milliseconds) when the * "ts_counter" was updated. Note that the timestamp will be the time stored - * by HBase, not the actual timestamp of the data point + * by HBase, not the actual timestamp of the data point. If nothing was + * found, the map will be empty but not null. * @throws IllegalArgumentException if the metric was not set or the tag map * was null */ public Deferred> getLastWriteTimes() { - // we need at least a metric name and the tags can't be null. Empty tags are - // fine, but the map can't be null. - if (metric == null || metric.length < 0) { - throw new IllegalArgumentException("Missing metric UID"); - } - if (tags == null) { - throw new IllegalArgumentException("Tag map was null"); - } - - final Scanner scanner = getScanner(); - scanner.setQualifier(TSMeta.COUNTER_QUALIFIER()); - final Deferred> results = new Deferred>(); - final ByteMap tsuids = new ByteMap(); - - /** - * Scanner callback that will call itself while iterating through the - * tsdb-meta table - */ - final class ScannerCB implements Callback>> { - - /** - * Starts the scanner and is called recursively to fetch the next set of - * rows from the scanner. - * @return The map of spans if loaded successfully, null if no data was - * found - */ - public Object scan() { - return scanner.nextRows().addCallback(this); - } - - /** - * Loops through each row of the scanner results and parses out data - * points and optional meta data - * @return null if no rows were found, otherwise the TreeMap with spans - */ + class ResolutionCB implements Callback>, Object> { @Override - public Object call(final ArrayList> rows) - throws Exception { - try { - if (rows == null) { - results.callback(tsuids); + public Deferred> call(Object arg0) throws Exception { + final Scanner scanner = getScanner(); + scanner.setQualifier(TSMeta.COUNTER_QUALIFIER()); + final Deferred> results = new Deferred>(); + final ByteMap tsuids = new ByteMap(); + + final class ErrBack implements Callback { + @Override + public Object call(final Exception e) throws Exception { + results.callback(e); return null; } + @Override + public String toString() { + return "Error callback"; + } + } + + /** + * Scanner callback that will call itself while iterating through the + * tsdb-meta table + */ + final class ScannerCB implements Callback>> { - for (final ArrayList row : rows) { - final byte[] tsuid = row.get(0).key(); - tsuids.put(tsuid, row.get(0).timestamp()); + /** + * Starts the scanner and is called recursively to fetch the next set of + * rows from the scanner. + * @return The map of spans if loaded successfully, null if no data was + * found + */ + public Object scan() { + return scanner.nextRows().addCallback(this).addErrback(new ErrBack()); + } + + /** + * Loops through each row of the scanner results and parses out data + * points and optional meta data + * @return null if no rows were found, otherwise the TreeMap with spans + */ + @Override + public Object call(final ArrayList> rows) + throws Exception { + try { + if (rows == null) { + results.callback(tsuids); + return null; + } + + for (final ArrayList row : rows) { + final byte[] tsuid = row.get(0).key(); + tsuids.put(tsuid, row.get(0).timestamp()); + } + return scan(); + } catch (Exception e) { + results.callback(e); + return null; + } } - return scan(); - } catch (Exception e) { - results.callback(e); - return null; } + + new ScannerCB().scan(); + return results; + } + @Override + public String toString() { + return "Last counter time callback"; } } - new ScannerCB().scan(); - return results; + if (metric_uid == null) { + return resolveMetric().addCallbackDeferring(new ResolutionCB()); + } + try { + return new ResolutionCB().call(null); + } catch (Exception e) { + return Deferred.fromError(e); + } } /** * Returns all TSMeta objects stored for timeseries defined by this query. The * query is similar to TsdbQuery without any aggregations. Returns an empty * list, when no TSMetas are found. Only returns stored TSMetas. + *

    + * NOTE: If you called {@link #setQuery(String, Map)} successfully this will + * immediately scan the meta table. But if you used the CTOR to set the + * metric and tags it will attempt to resolve those and may return an exception. * @return A list of existing TSMetas for the timeseries covered by the query. * @throws IllegalArgumentException When either no metric was specified or the * tag map was null (Empty map is OK). */ public Deferred> getTSMetas() { - // we need at least a metric name and the tags can't be null. Empty tags are - // fine, but the map can't be null. - if (metric == null || metric.length < 0) { - throw new IllegalArgumentException("Missing metric UID"); - } - if (tags == null) { - throw new IllegalArgumentException("Tag map was null"); - } - - final Scanner scanner = getScanner(); - scanner.setQualifier(TSMeta.META_QUALIFIER()); - final Deferred> results = new Deferred>(); - final List tsmetas = new ArrayList(); - final List> tsmeta_group = new ArrayList>(); - - final class TSMetaGroupCB implements Callback> { - + class ResolutionCB implements Callback>, Object> { @Override - public List call(ArrayList ts) throws Exception { - for (TSMeta tsm: ts) { - if (tsm != null) { - tsmetas.add(tsm); + public Deferred> call(final Object done) throws Exception { + final Scanner scanner = getScanner(); + scanner.setQualifier(TSMeta.META_QUALIFIER()); + final Deferred> results = new Deferred>(); + final List tsmetas = new ArrayList(); + final List> tsmeta_group = new ArrayList>(); + + final class TSMetaGroupCB implements Callback> { + @Override + public List call(ArrayList ts) throws Exception { + for (TSMeta tsm: ts) { + if (tsm != null) { + tsmetas.add(tsm); + } + } + results.callback(tsmetas); + return null; + } + @Override + public String toString() { + return "TSMeta callback"; } } - results.callback(tsmetas); - return null; - } - - } - - /** - * Scanner callback that will call itself while iterating through the - * tsdb-meta table. - * - * Keeps track of a Set of Deferred TSMeta calls. When all rows are scanned, - * will wait for all TSMeta calls to be completed and then create the result - * list. - */ - final class ScannerCB implements Callback>> { - - /** - * Starts the scanner and is called recursively to fetch the next set of - * rows from the scanner. - * @return The map of spans if loaded successfully, null if no data was - * found - */ - public Object scan() { - return scanner.nextRows().addCallback(this); - } - - /** - * Loops through each row of the scanner results and parses out data - * points and optional meta data - * @return null if no rows were found, otherwise the TreeMap with spans - */ - @Override - public Object call(final ArrayList> rows) - throws Exception { - try { - if (rows == null) { - Deferred.group(tsmeta_group).addCallback(new TSMetaGroupCB()); + + final class ErrBack implements Callback { + @Override + public Object call(final Exception e) throws Exception { + results.callback(e); return null; } - for (final ArrayList row : rows) { - tsmeta_group.add(TSMeta.parseFromColumn(tsdb, row.get(0), true)); + @Override + public String toString() { + return "Error callback"; + } + } + + /** + * Scanner callback that will call itself while iterating through the + * tsdb-meta table. + * + * Keeps track of a Set of Deferred TSMeta calls. When all rows are scanned, + * will wait for all TSMeta calls to be completed and then create the result + * list. + */ + final class ScannerCB implements Callback>> { + + /** + * Starts the scanner and is called recursively to fetch the next set of + * rows from the scanner. + * @return The map of spans if loaded successfully, null if no data was + * found + */ + public Object scan() { + return scanner.nextRows().addCallback(this).addErrback(new ErrBack()); + } + + /** + * Loops through each row of the scanner results and parses out data + * points and optional meta data + * @return null if no rows were found, otherwise the TreeMap with spans + */ + @Override + public Object call(final ArrayList> rows) + throws Exception { + try { + if (rows == null) { + Deferred.group(tsmeta_group) + .addCallback(new TSMetaGroupCB()).addErrback(new ErrBack()); + return null; + } + for (final ArrayList row : rows) { + tsmeta_group.add(TSMeta.parseFromColumn(tsdb, row.get(0), true)); + } + return scan(); + } catch (Exception e) { + results.callback(e); + return null; + } } - return scan(); - } catch (Exception e) { - results.callback(e); - return null; } + + new ScannerCB().scan(); + return results; + } + @Override + public String toString() { + return "TSMeta scan callback"; } } - new ScannerCB().scan(); - return results; + if (metric_uid == null) { + return resolveMetric().addCallbackDeferring(new ResolutionCB()); + } + try { + return new ResolutionCB().call(null); + } catch (Exception e) { + return Deferred.fromError(e); + } } public String toString() { final StringBuilder buf = new StringBuilder(); - buf.append("TSUIDQuery(metric=") - .append(Arrays.toString(metric)); - try { - buf.append("), tags=").append(Tags.resolveIds(tsdb, tags)); - } catch (NoSuchUniqueId e) { - buf.append("), tags=<").append(e.getMessage()).append('>'); - } - buf.append("))"); + buf.append("TSUIDQuery(metric=").append(metric) + .append(", tags=").append(tags) + .append(", tsuid=") + .append(tsuid != null ? UniqueId.uidToString(tsuid) : "null") + .append(", last_timestamp=").append(last_timestamp) + .append(", back_scan=").append(back_scan) + .append(", resolve_names=").append(resolve_names) + .append(")"); return buf.toString(); } + /** + * Converts the given metric and tags to a TSUID by resolving the strings to + * their UIDs. Note that the resulting TSUID may not exist if the combination + * was not written to TSDB + * @param tsdb The TSDB to use for storage access + * @param metric The metric name to resolve + * @param tags The tags to resolve. May not be empty. + * @return A deferred containing the TSUID when ready or an error such as + * a NoSuchUniqueName exception if the metric didn't exist or a + * DeferredGroupException if one of the tag keys or values did not exist. + * @throws IllegalArgumentException if the metric or tags were null or + * empty. + */ + public static Deferred tsuidFromMetric(final TSDB tsdb, + final String metric, final Map tags) { + if (metric == null || metric.isEmpty()) { + throw new IllegalArgumentException("The metric cannot be empty"); + } + if (tags == null || tags.isEmpty()) { + throw new IllegalArgumentException("Tags cannot be null or empty " + + "when getting a TSUID"); + } + + final byte[] metric_uid = new byte[TSDB.metrics_width()]; + + class TagsCB implements Callback> { + @Override + public byte[] call(final ArrayList tag_list) throws Exception { + final byte[] tsuid = new byte[metric_uid.length + + ((TSDB.tagk_width() + TSDB.tagv_width()) + * tag_list.size())]; + int idx = 0; + System.arraycopy(metric_uid, 0, tsuid, 0, metric_uid.length); + idx += metric_uid.length; + for (final byte[] t : tag_list) { + System.arraycopy(t, 0, tsuid, idx, t.length); + idx += t.length; + } + return tsuid; + } + @Override + public String toString() { + return "Tag resolution callback"; + } + } + + class MetricCB implements Callback, byte[]> { + @Override + public Deferred call(final byte[] uid) + throws Exception { + System.arraycopy(uid, 0, metric_uid, 0, uid.length); + return Tags.resolveAllAsync(tsdb, tags).addCallback(new TagsCB()); + } + @Override + public String toString() { + return "Metric resolution callback"; + } + } + + return tsdb.getUIDAsync(UniqueIdType.METRIC, metric) + .addCallbackDeferring(new MetricCB()); + } + /** * Attempts to retrieve the last data point for the given TSUID. - * This operates by checking the meta table for the {@link #COUNTER_QUALIFIER} + * This operates by checking the meta table for the {@code COUNTER_QUALIFIER} * and if found, parses the HBase timestamp for the counter (i.e. the time when * the counter was written) and tries to load the row in the data table for * the hour where that timestamp would have landed. If the counter does not @@ -287,186 +546,193 @@ public String toString() { * point was written * @return An {@link IncomingDataPoint} if data was found, null if not * @throws NoSuchUniqueId if one of the tag lookups failed + * @deprecated Please use {@link #getLastPoint} */ public static Deferred getLastPoint(final TSDB tsdb, final byte[] tsuid, final boolean resolve_names, final int max_lookups, final long last_timestamp) { - - final Deferred result = new Deferred(); - final long start_time; - final long time_limit; - if (max_lookups < 1) { - start_time = 0; - time_limit = 0; - } else { - start_time = System.currentTimeMillis() / 1000; - time_limit = start_time - (3600 * max_lookups); + final TSUIDQuery query = new TSUIDQuery(tsdb, tsuid); + query.last_timestamp = last_timestamp; + return query.getLastPoint(resolve_names, max_lookups); + } + + /** + * Resolve the UIDs to names. If the query was for a metric and tags then we + * can just use those. + * @param dp The data point to fill in values for + * @return A deferred with the data point or an exception if something went + * wrong. + */ + private Deferred resolveNames(final IncomingDataPoint dp) { + // If the caller gave us a metric and tags, save some time by NOT hitting + // our UID tables or storage. + if (metric != null) { + dp.setMetric(metric); + dp.setTags((HashMap)tags); + return Deferred.fromResult(dp); } - - final class ErrBack implements Callback { - public Object call(final Exception e) throws Exception { - Throwable ex = e; - while (ex.getClass().equals(DeferredGroupException.class)) { - if (ex.getCause() == null) { - LOG.warn("Unable to get to the root cause of the DGE"); - break; - } - ex = ex.getCause(); - } - if (ex instanceof RuntimeException) { - result.callback(ex); - } else { - result.callback(e); - } - return null; - } + + class TagsCB implements Callback> { + public IncomingDataPoint call(final HashMap tags) + throws Exception { + dp.setTags(tags); + return dp; + } + @Override + public String toString() { + return "Tags resolution CB"; + } } - /** - * Called after GetLastDataPointCB has completed. If nothing was found then - * we just return a null, otherwise we set the TSUID and optionally resolve - * the metric and tag names. - */ - final class ReturnCB implements Callback { - final long timestamp; - - public ReturnCB(final long timestamp) { - this.timestamp = timestamp; + class MetricCB implements Callback, String> { + public Deferred call(final String name) + throws Exception { + dp.setMetric(name); + final List tags = UniqueId.getTagPairsFromTSUID(tsuid); + return Tags.resolveIdsAsync(tsdb, tags).addCallback(new TagsCB()); } - - /** - * Callback implementation. If the time_limit was set and the result was - * null we call the local getPrevious() method to issue a Get on the - * previous row. - */ - public Object call(final IncomingDataPoint dp) - throws Exception { - if (dp == null) { - if (time_limit > 0) { - getPrevious(); - return null; - } - result.callback(null); - return null; - } - - dp.setTSUID(UniqueId.uidToString(tsuid)); - if (!resolve_names) { - result.callback(dp); - return null; - } - - class TagsCB implements Callback> { - public IncomingDataPoint call(final HashMap tags) - throws Exception { - dp.setTags(tags); - result.callback(dp); - return null; - } - } - - class MetricCB implements Callback { - public Object call(final String name) throws Exception { - dp.setMetric(name); - final List tags = UniqueId.getTagPairsFromTSUID(tsuid); - return Tags.resolveIdsAsync(tsdb, tags).addCallback(new TagsCB()); - } - } - - // start the resolve dance - final byte[] metric_uid = Arrays.copyOfRange(tsuid, 0, TSDB.metrics_width()); - return tsdb.getUidName(UniqueIdType.METRIC, metric_uid) - .addCallback(new MetricCB()); + @Override + public String toString() { + return "Metric resolution CB"; } - - /** - * Issues a GetRequest on the previous hour's row of data if we haven't - * exceeded the limit - * @return Null if we've hit the time limit or another deferred to wait - * on - */ - private void getPrevious() { - if (timestamp <= time_limit) { - // we hit our limit and didn't find a valid data point - result.callback(null); - return; + } + + final byte[] metric_uid = Arrays.copyOfRange(tsuid, 0, TSDB.metrics_width()); + return tsdb.getUidName(UniqueIdType.METRIC, metric_uid) + .addCallbackDeferring(new MetricCB()); + } + + /** + * Handles getting the results of the first GetRequest and keeps iterating + * back in time until we find a point or run out of back scans. + */ + private class LastPointCB implements Callback, + ArrayList> { + int iteration = 0; + + @Override + public Deferred call(final ArrayList row) + throws Exception { + if (row == null || row.isEmpty()) { + if (iteration >= back_scan) { + if (LOG.isDebugEnabled()) { + LOG.debug("No data points found within the time span for TSUID query " + + TSUIDQuery.this); + } + return Deferred.fromResult(null); } - // look one row into the past - final long previous_time = timestamp - 3600; - - final byte[] key = RowKey.rowKeyFromTSUID(tsdb, tsuid, previous_time); + last_timestamp -= 3600; + ++iteration; + final byte[] key = RowKey.rowKeyFromTSUID(tsdb, tsuid, last_timestamp); final GetRequest get = new GetRequest(tsdb.dataTable(), key); get.family(TSDB.FAMILY()); - tsdb.getClient().get(get).addCallback( - new Internal.GetLastDataPointCB(tsdb)) - .addCallback(new ReturnCB(previous_time)) - .addErrback(new ErrBack()); + return tsdb.getClient().get(get).addCallbackDeferring(this); } + + final IncomingDataPoint dp = + new Internal.GetLastDataPointCB(tsdb).call(row); + dp.setTSUID(UniqueId.uidToString(tsuid)); + if (!resolve_names) { + return Deferred.fromResult(dp); + } + + return resolveNames(dp); } - - /** - * Callback from the GetRequest that simply determines if the row is empty - * or not - */ - final class ExistsCB implements Callback> { - public Object call(final ArrayList row) + @Override + public String toString() { + return "LastDataPoint callback"; + } + } + + /** + * Callback that receives the result of a single meta table lookup to see if + * the last write counter was there or not. The input should contain just + * the counter column + */ + private class MetaCB implements Callback, + ArrayList> { + @Override + public Deferred call(final ArrayList row) throws Exception { - if (row == null || row.isEmpty() || row.get(0).value() == null) { - result.callback(null); - return null; - } - - // we want the timestamp in seconds so we can figure out what row the - // data *should* be in (though it's not guaranteed) - final long last_write = row.get(0).timestamp(); - final byte[] key = RowKey.rowKeyFromTSUID(tsdb, tsuid, last_write); - final GetRequest get = new GetRequest(tsdb.dataTable(), key); - get.family(TSDB.FAMILY()); - - tsdb.getClient().get(get).addCallback( - new Internal.GetLastDataPointCB(tsdb)) - .addCallback(new ReturnCB(0)) - .addErrback(new ErrBack()); + if (row == null) { + return Deferred.fromResult(null); + } + last_timestamp = Internal.baseTime(row.get(0).timestamp()); + final byte[] key = RowKey.rowKeyFromTSUID(tsdb, tsuid, last_timestamp); + final GetRequest get = new GetRequest(tsdb.dataTable(), key); + get.family(TSDB.FAMILY()); + return tsdb.getClient().get(get).addCallbackDeferring(new LastPointCB()); + } + @Override + public String toString() { + return "Meta TSCounter lookup"; + } + } + + /** + * Resolves the metric and tags (if set) to their UIDs, setting the local + * arrays. + * @return A deferred to wait on or catch exceptions in + * @throws IllegalArgumentException if the metric is empty || null or the + * tag list is null. + */ + private Deferred resolveMetric() { + if (metric == null || metric.isEmpty()) { + throw new IllegalArgumentException("The metric cannot be empty"); + } + if (tags == null) { + throw new IllegalArgumentException("Tags cannot be null or empty " + + "when getting a TSUID"); + } + + class TagsCB implements Callback> { + @Override + public Object call(final ArrayList tag_list) throws Exception { + setTagUIDs(tag_list); return null; } + @Override + public String toString() { + return "Tag resolution callback"; + } } - // we need to determine a course of action. We can either: - // 1) Lookup the last time a data point was written for a TSUID - // 2) Immediately fetch data from a row if we were given a timestamp - // 3) Or start at NOW and iterate backwards until we find a point or hit - // the user supplied limit - if (time_limit == 0) { - if (last_timestamp < 1) { - final GetRequest get = new GetRequest(tsdb.metaTable(), tsuid); - get.family(TSMeta.FAMILY()); - get.qualifier(TSMeta.COUNTER_QUALIFIER()); - tsdb.getClient().get(get) - .addCallback(new ExistsCB()) - .addErrback(new ErrBack()); - } else { - final byte[] key = RowKey.rowKeyFromTSUID(tsdb, tsuid, last_timestamp); - final GetRequest get = new GetRequest(tsdb.dataTable(), key); - get.family(TSDB.FAMILY()); - - tsdb.getClient().get(get).addCallback( - new Internal.GetLastDataPointCB(tsdb)) - .addCallback(new ReturnCB(0)) - .addErrback(new ErrBack()); + class MetricCB implements Callback, byte[]> { + @Override + public Deferred call(final byte[] uid) + throws Exception { + setMetricUID(uid); + if (tags.isEmpty()) { + setTagUIDs(new ArrayList(0)); + return null; + } + return Tags.resolveAllAsync(tsdb, tags).addCallback(new TagsCB()); + } + @Override + public String toString() { + return "Metric resolution callback"; } - } else { - final byte[] key = RowKey.rowKeyFromTSUID(tsdb, tsuid, start_time); - final GetRequest get = new GetRequest(tsdb.dataTable(), key); - get.family(TSDB.FAMILY()); - tsdb.getClient().get(get).addCallback( - new Internal.GetLastDataPointCB(tsdb)) - .addCallback(new ReturnCB(start_time)) - .addErrback(new ErrBack()); } - return result; - } + return tsdb.getUIDAsync(UniqueIdType.METRIC, metric) + .addCallbackDeferring(new MetricCB()); + } + + /** @param tsuid The TSUID to store */ + private void setTSUID(final byte[] tsuid) { + this.tsuid = tsuid; + } + + /** @param uid The metric UID to set */ + private void setMetricUID(final byte[] uid) { + metric_uid = uid; + } + + /** @param uids The tag UIDs to set */ + private void setTagUIDs(final ArrayList uids) { + tag_uids = uids; + } /** * Configures the scanner for a specific metric and optional tags @@ -474,11 +740,11 @@ public Object call(final ArrayList row) */ private Scanner getScanner() { final Scanner scanner = tsdb.getClient().newScanner(tsdb.metaTable()); - scanner.setStartKey(metric); + scanner.setStartKey(metric_uid); // increment the metric UID by one so we can scan all of the rows for the // given metric - final long stop = UniqueId.uidToLong(metric, TSDB.metrics_width()) + 1; + final long stop = UniqueId.uidToLong(metric_uid, TSDB.metrics_width()) + 1; scanner.setStopKey(UniqueId.longToUID(stop, TSDB.metrics_width())); scanner.setFamily(TSMeta.FAMILY()); @@ -501,7 +767,7 @@ private Scanner getScanner() { // ... start by skipping the metric ID. .append(TSDB.metrics_width()) .append("}"); - final Iterator tags = this.tags.iterator(); + final Iterator tags = this.tag_uids.iterator(); byte[] tag = tags.hasNext() ? tags.next() : null; // Tags and group_bys are already sorted. We need to put them in the // regexp in order by ID, which means we just merge two sorted lists. @@ -518,4 +784,4 @@ private Scanner getScanner() { return scanner; } -} +} \ No newline at end of file diff --git a/src/meta/UIDMeta.java b/src/meta/UIDMeta.java index 21f0e5470b..9d2f9a29ca 100644 --- a/src/meta/UIDMeta.java +++ b/src/meta/UIDMeta.java @@ -36,6 +36,7 @@ import com.stumbleupon.async.Deferred; import net.opentsdb.core.TSDB; +import net.opentsdb.uid.NoSuchUniqueId; import net.opentsdb.uid.UniqueId; import net.opentsdb.uid.UniqueId.UniqueIdType; import net.opentsdb.utils.JSON; @@ -67,7 +68,7 @@ */ @JsonIgnoreProperties(ignoreUnknown = true) @JsonAutoDetect(fieldVisibility = Visibility.PUBLIC_ONLY) -public final class UIDMeta { +public class UIDMeta { private static final Logger LOG = LoggerFactory.getLogger(UIDMeta.class); /** Charset used to convert Strings to byte arrays and back. */ diff --git a/src/normalize/NormalizePlugin.java b/src/normalize/NormalizePlugin.java new file mode 100644 index 0000000000..6eae2bd9b5 --- /dev/null +++ b/src/normalize/NormalizePlugin.java @@ -0,0 +1,21 @@ +package net.opentsdb.normalize; + +import com.stumbleupon.async.Deferred; +import net.opentsdb.core.TSDB; +import net.opentsdb.stats.StatsCollector; + +import java.util.Map; + +public abstract class NormalizePlugin { + + public abstract void initialize(final TSDB tsdb); + + public abstract Deferred shutdown(); + + public abstract String version(); + + public abstract void collectStats(final StatsCollector collector); + + public abstract Map normalizeTags(Map tags); + +} \ No newline at end of file diff --git a/src/opentsdb.conf b/src/opentsdb.conf index bed259d587..8ba7028a52 100644 --- a/src/opentsdb.conf +++ b/src/opentsdb.conf @@ -6,17 +6,16 @@ tsd.network.port = # The IPv4 network address to bind to, defaults to all addresses # tsd.network.bind = 0.0.0.0 -# Enables Nagel's algorithm to reduce the number of packets sent over the -# network, default is True -#tsd.network.tcpnodelay = true +# Disable Nagel's algorithm, default is True +#tsd.network.tcp_no_delay = true -# Determines whether or not to send keepalive packets to peers, default +# Determines whether or not to send keepalive packets to peers, default # is True -#tsd.network.keepalive = true +#tsd.network.keep_alive = true -# Determines if the same socket should be used for new connections, default +# Determines if the same socket should be used for new connections, default # is True -#tsd.network.reuseaddress = true +#tsd.network.reuse_address = true # Number of worker threads dedicated to Netty, defaults to # of CPUs * 2 #tsd.network.worker_threads = 8 @@ -38,14 +37,25 @@ tsd.http.cachedir = # is False #tsd.core.auto_create_metrics = false +# Whether or not to enable the built-in UI Rpc Plugins, default +# is True +#tsd.core.enable_ui = true + +# Whether or not to enable the built-in API Rpc Plugins, default +# is True +#tsd.core.enable_api = true + # --------- STORAGE ---------- # Whether or not to enable data compaction in HBase, default is True #tsd.storage.enable_compaction = true -# How often, in milliseconds, to flush the data point queue to storage, +# How often, in milliseconds, to flush the data point queue to storage, # default is 1,000 # tsd.storage.flush_interval = 1000 +# Max number of rows to be returned per Scanner round trip +# tsd.storage.hbase.scanner.maxNumRows = 128 + # Name of the HBase table where data points are stored, default is "tsdb" #tsd.storage.hbase.data_table = tsdb @@ -55,6 +65,19 @@ tsd.http.cachedir = # Path under which the znode for the -ROOT- region is located, default is "/hbase" #tsd.storage.hbase.zk_basedir = /hbase -# A comma separated list of Zookeeper hosts to connect to, with or without +# A comma separated list of Zookeeper hosts to connect to, with or without # port specifiers, default is "localhost" -#tsd.storage.hbase.zk_quorum = localhost \ No newline at end of file +#tsd.storage.hbase.zk_quorum = localhost + +# --------- COMPACTIONS --------------------------------- +# Frequency at which compaction thread wakes up to flush stuff in seconds, default 10 +# tsd.storage.compaction.flush_interval = 10 + +# Minimum rows attempted to compact at once, default 100 +# tsd.storage.compaction.min_flush_threshold = 100 + +# Maximum number of rows, compacted concirrently, default 10000 +# tsd.storage.compaction.max_concurrent_flushes = 10000 + +# Compaction flush speed multiplier, default 2 +# tsd.storage.compaction.flush_speed = 2 diff --git a/src/parser.jj b/src/parser.jj new file mode 100644 index 0000000000..29b00c9e40 --- /dev/null +++ b/src/parser.jj @@ -0,0 +1,74 @@ +/** Options required by Maven */ +options { + STATIC = false; + LOOKAHEAD = 5; +} + +PARSER_BEGIN(SyntaxChecker) +package net.opentsdb.query.expression.parser; + +import net.opentsdb.query.expression.ExpressionTree; +import net.opentsdb.core.TSQuery; + +import java.util.List; +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; + +/** +* A simple class for validating the expressions +* @since 2.3 +*/ +public class SyntaxChecker { + + private TSQuery data_query; + private List metricQueries; + + public void setTSQuery(TSQuery data_query) { + this.data_query = data_query; + } + + public void setMetricQueries(List metricQueries) { + this.metricQueries = metricQueries; + } + + public static void main(String[] args) { + try { + new SyntaxChecker(new java.io.StringReader(args[0])).EXPRESSION(); + System.out.println("Syntax is okay"); + } catch (Throwable e) { + // Catching Throwable is ugly but JavaCC throws Error objects! + System.out.println("Syntax check failed: " + e.getMessage()); + } + } +} + +PARSER_END(SyntaxChecker) + +SKIP: { " " | "\t" | "\n" | "\r" } +TOKEN: { } +TOKEN: { } + +ExpressionTree EXPRESSION(): {Token name; int paramIndex=0;} { + name= { ExpressionTree tree=new ExpressionTree(name.image,data_query); } + "(" PARAMETER(tree, paramIndex++) ("," PARAMETER(tree, paramIndex++))* ")" + {return tree;} +} + +void PARAMETER(ExpressionTree tree, int paramIndex): {String metric; Token param; ExpressionTree subTree;} { + subTree=EXPRESSION() {tree.addSubExpression(subTree, paramIndex);} | + metric=METRIC() {metricQueries.add(metric); tree.addSubMetricQuery(metric, metricQueries.size()-1, paramIndex);} | + param= {tree.addFunctionParameter(param.image);} +} + +// metric is agg:[interval-agg:][rate:]metric[{tag=value,...}] +String METRIC() : {Token agg,itvl,rate,metric,tagk,tagv; StringBuilder builder = new StringBuilder(); + Joiner JOINER = Joiner.on(",").skipNulls(); + List tagPairs = Lists.newArrayList(); + } { + agg = ":" { builder.append(agg.image).append(":"); } + (itvl= ":" { builder.append(itvl.image).append(":"); })? + (rate= ":" { builder.append(rate.image).append(":"); })? + metric= { builder.append(metric.image); } + ("{" tagk= "=" tagv= {tagPairs.add(tagk+"="+tagv);} + ("," tagk= "=" tagv= {tagPairs.add(tagk+"="+tagv);})* "}")? + {if (tagPairs.size() > 0) builder.append("{").append(JOINER.join(tagPairs)).append("}"); return builder.toString();} } \ No newline at end of file diff --git a/src/query/QueryLimitOverride.java b/src/query/QueryLimitOverride.java new file mode 100644 index 0000000000..dad9bb140a --- /dev/null +++ b/src/query/QueryLimitOverride.java @@ -0,0 +1,337 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query; + +import java.io.File; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +import org.jboss.netty.util.HashedWheelTimer; +import org.jboss.netty.util.Timeout; +import org.jboss.netty.util.TimerTask; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.base.Objects; +import com.google.common.base.Strings; +import com.google.common.collect.Maps; +import com.google.common.io.Files; + +import net.opentsdb.core.Const; +import net.opentsdb.core.TSDB; +import net.opentsdb.utils.JSON; + +/** + * A class used for numeric query overrides using regular expression matching. + * The class will attempt to load from a locally cached file on construction if + * a file is given. Each time one or more items are added or removed to the list + * the file will be updated if it was configured. + * This class is thread safe. + * + * @since 2.4 + */ +public class QueryLimitOverride implements TimerTask { + private static final Logger LOG = LoggerFactory.getLogger(QueryLimitOverride.class); + + /** Used for deserialization the proper object. Without this the hash codes + /* won't function properly as Jackson may just create a hash map for our simple + /* object. */ + public static TypeReference> TR_OVERRIDES = + new TypeReference>() {}; + + /** The list of overrides */ + + /** Keyed on the raw regex so we can update objects properly. */ + private final Map overrides; + + /** The default byte limit to use if the string didn't match. */ + private long default_byte_limit; + + /** The default data points limit to use if the string didn't match. */ + private long default_data_points_limit; + + /** The optional file location to read/write */ + private String file_location; + + /** How often, in seconds, to reload from the file */ + private int reload_interval; + + /** A timer for refreshing data from the config */ + private HashedWheelTimer timer; + + /** + * Default ctor. If a file location is given, it will be read on construction + * and reloaded every interval seconds. Note that if there is a problem reading + * or parsing the config file (when set) the ctor will continue loading with + * the defaults and if an interval is set, will attempt to reload at a later + * time. + * @param tsdb The TSDB to which we belong. + * @throws IllegalArgumentException if the default limits are less than zero. + */ + public QueryLimitOverride(final TSDB tsdb) { + overrides = Maps.newConcurrentMap(); + default_byte_limit = tsdb.getConfig().getLong("tsd.query.limits.bytes.default"); + default_data_points_limit = tsdb.getConfig() + .getLong("tsd.query.limits.data_points.default"); + if (tsdb.getConfig().hasProperty("tsd.query.limits.overrides.interval")) { + reload_interval = tsdb.getConfig().getInt("tsd.query.limits.overrides.interval"); + } else { + reload_interval = 0; + } + file_location = tsdb.getConfig().getString("tsd.query.limits.overrides.config"); + timer = (HashedWheelTimer) tsdb.getTimer(); + if (default_byte_limit < 0) { + throw new IllegalArgumentException("The default byte limit cannot be negative"); + } + if (default_data_points_limit < 0) { + throw new IllegalArgumentException("The default data points limit cannot" + + " be negative"); + } + + if (!Strings.isNullOrEmpty(file_location)) { + loadFromFile(); + if (reload_interval > 0) { + timer.newTimeout(this, reload_interval, TimeUnit.SECONDS); + } + } + } + + /** @return The default byte limit used when a match fails */ + public long getDefaultByteLimit() { + return default_byte_limit; + } + + /** @return The default data points limit used when a match fails */ + public long getDefaultDataPointsLimit() { + return default_data_points_limit; + } + + /** + * Iterates over the list of overrides and return the first that matches or + * the default if no match is found. + * NOTE: The set of expressions is not sorted so if more than one regex + * matches the string, the result is indeterministic. + * If the metric is null or empty, the default limit is returned. + * @param metric The string to match + * @return The matched or default limit. + */ + public synchronized long getByteLimit(final String metric) { + if (metric == null || metric.isEmpty()) { + return default_byte_limit; + } + for (final QueryLimitOverrideItem item : overrides.values()) { + if (item.matches(metric)) { + return item.getByteLimit(); + } + } + return default_byte_limit; + } + + /** + * Iterates over the list of overrides and return the first that matches or + * the default if no match is found. + * NOTE: The set of expressions is not sorted so if more than one regex + * matches the string, the result is indeterministic. + * If the metric is null or empty, the default limit is returned. + * @param metric The string to match + * @return The matched or default limit. + */ + public synchronized long getDataPointLimit(final String metric) { + if (metric == null || metric.isEmpty()) { + return default_data_points_limit; + } + for (final QueryLimitOverrideItem item : overrides.values()) { + if (item.matches(metric)) { + return item.getDataPointsLimit(); + } + } + return default_data_points_limit; + } + + /** @return An unmodifiable collection of the items. WARNING: Don't modify the items! + * They are not duplicates (right now)*/ + public Collection getLimits() { + return Collections.unmodifiableCollection(overrides.values()); + } + + @Override + public String toString() { + return JSON.serializeToString(this); + } + + /** @param timeout The timeout reference. */ + @Override + public void run(final Timeout timeout) { + try { + loadFromFile(); + } catch (RuntimeException e) { + LOG.error("Failed to read cache file on auto reload: " + this, e); + } finally { + timer.newTimeout(this, reload_interval, TimeUnit.SECONDS); + } + } + + /** + * Attempts to load the file from disk + */ + private void loadFromFile() { + // load from disk if the caller gave us a file + if (file_location != null && !file_location.isEmpty()) { + final File file = new File(file_location); + if (!file.exists()) { + LOG.warn("Query override file " + file_location + " does not exist"); + return; + } + try { + final String raw_json = Files.toString(file, Const.UTF8_CHARSET); + if (raw_json != null && !raw_json.isEmpty()) { + final Set cached_items = + JSON.parseToObject(raw_json, TR_OVERRIDES); + + // iterate so we only change bits that are different. + for (final QueryLimitOverrideItem override : cached_items) { + QueryLimitOverrideItem existing = overrides.get(override.getRegex()); + if (existing == null || !existing.equals(override)) { + overrides.put(override.getRegex(), override); + } + } + + // reverse quadratic, woot! Ugly but if the limit file is so big that + // this takes over 60 seconds or starts blocking queries on modifications + // to the map then something is really wrong. + final Iterator> iterator = + overrides.entrySet().iterator(); + while (iterator.hasNext()) { + final Entry entry = iterator.next(); + boolean matched = false; + for (final QueryLimitOverrideItem override : cached_items) { + if (override.getRegex().equals(entry.getKey())) { + matched = true; + break; + } + } + if (!matched) { + iterator.remove(); + } + } + } + LOG.info("Successfully loaded query overrides: " + this); + } catch (Exception e) { + LOG.error("Failed to read cache file for query limit override: " + + this, e); + } + } + } + + /** A simple class for ser/des of the items along with some validation */ + public static class QueryLimitOverrideItem { + + /** The regular expression provided by the user */ + private String regex; + + /** A compiled pattern for speedier operation */ + private Pattern pattern; + + /** The byte limit to use when matched */ + private long byte_limit = 0; + + /** The data points limit to use when matched */ + private long data_points_limit = 0; + + @Override + public int hashCode() { + return Objects.hashCode(regex, byte_limit, data_points_limit); + } + + @Override + public boolean equals(final Object item) { + if (item == this) { + return true; + } + if (item == null) { + return false; + } + return regex.equals(((QueryLimitOverrideItem)item).regex) && + byte_limit == ((QueryLimitOverrideItem)item).byte_limit && + data_points_limit == ((QueryLimitOverrideItem)item).data_points_limit; + } + + /** @return The regular expression */ + public String getRegex() { + return regex; + } + + /** @param regex The regular expression + * @throws PatternSyntaxException if the regex can't be compiled */ + public void setRegex(final String regex) { + this.regex = regex; + pattern = Pattern.compile(regex); + } + + /** @return The byte limit to use for this override */ + public long getByteLimit() { + return byte_limit; + } + + /** @param byte_limit The byte limit to use for this override */ + public void setByteLimit(final long byte_limit) { + this.byte_limit = byte_limit; + } + + /** @return The data points limit to use for this override. */ + public long getDataPointsLimit() { + return data_points_limit; + } + + /** @param data_points_limit The data points limit to use for this override. */ + public void setDataPointsLimit(final long data_points_limit) { + this.data_points_limit = data_points_limit; + } + + /** @param string The string to match. + * @return true if the string matches this regex pattern, false if not. + * If the string is null or empty we return false. */ + public boolean matches(final String string) { + if (string == null || string.isEmpty()) { + return false; + } + if (pattern == null || regex == null || regex.isEmpty()) { + return false; + } + return pattern.matcher(string).find(); + } + + /** @throws IllegalArgumentException if the limit is less than zero or + * the regular expression is null or empty */ + public void validate() { + if (byte_limit < 0) { + throw new IllegalArgumentException("The byte limit must be 0 or greater"); + } + if (data_points_limit < 0) { + throw new IllegalArgumentException("The data points limit must be 0 or greater"); + } + if (regex == null || regex.isEmpty()) { + throw new IllegalArgumentException("The regex cannot be empty or null"); + } + } + } +} diff --git a/src/query/QueryUtil.java b/src/query/QueryUtil.java new file mode 100644 index 0000000000..f04e3fb0b1 --- /dev/null +++ b/src/query/QueryUtil.java @@ -0,0 +1,662 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2010-2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.Map.Entry; + +import net.opentsdb.core.Const; +import net.opentsdb.core.Internal; +import net.opentsdb.core.RowKey; +import net.opentsdb.core.TSDB; +import net.opentsdb.uid.UniqueId; + +import org.hbase.async.Bytes; +import org.hbase.async.FuzzyRowFilter; +import org.hbase.async.FuzzyRowFilter.FuzzyFilterPair; +import org.hbase.async.KeyRegexpFilter; +import org.hbase.async.Bytes.ByteMap; +import org.hbase.async.FilterList.Operator; +import org.hbase.async.FilterList; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Strings; +import com.google.common.collect.Lists; + +import org.hbase.async.Scanner; + +/** + * A simple class with utility methods for executing queries against the storage + * layer. + * @since 2.2 + */ +public class QueryUtil { + private static final Logger LOG = LoggerFactory.getLogger(QueryUtil.class); + + /** + * Crafts a regular expression for scanning over data table rows and filtering + * time series that the user doesn't want. At least one of the parameters + * must be set and have values. + * NOTE: This method will sort the group bys. + * @param group_bys An optional list of tag keys that we want to group on. May + * be null. + * @param row_key_literals An optional list of key value pairs to filter on. + * May be null. + * @return A regular expression string to pass to the storage layer. + */ + public static String getRowKeyUIDRegex(final List group_bys, + final ByteMap row_key_literals) { + return getRowKeyUIDRegex(group_bys, row_key_literals, false, null, null); + } + + /** + * Crafts a regular expression for scanning over data table rows and filtering + * time series that the user doesn't want. Also fills in an optional fuzzy + * mask and key as it builds the regex if configured to do so. + * @param group_bys An optional list of tag keys that we want to group on. May + * be null. + * @param row_key_literals An optional list of key value pairs to filter on. + * May be null. + * @param explicit_tags Whether or not explicit tags are enabled so that the + * regex only picks out series with the specified tags + * @param fuzzy_key An optional fuzzy filter row key + * @param fuzzy_mask An optional fuzzy filter mask + * @return A regular expression string to pass to the storage layer. + * @since 2.3 + */ + public static String getRowKeyUIDRegex( + final List group_bys, + final ByteMap row_key_literals, + final boolean explicit_tags, + final byte[] fuzzy_key, + final byte[] fuzzy_mask) { + if (group_bys != null) { + Collections.sort(group_bys, Bytes.MEMCMP); + } + final int prefix_width = Const.SALT_WIDTH() + TSDB.metrics_width() + + Const.TIMESTAMP_BYTES; + final short name_width = TSDB.tagk_width(); + final short value_width = TSDB.tagv_width(); + final short tagsize = (short) (name_width + value_width); + // Generate a regexp for our tags. Say we have 2 tags: { 0 0 1 0 0 2 } + // and { 4 5 6 9 8 7 }, the regexp will be: + // "^.{7}(?:.{6})*\\Q\000\000\001\000\000\002\\E(?:.{6})*\\Q\004\005\006\011\010\007\\E(?:.{6})*$" + final StringBuilder buf = new StringBuilder( + 15 // "^.{N}" + "(?:.{M})*" + "$" + + ((13 + tagsize) // "(?:.{M})*\\Q" + tagsize bytes + "\\E" + * ((row_key_literals == null ? 0 : row_key_literals.size()) + + (group_bys == null ? 0 : group_bys.size() * 3)))); + // In order to avoid re-allocations, reserve a bit more w/ groups ^^^ + + // Alright, let's build this regexp. From the beginning... + buf.append("(?s)" // Ensure we use the DOTALL flag. + + "^.{") + // ... start by skipping the salt, metric ID and timestamp. + .append(Const.SALT_WIDTH() + TSDB.metrics_width() + Const.TIMESTAMP_BYTES) + .append("}"); + + final Iterator> it = row_key_literals == null ? + new ByteMap().iterator() : row_key_literals.iterator(); + int fuzzy_offset = Const.SALT_WIDTH() + TSDB.metrics_width(); + if (fuzzy_mask != null) { + // make sure to skip the timestamp when scanning + while (fuzzy_offset < prefix_width) { + fuzzy_mask[fuzzy_offset++] = 1; + } + } + + while(it.hasNext()) { + Entry entry = it.hasNext() ? it.next() : null; + // TODO - This look ahead may be expensive. We need to get some data around + // whether it's faster for HBase to scan with a look ahead or simply pass + // the rows back to the TSD for filtering. + final boolean not_key = + entry.getValue() != null && entry.getValue().length == 0; + + // Skip any number of tags. + if (!explicit_tags) { + buf.append("(?:.{").append(tagsize).append("})*"); + } else if (fuzzy_mask != null) { + // TODO - see if we can figure out how to improve the fuzzy filter by + // setting explicit tag values whenever we can. In testing there was + // a conflict between the row key regex and fuzzy filter that prevented + // results from returning properly. + System.arraycopy(entry.getKey(), 0, fuzzy_key, fuzzy_offset, name_width); + fuzzy_offset += name_width; + for (int i = 0; i < value_width; i++) { + fuzzy_mask[fuzzy_offset++] = 1; + } + } + if (not_key) { + // start the lookahead as we have a key we explicitly do not want in the + // results + buf.append("(?!"); + } + buf.append("\\Q"); + + addId(buf, entry.getKey(), true); + if (entry.getValue() != null && entry.getValue().length > 0) { // Add a group_by. + // We want specific IDs. List them: /(AAA|BBB|CCC|..)/ + buf.append("(?:"); + for (final byte[] value_id : entry.getValue()) { + if (value_id == null) { + continue; + } + buf.append("\\Q"); + addId(buf, value_id, true); + buf.append('|'); + } + // Replace the pipe of the last iteration. + buf.setCharAt(buf.length() - 1, ')'); + } else { + buf.append(".{").append(value_width).append('}'); // Any value ID. + } + + if (not_key) { + // be sure to close off the look ahead + buf.append(")"); + } + } + // Skip any number of tags before the end. + if (!explicit_tags) { + buf.append("(?:.{").append(tagsize).append("})*"); + } + buf.append("$"); + return buf.toString(); + } + + /** + * Crafts a regular expression for scanning over data table rows and filtering + * time series that the user doesn't want. + * @param row_key_literals An optional list of key value pairs to filter on. + * May be null. + * @param explicit_tags Whether or not explicit tags are enabled so that the + * regex only picks out series with the specified tags + * @return A regular expression string to pass to the storage layer. + */ + private static String getRowKeyUIDRegex( + final ByteMap row_key_literals, + final boolean explicit_tags) { + final int prefix_width = Const.SALT_WIDTH() + TSDB.metrics_width() + + Const.TIMESTAMP_BYTES; + final short name_width = TSDB.tagk_width(); + final short value_width = TSDB.tagv_width(); + final short tagsize = (short) (name_width + value_width); + // Generate a regexp for our tags. Say we have 2 tags: { 0 0 1 0 0 2 } + // and { 4 5 6 9 8 7 }, the regexp will be: + // "^.{7}(?:.{6})*\\Q\000\000\001\000\000\002\\E(?:.{6})*\\Q\004\005\006\011\010\007\\E(?:.{6})*$" + final StringBuilder buf = new StringBuilder( + 15 // "^.{N}" + "(?:.{M})*" + "$" + + ((13 + tagsize) // "(?:.{M})*\\Q" + tagsize bytes + "\\E" + * ((row_key_literals == null ? 0 : row_key_literals.size())))); + + // Alright, let's build this regexp. From the beginning... + buf.append("(?s)" // Ensure we use the DOTALL flag. + + "^.{") + // ... start by skipping the salt, metric ID and timestamp. + .append(prefix_width) + .append("}"); + + final Iterator> it = row_key_literals == null ? + new ByteMap().iterator() : row_key_literals.iterator(); + + while(it.hasNext()) { + Entry entry = it.hasNext() ? it.next() : null; + // TODO - This look ahead may be expensive. We need to get some data around + // whether it's faster for HBase to scan with a look ahead or simply pass + // the rows back to the TSD for filtering. + final boolean not_key = + entry.getValue() != null && entry.getValue().length == 0; + + // Skip any number of tags. + if (!explicit_tags) { + buf.append("(?:.{").append(tagsize).append("})*"); + } + + if (not_key) { + // start the lookahead as we have a key we explicitly do not want in the + // results + buf.append("(?!"); + } + buf.append("\\Q"); + + addId(buf, entry.getKey(), true); + if (entry.getValue() != null && entry.getValue().length > 0) { // Add a group_by. + // We want specific IDs. List them: /(AAA|BBB|CCC|..)/ + buf.append("(?:"); + for (final byte[] value_id : entry.getValue()) { + if (value_id == null) { + continue; + } + buf.append("\\Q"); + addId(buf, value_id, true); + buf.append('|'); + } + // Replace the pipe of the last iteration. + buf.setCharAt(buf.length() - 1, ')'); + } else { + buf.append(".{").append(value_width).append('}'); // Any value ID. + } + + if (not_key) { + // be sure to close off the look ahead + buf.append(")"); + } + } + + // Skip any number of tags before the end. + if (!explicit_tags) { + buf.append("(?:.{").append(tagsize).append("})*"); + } + buf.append("$"); + return buf.toString(); + } + + /** + * Crafts a list of FuzzyFilters for scanning over data table rows and + * filtering time series that the user doesn't want. + * Note: The caller has to restrict the scan to proper start and stop + * for the filter to work correctly. + * @param row_key_literals A list of key value pairs to filter on. + * @param fuzzy_key The starting row key we'll adjust for proper filtering. + * @return A sorted, non-empty list of FuzzyFilterPair + */ + private static List buildFuzzyFilters( + final ByteMap row_key_literals, + final byte[] fuzzy_key) { + final int prefix_width = Const.SALT_WIDTH() + TSDB.metrics_width() + + Const.TIMESTAMP_BYTES; + final short name_width = TSDB.tagk_width(); + final short value_width = TSDB.tagv_width(); + final short tag_width = (short) (name_width + value_width); + int row_key_size = prefix_width; + if (row_key_literals != null) { + for(byte[][] v: row_key_literals.values()) { + final boolean not_key = v!=null && v.length==0; + if (!not_key) { + row_key_size += tag_width; + } + } + } + final List fuzzy_filter_pairs = + new ArrayList(row_key_literals.size()); + + // Initialize first_fuzzy_key and first_fuzzy_mask + // these will serve as model for the fuzzy filter list + // generated for tags with multiple values (|) + byte[] first_fuzzy_key = Arrays.copyOf(fuzzy_key, fuzzy_key.length); + byte[] first_fuzzy_mask = new byte[fuzzy_key.length]; + int fuzzy_offset = 0; + + // TODO - see if it's less expensive to skip the salt, timestamp and metric. + // skip salt & timestamp (filtering should be done by start/stop + // of the scanner) + while(fuzzy_offset < prefix_width) { + first_fuzzy_key[fuzzy_offset] = 0; + first_fuzzy_mask[fuzzy_offset++] = + (row_key_literals != null) ? (byte)1 : (byte)0; + } + + // first pass to build the key and mask + Iterator> it = row_key_literals.iterator(); + while(it.hasNext()) { + Entry entry = it.next(); + final boolean not_key = + entry.getValue() != null && entry.getValue().length == 0; + + if (!not_key) { + final byte[] tag_key = entry.getKey(); + System.arraycopy(tag_key, 0, + first_fuzzy_key, fuzzy_offset, name_width); + for (int i=0; i 0) { + tag_value = entry.getValue()[0]; + } else { + tag_value = null; + } + + if (tag_value!=null) { + System.arraycopy(tag_value, 0, + first_fuzzy_key, fuzzy_offset, value_width); + for (int i=0; i skip + for (int i=0; i entry = it.next(); + fuzzy_offset += name_width; + + // if multiple values value, generate a new combination of filters + // for each value + if (entry.getValue()!=null && entry.getValue().length > 1) { + for (int i=1; i { + @Override + public int compare(FuzzyFilterPair pair1, FuzzyFilterPair pair2) { + return Bytes.memcmp(pair2.getRowKey(), pair1.getRowKey()); + } + } + private static FuzzyFilterComparator FUZZY_FILTER_CMP = new FuzzyFilterComparator(); + + /** + * Sets a filter or filter list on the scanner based on whether or not the + * query had tags it needed to match. + * NOTE: This method will sort the group bys. + * @param scanner The scanner to modify. + * @param group_bys An optional list of tag keys that we want to group on. May + * be null. + * @param row_key_literals An optional list of key value pairs to filter on. + * May be null. + * @param explicit_tags Whether or not explicit tags are enabled so that the + * regex only picks out series with the specified tags + * @param enable_fuzzy_filter Whether or not a fuzzy filter should be used + * in combination with the explicit tags param. If explicit tags is disabled + * then this param is ignored. + * @param end_time The end of the query time so the fuzzy filter knows when + * to stop scanning. + */ + public static void setDataTableScanFilter( + final Scanner scanner, + final List group_bys, + final ByteMap row_key_literals, + final boolean explicit_tags, + final boolean enable_fuzzy_filter, + final int end_time) { + + // no-op + if ((group_bys == null || group_bys.isEmpty()) + && (row_key_literals == null || row_key_literals.isEmpty())) { + return; + } + + if (group_bys != null) { + Collections.sort(group_bys, Bytes.MEMCMP); + } + + final int prefix_width = Const.SALT_WIDTH() + TSDB.metrics_width() + + Const.TIMESTAMP_BYTES; + + final FuzzyRowFilter fuzzy_filter; + if (explicit_tags && + enable_fuzzy_filter && + row_key_literals != null && + !row_key_literals.isEmpty()) { + + final byte[] fuzzy_key = new byte[prefix_width + (row_key_literals.size() * + (TSDB.tagk_width() + TSDB.tagv_width()))]; + System.arraycopy(scanner.getCurrentKey(), 0, fuzzy_key, 0, + scanner.getCurrentKey().length); + + final List fuzzy_filter_pairs = + buildFuzzyFilters(row_key_literals, fuzzy_key); + + // The Fuzzy Filter list is sorted: the first and last filters row key + // can be used to build the stop key for the scanner + final byte[] stop_key = Arrays.copyOf( + fuzzy_filter_pairs.get(fuzzy_filter_pairs.size() - 1).getRowKey(), + fuzzy_key.length); + System.arraycopy(scanner.getCurrentKey(), 0, stop_key, 0, prefix_width); + Internal.setBaseTime(stop_key, end_time); + int idx = prefix_width + TSDB.tagk_width(); + // max out the tag values + while (idx < stop_key.length) { + for (int i = 0; i < TSDB.tagv_width(); i++) { + stop_key[idx++] = (byte) 0xFF; + } + idx += TSDB.tagk_width(); + } + + scanner.setStartKey(fuzzy_key); + scanner.setStopKey(stop_key); + fuzzy_filter = new FuzzyRowFilter(fuzzy_filter_pairs); + } else { + fuzzy_filter = null; + } + + final String regex = getRowKeyUIDRegex(row_key_literals, explicit_tags); + final KeyRegexpFilter regex_filter; + if (!Strings.isNullOrEmpty(regex)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Regex for scanner: " + scanner + ": " + + byteRegexToString(regex)); + } + regex_filter = new KeyRegexpFilter(regex.toString(), + Const.ASCII_CHARSET); + } else { + regex_filter = null; + } + + if (fuzzy_filter != null && !Strings.isNullOrEmpty(regex)) { + final FilterList filter = new FilterList(Lists.newArrayList(fuzzy_filter, + regex_filter), Operator.MUST_PASS_ALL); + scanner.setFilter(filter); + } else if (fuzzy_filter != null) { + scanner.setFilter(fuzzy_filter); + } else if (!Strings.isNullOrEmpty(regex)) { + scanner.setFilter(regex_filter); + } + } + + /** + * Creates a regular expression with a list of or'd TUIDs to compare + * against the rows in storage. + * @param tsuids The list of TSUIDs to scan for + * @return A regular expression string to pass to the storage layer. + */ + public static String getRowKeyTSUIDRegex(final List tsuids) { + Collections.sort(tsuids); + + // first, convert the tags to byte arrays and count up the total length + // so we can allocate the string builder + final short metric_width = TSDB.metrics_width(); + int tags_length = 0; + final ArrayList uids = new ArrayList(tsuids.size()); + for (final String tsuid : tsuids) { + final String tags = tsuid.substring(metric_width * 2); + final byte[] tag_bytes = UniqueId.stringToUid(tags); + tags_length += tag_bytes.length; + uids.add(tag_bytes); + } + + // Generate a regexp for our tags based on any metric and timestamp (since + // those are handled by the row start/stop) and the list of TSUID tagk/v + // pairs. The generated regex will look like: ^.{7}(tags|tags|tags)$ + // where each "tags" is similar to \\Q\000\000\001\000\000\002\\E + final StringBuilder buf = new StringBuilder( + 13 // "(?s)^.{N}(" + ")$" + + (tsuids.size() * 11) // "\\Q" + "\\E|" + + tags_length); // total # of bytes in tsuids tagk/v pairs + + // Alright, let's build this regexp. From the beginning... + buf.append("(?s)" // Ensure we use the DOTALL flag. + + "^.{") + // ... start by skipping the metric ID and timestamp. + .append(Const.SALT_WIDTH() + metric_width + Const.TIMESTAMP_BYTES) + .append("}("); + + for (final byte[] tags : uids) { + // quote the bytes + buf.append("\\Q"); + addId(buf, tags, true); + buf.append('|'); + } + + // Replace the pipe of the last iteration, close and set + buf.setCharAt(buf.length() - 1, ')'); + buf.append("$"); + return buf.toString(); + } + + /** + * Compiles an HBase scanner against the main data table + * @param tsdb The TSDB with a configured HBaseClient + * @param salt_bucket An optional salt bucket ID for salting the start/stop + * keys. + * @param metric The metric to scan for + * @param start The start time stamp in seconds + * @param stop The stop timestamp in seconds + * @param table The table name to scan over + * @param family The table family to scan over + * @return A scanner ready for processing. + */ + public static Scanner getMetricScanner(final TSDB tsdb, final int salt_bucket, + final byte[] metric, final int start, final int stop, + final byte[] table, final byte[] family) { + final short metric_width = TSDB.metrics_width(); + final int metric_salt_width = metric_width + Const.SALT_WIDTH(); + final byte[] start_row = new byte[metric_salt_width + Const.TIMESTAMP_BYTES]; + final byte[] end_row = new byte[metric_salt_width + Const.TIMESTAMP_BYTES]; + + if (Const.SALT_WIDTH() > 0) { + final byte[] salt = RowKey.getSaltBytes(salt_bucket); + System.arraycopy(salt, 0, start_row, 0, Const.SALT_WIDTH()); + System.arraycopy(salt, 0, end_row, 0, Const.SALT_WIDTH()); + } + + Bytes.setInt(start_row, start, metric_salt_width); + Bytes.setInt(end_row, stop, metric_salt_width); + + System.arraycopy(metric, 0, start_row, Const.SALT_WIDTH(), metric_width); + System.arraycopy(metric, 0, end_row, Const.SALT_WIDTH(), metric_width); + + final Scanner scanner = tsdb.getClient().newScanner(table); + scanner.setMaxNumRows(tsdb.getConfig().scanner_maxNumRows()); + scanner.setStartKey(start_row); + scanner.setStopKey(end_row); + scanner.setFamily(family); + return scanner; + } + + /** + * Appends the given UID to the given regular expression buffer + * @param buf The String buffer to modify + * @param id The UID to add + * @param close Whether or not to append "\\E" to the end + */ + public static void addId(final StringBuilder buf, final byte[] id, + final boolean close) { + boolean backslash = false; + for (final byte b : id) { + buf.append((char) (b & 0xFF)); + if (b == 'E' && backslash) { // If we saw a `\' and now we have a `E'. + // So we just terminated the quoted section because we just added \E + // to `buf'. So let's put a litteral \E now and start quoting again. + buf.append("\\\\E\\Q"); + } else { + backslash = b == '\\'; + } + } + if (close) { + buf.append("\\E"); + } + } + + /** + * Little helper to print out the regular expression by converting the UID + * bytes to an array. + * @param regexp The regex string to print to the debug log + */ + public static String byteRegexToString(final String regexp) { + final StringBuilder buf = new StringBuilder(); + for (int i = 0; i < regexp.length(); i++) { + if (i > 0 && regexp.charAt(i - 1) == 'Q') { + if (regexp.charAt(i - 3) == '*') { + // tagk + byte[] tagk = new byte[TSDB.tagk_width()]; + for (int x = 0; x < TSDB.tagk_width(); x++) { + tagk[x] = (byte)regexp.charAt(i + x); + } + i += TSDB.tagk_width(); + buf.append(Arrays.toString(tagk)); + } else { + // tagv + byte[] tagv = new byte[TSDB.tagv_width()]; + for (int x = 0; x < TSDB.tagv_width(); x++) { + tagv[x] = (byte)regexp.charAt(i + x); + } + i += TSDB.tagv_width(); + buf.append(Arrays.toString(tagv)); + } + } else { + buf.append(regexp.charAt(i)); + } + } + return buf.toString(); + } + + /** + * Compares two timestamps where either can be in seconds or milliseconds. + * + * @param ts1 The first timestamp in either seconds or milliseconds. + * @param ts2 The second timestamp in either seconds or milliseconds. + * @return Whether the first timestamp is after the second + */ + public static boolean isTimestampAfter(long ts1, long ts2) { + boolean ts1InSeconds = (ts1 & Const.SECOND_MASK) == 0; + boolean ts2InSeconds = (ts2 & Const.SECOND_MASK) == 0; + + if (ts1InSeconds && !ts2InSeconds) { + ts1 *= 1000L; + } else if (!ts1InSeconds && ts2InSeconds) { + ts2 *= 1000L; + } + + return ts1 > ts2; + } +} diff --git a/src/query/expression/Absolute.java b/src/query/expression/Absolute.java new file mode 100644 index 0000000000..2b3cfb13f1 --- /dev/null +++ b/src/query/expression/Absolute.java @@ -0,0 +1,89 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.expression; + +import java.util.ArrayList; +import java.util.List; + +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.MutableDataPoint; +import net.opentsdb.core.SeekableView; +import net.opentsdb.core.TSQuery; + +/** + * Modifies each data point in the series with the absolute value, tossing away + * the signed component. + * @since 2.3 + */ +public class Absolute implements Expression { + + @Override + public DataPoints[] evaluate(final TSQuery data_query, + final List query_results, final List params) { + if (data_query == null) { + throw new IllegalArgumentException("Missing time series query"); + } + if (query_results == null || query_results.isEmpty()) { + return new DataPoints[]{}; + } + + int num_results = 0; + for (DataPoints[] results: query_results) { + num_results += results.length; + } + + final DataPoints[] results = new DataPoints[num_results]; + int ix = 0; + // one or more sub queries (m=...&m=...&m=...) + for (final DataPoints[] sub_query_result : query_results) { + // group bys (m=sum:foo{host=*}) + for (final DataPoints dps : sub_query_result) { + results[ix++] = abs(dps); + } + } + return results; + } + + /** + * Iterate over each data point and store the absolute value + * @param points The data points to modify + * @return The resulting data points + */ + private DataPoints abs(final DataPoints points) { + // TODO(cl) - Using an array as the size function may not return the exact + // results and we should figure a way to avoid copying data anyway. + final List dps = new ArrayList(); + + final SeekableView view = points.iterator(); + while (view.hasNext()) { + DataPoint pt = view.next(); + if (pt.isInteger()) { + dps.add(MutableDataPoint.ofLongValue( + pt.timestamp(), Math.abs(pt.longValue()))); + } else { + dps.add(MutableDataPoint.ofDoubleValue( + pt.timestamp(), Math.abs(pt.doubleValue()))); + } + } + final DataPoint[] results = new DataPoint[dps.size()]; + dps.toArray(results); + return new PostAggregatedDataPoints(points, results); + } + + @Override + public String writeStringField(List queryParams, String innerExpression) { + return "absolute(" + innerExpression + ")"; + } + +} diff --git a/src/query/expression/Alias.java b/src/query/expression/Alias.java new file mode 100644 index 0000000000..737d1c46ba --- /dev/null +++ b/src/query/expression/Alias.java @@ -0,0 +1,100 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.expression; + +import java.util.ArrayList; +import java.util.List; + +import com.google.common.base.Joiner; + +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.MutableDataPoint; +import net.opentsdb.core.SeekableView; +import net.opentsdb.core.TSQuery; + +/** + * Returns an alias if provided or the original metric name if not. The alias + * may optionally contain a template for tag replacement so that tags are + * advanced to the metric name for systems that require it. (e.g. flatten + * a name for Graphite). + * @since 2.3 + */ +public class Alias implements Expression { + + static Joiner COMMA_JOINER = Joiner.on(',').skipNulls(); + + @Override + public DataPoints[] evaluate(final TSQuery data_query, + final List query_results, final List params) { + if (data_query == null) { + throw new IllegalArgumentException("Missing time series query"); + } + if (query_results == null || query_results.isEmpty()) { + return new DataPoints[]{}; + } + if (params == null || params.isEmpty()) { + throw new IllegalArgumentException("Missing the alias"); + } + final String alias_template = COMMA_JOINER.join(params); + + int num_results = 0; + for (DataPoints[] results: query_results) { + num_results += results.length; + } + + final DataPoints[] results = new DataPoints[num_results]; + int ix = 0; + // one or more sub queries (m=...&m=...&m=...) + for (final DataPoints[] sub_query_result : query_results) { + // group bys (m=sum:foo{host=*}) + for (final DataPoints dps : sub_query_result) { + // TODO(cl) - Using an array as the size function may not return the exact + // results and we should figure a way to avoid copying data anyway. + final List new_dps_list = new ArrayList(); + final SeekableView view = dps.iterator(); + while (view.hasNext()) { + DataPoint pt = view.next(); + if (pt.isInteger()) { + new_dps_list.add(MutableDataPoint.ofLongValue( + pt.timestamp(), Math.abs(pt.longValue()))); + } else { + new_dps_list.add(MutableDataPoint.ofDoubleValue( + pt.timestamp(), Math.abs(pt.doubleValue()))); + } + } + + final DataPoint[] new_dps = new DataPoint[dps.size()]; + new_dps_list.toArray(new_dps); + final PostAggregatedDataPoints padps = new PostAggregatedDataPoints( + dps, new_dps); + + padps.setAlias(alias_template); + results[ix++] = padps; + } + } + return results; + } + + @Override + public String writeStringField(final List query_params, + final String innerExpression) { + final StringBuilder buf = new StringBuilder(); + buf.append("alias(") + .append(innerExpression) + .append(query_params == null || query_params.isEmpty() + ? "" : "," + COMMA_JOINER.join(query_params)) + .append(")"); + return buf.toString(); + } +} \ No newline at end of file diff --git a/src/query/expression/DiffSeries.java b/src/query/expression/DiffSeries.java new file mode 100644 index 0000000000..be8634b628 --- /dev/null +++ b/src/query/expression/DiffSeries.java @@ -0,0 +1,85 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.expression; + +import java.util.List; + +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.TSDB; +import net.opentsdb.core.TSQuery; +import net.opentsdb.query.expression.VariableIterator.SetOperator; + +/** + * Performs a UNION set join on up to 26 metric query results and returns the + * difference. + */ +public class DiffSeries implements Expression { + /** The TSDB used for UID to name lookups */ + final TSDB tsdb; + + /** + * Default ctor. + * @param tsdb The TSDB used for UID to name lookups + */ + public DiffSeries(final TSDB tsdb) { + this.tsdb = tsdb; + } + + @Override + public DataPoints[] evaluate(final TSQuery data_query, + final List query_results, final List params) { + if (data_query == null) { + throw new IllegalArgumentException("Missing time series query"); + } + if (query_results == null || query_results.isEmpty()) { + return new DataPoints[]{}; + } + + if (query_results.size() < 2 || query_results.size() > 26) { + throw new IllegalArgumentException("Must have 2 to 26 series, got " + + query_results.size() + " instead"); + } + + final StringBuilder buf = new StringBuilder(); + char v = 'a'; + for (int i = 0; i < query_results.size(); i++) { + buf.append(v++); + if (i < query_results.size() - 1) { + buf.append(" - "); + } + } + + final ExpressionIterator expression = new ExpressionIterator("diffSeries", + buf.toString(), SetOperator.UNION, false, false); + v = 'a'; + for (final DataPoints[] dps : query_results) { + final TimeSyncedIterator it = new TimeSyncedIterator( + Character.toString(v++), null, dps); + expression.addResults(it.getId(), it); + } + expression.compile(); + + final DataPoints[] results = new DataPoints[expression.values().length]; + for (int i = 0; i < expression.values().length; i++) { + results[i] = new EDPtoDPS(tsdb, i, expression); + } + return results; + } + + @Override + public String writeStringField(final List query_params, + final String inner_expression) { + return "diffSeries(" + inner_expression + ")"; + } + +} diff --git a/src/query/expression/DivideSeries.java b/src/query/expression/DivideSeries.java new file mode 100644 index 0000000000..cda975cd3f --- /dev/null +++ b/src/query/expression/DivideSeries.java @@ -0,0 +1,86 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.expression; + +import java.util.List; + +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.TSDB; +import net.opentsdb.core.TSQuery; +import net.opentsdb.query.expression.VariableIterator.SetOperator; + +/** + * Performs a UNION set join on up to 26 metric query results and returns the + * quotient. + */ +public class DivideSeries implements Expression { + /** The TSDB used for UID to name lookups */ + final TSDB tsdb; + + /** + * Default ctor. + * @param tsdb The TSDB used for UID to name lookups + */ + public DivideSeries(final TSDB tsdb) { + this.tsdb = tsdb; + } + + @Override + public DataPoints[] evaluate(final TSQuery data_query, + final List query_results, final List params) { + if (data_query == null) { + throw new IllegalArgumentException("Missing time series query"); + } + if (query_results == null || query_results.isEmpty()) { + return new DataPoints[]{}; + } + + if (query_results.size() < 2 || query_results.size() > 26) { + throw new IllegalArgumentException("Must have 2 to 26 series, got " + + query_results.size() + " instead"); + } + + final StringBuilder buf = new StringBuilder(); + char v = 'a'; + for (int i = 0; i < query_results.size(); i++) { + buf.append(v++); + if (i < query_results.size() - 1) { + buf.append(" / "); + } + } + + final ExpressionIterator expression = new ExpressionIterator("divideSeries", + buf.toString(), SetOperator.UNION, false, false); + v = 'a'; + + for (final DataPoints[] dps : query_results) { + final TimeSyncedIterator it = new TimeSyncedIterator( + Character.toString(v++), null, dps); + expression.addResults(it.getId(), it); + } + expression.compile(); + + final DataPoints[] results = new DataPoints[expression.values().length]; + for (int i = 0; i < expression.values().length; i++) { + results[i] = new EDPtoDPS(tsdb, i, expression); + } + return results; + } + + @Override + public String writeStringField(final List query_params, + final String inner_expression) { + return "divideSeries(" + inner_expression + ")"; + } + +} diff --git a/src/query/expression/EDPtoDPS.java b/src/query/expression/EDPtoDPS.java new file mode 100644 index 0000000000..5666ce34e4 --- /dev/null +++ b/src/query/expression/EDPtoDPS.java @@ -0,0 +1,256 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.expression; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import org.hbase.async.Bytes.ByteMap; + +import com.stumbleupon.async.Callback; +import com.stumbleupon.async.Deferred; + +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.MutableDataPoint; +import net.opentsdb.core.SeekableView; +import net.opentsdb.core.TSDB; +import net.opentsdb.core.Tags; +import net.opentsdb.meta.Annotation; +import net.opentsdb.uid.UniqueId.UniqueIdType; +import net.opentsdb.utils.ByteSet; + +/** + * An ugly temporary class for converting from an expression datapoint to a + * standard data point for serialization in the default query format. + */ +public class EDPtoDPS implements DataPoints { + /** The TSDB used for UID to name lookups */ + private final TSDB tsdb; + + /** The index of this data point in the iterator */ + private final int index; + + /** The iterator that contains the results for this data point */ + private final ExpressionIterator iterator; + + /** The list of data points from the iterator from which we read */ + private final ExpressionDataPoint[] edps; + + /** + * Default ctor + * @param tsdb The TSDB used for UID to name lookups + * @param index The index of this data point in the iterator + * @param iterator The iterator that contains the results for this data point + */ + public EDPtoDPS(final TSDB tsdb, final int index, + final ExpressionIterator iterator) { + this.tsdb = tsdb; + this.index = index; + this.iterator = iterator; + edps = iterator.values(); + } + + @Override + public String metricName() { + try { + return metricNameAsync().joinUninterruptibly(); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Should never be here", e); + } + } + + @Override + public Deferred metricNameAsync() { + if (edps[index].metricUIDs() == null) { + throw new IllegalStateException("Iterator UID was null for index " + + index + " and iterator " + iterator); + } + final byte[] uid = edps[index].metricUIDs().iterator().next(); + return tsdb.getUidName(UniqueIdType.METRIC, uid); + } + + @Override + public byte[] metricUID() { + if (edps[index].metricUIDs() == null) { + throw new IllegalStateException("Iterator UID was null for index " + + index + " and iterator " + iterator); + } + return edps[index].metricUIDs().iterator().next(); + } + + @Override + public Map getTags() { + try { + return getTagsAsync().joinUninterruptibly(); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Should never be here", e); + } + } + + @Override + public Deferred> getTagsAsync() { + return Tags.getTagsAsync(tsdb, edps[index].tags()); + } + + @Override + public ByteMap getTagUids() { + return edps[index].tags(); + } + + @Override + public List getAggregatedTags() { + try { + return getAggregatedTagsAsync().joinUninterruptibly(); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Should never be here", e); + } + } + + @Override + public Deferred> getAggregatedTagsAsync() { + final ByteSet tagks = edps[index].aggregatedTags(); + final List aggregated_tags = new ArrayList(tagks.size()); + + final List> names = + new ArrayList>(tagks.size()); + for (final byte[] tagk : tagks) { + names.add(tsdb.getUidName(UniqueIdType.TAGK, tagk)); + } + + /** Adds the names to the aggregated_tags list */ + final class ResolveCB implements Callback, ArrayList> { + @Override + public List call(final ArrayList names) throws Exception { + for (final String name : names) { + aggregated_tags.add(name); + } + return aggregated_tags; + } + } + + return Deferred.group(names).addCallback(new ResolveCB()); + } + + @Override + public List getAggregatedTagUids() { + final List agg_tags = new ArrayList( + edps[index].aggregatedTags()); + return agg_tags; + } + + @Override + public List getTSUIDs() { + // TODO Fix it up + return Collections.emptyList(); + } + + @Override + public List getAnnotations() { + // TODO Fix it up + return Collections.emptyList(); + } + + @Override + public int size() { + // TODO Estimate + return -1; + } + + @Override + public int aggregatedSize() { + // TODO Estimate + return -1; + } + + @Override + public SeekableView iterator() { + return new Iterator(); + } + + @Override + public long timestamp(int i) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isInteger(int i) { + throw new UnsupportedOperationException(); + } + + @Override + public long longValue(int i) { + throw new UnsupportedOperationException(); + } + + @Override + public double doubleValue(int i) { + throw new UnsupportedOperationException(); + } + + @Override + public int getQueryIndex() { + // TODO Fix it up + return 0; + } + + @Override + public boolean isPercentile() { + return false; + } + + @Override + public float getPercentile() { + throw new UnsupportedOperationException("getPercentile not supported"); + } + + /** + * Simple class that fills the local data point while iterating through the + * expression data points at the proper index. + */ + private class Iterator implements SeekableView { + /** A data pont to mutate as we iterate */ + final MutableDataPoint dp = new MutableDataPoint(); + + @Override + public boolean hasNext() { + return iterator.hasNext(index); + } + + @Override + public DataPoint next() { + iterator.next(index); + dp.reset(edps[index].timestamp(), edps[index].toDouble()); + return dp; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + @Override + public void seek(long timestamp) { + throw new UnsupportedOperationException(); + } + + } +} diff --git a/src/query/expression/Expression.java b/src/query/expression/Expression.java new file mode 100644 index 0000000000..d7d15acc1d --- /dev/null +++ b/src/query/expression/Expression.java @@ -0,0 +1,46 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.expression; + +import java.util.List; + +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.TSQuery; + +/** + * The interface for various expressions/functions used when querying OpenTSDB. + * @since 2.3 + */ +public interface Expression { + + /** + * Computes a set of results given the results of a {@link TSQuery} that may + * include multiple metrics and/or group by result sets. + * @param data_query The original query from the user + * @param results The results of the query + * @param params Parameters parsed from the expression endpoint related to + * the implementing function + * @return An array of data points resulting from the implementation + */ + public DataPoints[] evaluate(TSQuery data_query, + List results, List params); + + /** + * TODO - document me! + * @param params + * @param inner_expression + * @return + */ + public String writeStringField(List params, String inner_expression); + +} diff --git a/src/query/expression/ExpressionDataPoint.java b/src/query/expression/ExpressionDataPoint.java new file mode 100644 index 0000000000..a7eaba7f88 --- /dev/null +++ b/src/query/expression/ExpressionDataPoint.java @@ -0,0 +1,258 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.expression; + +import java.util.HashSet; +import java.util.Set; + +import org.hbase.async.Bytes.ByteMap; + +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.MutableDataPoint; +import net.opentsdb.utils.ByteSet; + +/** + * Contains the information for a series that has been processed through an + * expression iterator. Each time a metric data point series set is added we + * add the metric and compute the tag sets. + *

    + * As the iterator progresses, it will call into the {@link #reset} methods. + * @since 2.3 + */ +public class ExpressionDataPoint implements DataPoint { + + /** A list of metric UIDs wrapped up into this expression result */ + private final ByteSet metric_uids; + + /** The list of tag key/value pairs common to all series in this expression */ + private final ByteMap tags; + + /** The list of aggregated tag keys common to all series in this expression */ + private final ByteSet aggregated_tags; + + /** The list of TSUIDs from all series in this expression */ + private final Set tsuids; + + /** The size of the aggregated results. + * TODO - this is simply the size of the first series added. We need a way + * to compute this properly. + */ + private long size; + + /** The total number of raw data points in all series */ + private long raw_size; + + /** The data point overwritten each time through the iterator */ + private final MutableDataPoint dp; + + /** An index in the original {@link TimeSyncedIterator} iterator array */ + private int index; + + /** + * Default ctor that simply sets up new objects for all internal fields. + * TODO - lazily initialize the field to avoid unused objects + */ + public ExpressionDataPoint() { + metric_uids = new ByteSet(); + tags = new ByteMap(); + aggregated_tags = new ByteSet(); + tsuids = new HashSet(); + dp = new MutableDataPoint(); + } + + /** + * Ctor that sets up the meta data maps and initializes an empty dp + * @param dps The data point to pull meta from + */ + @SuppressWarnings("unchecked") + public ExpressionDataPoint(final DataPoints dps) { + metric_uids = new ByteSet(); + metric_uids.add(dps.metricUID()); + tags = dps.getTagUids() != null ? + (ByteMap) dps.getTagUids().clone() : new ByteMap(); + aggregated_tags = new ByteSet(); + if (dps.getAggregatedTagUids() != null) { + for (final byte[] tagk : dps.getAggregatedTagUids()) { + aggregated_tags.add(tagk); + } + } + tsuids = new HashSet(dps.getTSUIDs()); + // TODO - restore when these are faster + //size = dps.size(); + //raw_size = dps.aggregatedSize(); + dp = new MutableDataPoint(); + dp.reset(Long.MAX_VALUE, Double.NaN); + } + + /** + * Ctor that clones the meta data of the existing dps and sets up an empty value + * @param dps The data point to pull meta from + */ + @SuppressWarnings("unchecked") + public ExpressionDataPoint(final ExpressionDataPoint dps) { + metric_uids = new ByteSet(); + metric_uids.addAll(dps.metric_uids); + tags = (ByteMap) dps.tags.clone(); + aggregated_tags = new ByteSet(); + aggregated_tags.addAll(dps.aggregated_tags); + tsuids = new HashSet(dps.tsuids); + size = dps.size; + raw_size = dps.raw_size; + dp = new MutableDataPoint(); + dp.reset(Long.MAX_VALUE, Double.NaN); + } + + /** + * Add another metric series to this collection, computing the tag and + * agg intersections and incrementing the size. + * @param dps The series to add + */ + public void add(final DataPoints dps) { + metric_uids.add(dps.metricUID()); + + // TODO - tags intersection + + for (final byte[] tagk : dps.getAggregatedTagUids()) { + aggregated_tags.add(tagk); + } + + tsuids.addAll(dps.getTSUIDs()); + // TODO - this ain't right. We need to number of dps emitted from HERE. For + // now we'll just take the first dps size. If it's downsampled then this + // will be accurate. + //size += dps.size(); + // TODO - restore when this is faster + //raw_size += dps.aggregatedSize(); + } + + /** + * Add another metric series to this collection, computing the tag and + * agg intersections and incrementing the size. + * @param dps The series to add + */ + public void add(final ExpressionDataPoint dps) { + metric_uids.addAll(dps.metric_uids); + + // TODO - tags intersection + + aggregated_tags.addAll(dps.aggregated_tags); + + tsuids.addAll(dps.tsuids); + // TODO - this ain't right. We need to number of dps emitted from HERE. For + // now we'll just take the first dps size. If it's downsampled then this + // will be accurate. + //size += dps.size(); + raw_size += dps.raw_size; + } + + /** @return the metric UIDs */ + public ByteSet metricUIDs() { + return metric_uids; + } + + /** @return the list of common tag pairs in the series */ + public ByteMap tags() { + return tags; + } + + /** @return the list of aggregated tags */ + public ByteSet aggregatedTags() { + return aggregated_tags; + } + + /** @return the list of TSUIDs aggregated into this series */ + public Set tsuids() { + return tsuids; + } + + /** @return the aggregated number of data points in this series */ + public long size() { + return size; + } + + /** @return the number of raw data points in this series */ + public long rawSize() { + return raw_size; + } + + /** + * Stores a Double data point + * @param timestamp The timestamp + * @param value The value + */ + public void reset(final long timestamp, final double value) { + dp.reset(timestamp, value); + } + + /** + * Stores a data point pulled from the given data point interface + * @param dp the data point to read from + */ + public void reset(final DataPoint dp) { + this.dp.reset(dp); + } + + @Override + public String toString() { + final StringBuffer buf = new StringBuffer(); + buf.append("ExpressionDataPoint(metricUIDs=") + .append(metric_uids) + .append(", tsuids=") + .append(tsuids) + .append(")"); + return buf.toString(); + } + + // DataPoint implementations + + @Override + public long timestamp() { + return dp.timestamp(); + } + + @Override + public boolean isInteger() { + return dp.isInteger(); + } + + @Override + public long longValue() { + return dp.longValue(); + } + + @Override + public double doubleValue() { + return dp.doubleValue(); + } + + @Override + public double toDouble() { + return dp.toDouble(); + } + + /** @param index The index in the {@link TimeSyncedIterator} array */ + public void setIndex(final int index) { + this.index = index; + } + + /** @return the index in the {@link TimeSyncedIterator} array */ + public int getIndex() { + return index; + } + + @Override + public long valueCount() { + return 1; + } +} diff --git a/src/query/expression/ExpressionFactory.java b/src/query/expression/ExpressionFactory.java new file mode 100644 index 0000000000..43358e6eb6 --- /dev/null +++ b/src/query/expression/ExpressionFactory.java @@ -0,0 +1,96 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.expression; + +import java.util.HashMap; +import java.util.Map; + +import net.opentsdb.core.TSDB; + +/** + * A static class that stores and instantiates a static map of the available + * functions. + * TODO - Enable plugable expression and load from the class path. + * Since 2.3 + */ +public final class ExpressionFactory { + + private static Map available_functions = + new HashMap(); + + static { + available_functions.put("alias", new Alias()); + available_functions.put("scale", new Scale()); + available_functions.put("absolute", new Absolute()); + available_functions.put("movingAverage", new MovingAverage()); + available_functions.put("highestCurrent", new HighestCurrent()); + available_functions.put("highestMax", new HighestMax()); + available_functions.put("shift", new TimeShift()); + available_functions.put("timeShift", new TimeShift()); + available_functions.put("firstDiff", new FirstDifference()); + } + + /** Don't instantiate me! */ + private ExpressionFactory() { } + + /** + * Adds more functions to the map that depend on an instantiated TSDB object. + * Only call this once please. + * @param tsdb The TSDB object to initialize with + */ + public static void addTSDBFunctions(final TSDB tsdb) { + available_functions.put("divideSeries", new DivideSeries(tsdb)); + available_functions.put("divide", new DivideSeries(tsdb)); + available_functions.put("sumSeries", new SumSeries(tsdb)); + available_functions.put("sum", new SumSeries(tsdb)); + available_functions.put("diffSeries", new DiffSeries(tsdb)); + available_functions.put("difference", new DiffSeries(tsdb)); + available_functions.put("multiplySeries", new MultiplySeries(tsdb)); + available_functions.put("multiply", new MultiplySeries(tsdb)); + } + + /** + * Add an expression to the map. + * WARNING: The map is not thread safe so don't use this to dynamically + * modify the map while the TSD is running. + * @param name The name of the expression + * @param expr The expression object to store. + * @throws IllegalArgumentException if the name is null or empty or the + * function is null. + */ + static void addFunction(final String name, final Expression expr) { + if (name == null || name.isEmpty()) { + throw new IllegalArgumentException("Missing function name"); + } + if (expr == null) { + throw new IllegalArgumentException("Function cannot be null"); + } + available_functions.put(name, expr); + } + + /** + * Returns the expression function given the name + * @param function The name of the expression to use + * @return The expression when located + * @throws UnsupportedOperationException if the requested function hasn't + * been stored in the map. + */ + public static Expression getByName(final String function) { + final Expression expression = available_functions.get(function); + if (expression == null) { + throw new UnsupportedOperationException("Function " + function + + " has not been implemented"); + } + return expression; + } +} diff --git a/src/query/expression/ExpressionIterator.java b/src/query/expression/ExpressionIterator.java new file mode 100644 index 0000000000..76a93b860c --- /dev/null +++ b/src/query/expression/ExpressionIterator.java @@ -0,0 +1,487 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.expression; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import net.opentsdb.core.FillPolicy; +import net.opentsdb.core.IllegalDataException; +import net.opentsdb.query.expression.VariableIterator.SetOperator; +import net.opentsdb.utils.ByteSet; + +import org.apache.commons.jexl2.JexlContext; +import org.apache.commons.jexl2.JexlEngine; +import org.apache.commons.jexl2.JexlException; +import org.apache.commons.jexl2.MapContext; +import org.apache.commons.jexl2.Script; +import org.apache.commons.jexl2.scripting.JexlScriptEngineFactory; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableSet; + +/** + * A iterator that applies an expression to the results of multiple sub queries. + * To use this class: + * - Instantiate with a valid expression + * - Call {@link #getVariableNames()} and iterate over a set of TSSubQueries and + * their results. For each query that matches a variable name, call + * {@link #addResults(String, ITimeSyncedIterator)} with the result set. + * - Call {@link #compile()} to setup the meta data, fills and compute the + * intersection of the series. + * - Call {@link #values()} and store the reference. Results for each + * series will be written here as you iterate. + * - Call {@link #hasNext()} and {@link #next(int)} to iterate over results. + * - At each iteration, fetch the timestamp and value from the data points array. + *

    + * Iteration is performed across all series supplied to the iterator, synchronizing + * on the timestamps and substituting fill values where appropriate. + *

    + * WARNING: You MUST supply a result set and associated sub query to match each + * of the variable names in the expression. If you fail to do so, when you call + * {@link #compile()} you'll get an exception. + *

    + * NOTE: Right now this class only supports intersection on the series so that + * each metric result must contain series with the same tags based on the flags + * provided in the ctor. + * NOTE: If a result set doesn't include a fill policy, we default to ZERO for + * "missing" data points. + */ +public class ExpressionIterator implements ITimeSyncedIterator { + private static final Logger LOG = LoggerFactory.getLogger(ExpressionIterator.class); + + /** This is only here to to force the shade plugin to include the class in the fat-jar */ + private static final JexlScriptEngineFactory JEXL_FACTORY = null; + + /** Docs don't say whether this is thread safe or not. SOME methods are marked + * as not thread safe, so I assume it's ok to instantiate one of these guys + * and keep creating scripts from it. + */ + public final static JexlEngine JEXL_ENGINE = new JexlEngine(); + + /** Whether or not to intersect on the query tagks instead of the result set + * tagks */ + private final boolean intersect_on_query_tagks; + + /** Whether or not to include the aggregated tags in the result set */ + private final boolean include_agg_tags; + + /** List of iterators and their IDs */ + private final Map results; + + /** The compiled expression */ + private final Script expression; + + /** The context where we'll dump results for processing through the expression */ + private final JexlContext context = new MapContext(); + + /** A list of unique variable names pulled from the expression */ + private final Set names; + + /** The intersection iterator we'll use for processing */ + // TODO - write an interface to allow other set operators, e.g. union, disjoint + private VariableIterator iterator; + + /** A map of results from the intersection iterator to pass to the expression */ + private Map iteration_results; + + /** The results of processing the expressions */ + private ExpressionDataPoint[] dps; + + /** The ID of this iterator */ + private final String id; + + /** The index of this iterator in expressions */ + private int index; + + /** A fill policy for this expression if data is missing */ + private NumericFillPolicy fill_policy; + + /** The set operator to use for joining sets */ + private SetOperator set_operator; + + // NOTE - if the query is set to NONE for the aggregation and the query has + // no tagk filters then we shouldn't set the II's intersect_on_query_tagks + /** + * Default Ctor that compiles the expression for use with this iterator. + * @param id The id of this iterator. + * @param expression The expression to compile and use + * @param set_operator The type of set operator to use + * @param intersect_on_query_tagks Whether or not to include only the query + * specified tags during intersection + * @param include_agg_tags Whether or not to include aggregated tags during + * intersection + * @throws IllegalArgumentException if the expression is null or empty or doesn't + * contain any variables. + * @throws JexlException if the expression isn't valid + */ + public ExpressionIterator(final String id, final String expression, + final SetOperator set_operator, + final boolean intersect_on_query_tagks, final boolean include_agg_tags) { + if (expression == null || expression.isEmpty()) { + throw new IllegalArgumentException("The expression cannot be null"); + } + if (set_operator == null) { + throw new IllegalArgumentException("The set operator cannot be null"); + } + this.id = id; + this.intersect_on_query_tagks = intersect_on_query_tagks; + this.include_agg_tags = include_agg_tags; + results = new HashMap(); + this.expression = JEXL_ENGINE.createScript(expression); + names = new HashSet(); + extractVariableNames(); + if (names.size() < 1) { + throw new IllegalArgumentException( + "The expression didn't appear to have any variables"); + } + this.set_operator = set_operator; + fill_policy = new NumericFillPolicy(FillPolicy.NOT_A_NUMBER); + } + + /** + * Copy constructor that setups up a dupe of this iterator with fresh sub + * iterator objects for use in a nested expression. + * @param iterator The expression to copy from. + */ + private ExpressionIterator(final ExpressionIterator iterator) { + id = iterator.id; + // need to recompile, don't know if we'll run into threading issues + expression = JEXL_ENGINE.createScript(iterator.expression.toString()); + intersect_on_query_tagks = iterator.intersect_on_query_tagks; + include_agg_tags = iterator.include_agg_tags; + set_operator = iterator.set_operator; + + results = new HashMap(); + for (Entry entry : iterator.results.entrySet()) { + results.put(entry.getKey(), entry.getValue().getCopy()); + } + + names = new HashSet(); + extractVariableNames(); + if (names.size() < 1) { + throw new IllegalArgumentException( + "The expression didn't appear to have any variables"); + } + } + + @Override + public String toString() { + final StringBuffer buf = new StringBuffer(); + buf.append("ExpressionIterator(id=") + .append(id) + .append(", expression=\"") + .append(expression.toString()) + .append(", setOperator=") + .append(set_operator) + .append(", fillPolicy=") + .append(fill_policy) + .append(", intersectOnQueryTagks=") + .append(intersect_on_query_tagks) + .append(", includeAggTags=") + .append(include_agg_tags) + .append(", index=") + .append(index) + .append("\", VariableIterator=") + .append(iterator) + .append(", dps=") + .append(dps) + .append(", results=") + .append(results) + .append(")"); + return buf.toString(); + } + + /** + * Adds a sub query result object to the iterator. + * TODO - accept a proper object, not a map + * @param id The ID of source iterator. + * @param iterator The source iterator. + * @throws IllegalArgumentException if the object is missing required data + */ + public void addResults(final String id, final ITimeSyncedIterator iterator) { + if (id == null) { + throw new IllegalArgumentException("Missing ID"); + } + if (iterator == null) { + throw new IllegalArgumentException("Iterator cannot be null"); + } + results.put(id, iterator); + } + + /** + * Builds the iterator by computing the intersection of all series in all sets + * and sets up the output. + * @throws IllegalArgumentException if there aren't any results, or we don't + * have a result for each variable, or something else is wrong. + * @throws IllegalDataException if no series were left after computing the + * intersection. + */ + public void compile() { + if (LOG.isDebugEnabled()) { + LOG.debug("Compiling " + this); + } + if (results.size() < 1) { + throw new IllegalArgumentException("No results for any variables in " + + "the expression: " + this); + } + if (results.size() < names.size()) { + throw new IllegalArgumentException("Not enough query results [" + + results.size() + " total results found] for the expression variables [" + + names.size() + " expected] " + this); + } + + // don't care if we have extra results, but we had darned well better make + // sure we have a result set for each variable + for (final String variable : names) { + // validation + final ITimeSyncedIterator it = results.get(variable.toLowerCase()); + if (it == null) { + throw new IllegalArgumentException("Missing results for variable " + variable); + } + + if (it instanceof ExpressionIterator) { + ((ExpressionIterator)it).compile(); + } + if (LOG.isDebugEnabled()) { + LOG.debug("Matched variable " + variable + " to " + it); + } + } + + // TODO implement other set functions + switch (set_operator) { + case INTERSECTION: + iterator = new IntersectionIterator(id, results, intersect_on_query_tagks, + include_agg_tags); + break; + case UNION: + iterator = new UnionIterator(id, results, intersect_on_query_tagks, + include_agg_tags); + } + iteration_results = iterator.getResults(); + + dps = new ExpressionDataPoint[iterator.getSeriesSize()]; + for (int i = 0; i < iterator.getSeriesSize(); i++) { + final Iterator> it = + iteration_results.entrySet().iterator(); + Entry entry = it.next(); + + if (entry.getValue() == null || entry.getValue()[i] == null) { + dps[i] = new ExpressionDataPoint(); + } else { + dps[i] = new ExpressionDataPoint(entry.getValue()[i]); + } + while (it.hasNext()) { + entry = it.next(); + if (entry.getValue() != null && entry.getValue()[i] != null) { + dps[i].add(entry.getValue()[i]); + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Finished compiling " + this); + } + } + + /** + * Checks to see if we have another value in any of the series. + * Make sure to call {@link #compile()} first. + * @return True if there is more data to process, false if not + */ + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + /** + * Fetches the next set of data and computes a value for the expression. + * Make sure to call {@link #compile()} first. + * And make sure to call {@link #hasNext()} before calling this. + * @return A link to the data points for this result set + * @throws IllegalDataException if there wasn't any data left in any of the + * series. + * @throws JexlException if something went pear shaped processing the expression + */ + public ExpressionDataPoint[] next(final long timestamp) { + + // fetch the timestamp ONCE to save some cycles. + // final long timestamp = iterator.nextTimestamp(); + iterator.next(); + + // set aside a couple of addresses for the variables + double val; + double result; + for (int i = 0; i < iterator.getSeriesSize(); i++) { + // this here is why life sucks. there MUST be a better way to bind variables + for (final String variable : names) { + if (iteration_results.get(variable)[i] == null) { + context.set(variable, results.get(variable).getFillPolicy().getValue()); + } else { + val = iteration_results.get(variable)[i].toDouble(); + if (Double.isNaN(val)) { + context.set(variable, results.get(variable).getFillPolicy().getValue()); + } else { + context.set(variable, val); + } + } + } + final Object output = expression.execute(context); + if (output instanceof Double) { + result = (Double) output; + } else if (output instanceof Boolean) { + result = (((Boolean) output) ? 1 : 0); + } else { + throw new IllegalStateException("Expression returned a result of type: " + + output.getClass().getName() + " for " + this); + } + dps[i].reset(timestamp, result); + } + return dps; + } + + /** @return a list of expression results. You can keep this list and check the + * results on each call to {@link #next(int)} */ + @Override + public ExpressionDataPoint[] values() { + return dps; + } + + /** + * Pulls the variable names from the expression and stores them in {@link #names} + */ + private void extractVariableNames() { + if (expression == null) { + throw new IllegalArgumentException("The expression was null"); + } + + for (final List exp_list : JEXL_ENGINE.getVariables(expression)) { + for (final String variable : exp_list) { + names.add(variable); + } + } + } + + /** @return an immutable set of the variable IDs used in the expression. Case + * sensitive. */ + public Set getVariableNames() { + return ImmutableSet.copyOf(names); + } + + public void setSetOperator(final SetOperator set_operator) { + this.set_operator = set_operator; + } + + @Override + public long nextTimestamp() { + return iterator.nextTimestamp(); + } + + @Override + public int size() { + return dps.length; + } + + @Override + public void nullIterator(int index) { + if (index < 0 || index >= dps.length) { + throw new IllegalArgumentException("Index out of bounds"); + } + // TODO - do it + } + + @Override + public int getIndex() { + return index; + } + + @Override + public void setIndex(int index) { + this.index = index; + } + + @Override + public String getId() { + return id; + } + + @Override + public ByteSet getQueryTagKs() { + return null; + } + + @Override + public void setFillPolicy(NumericFillPolicy policy) { + fill_policy = policy; + } + + @Override + public NumericFillPolicy getFillPolicy() { + return fill_policy; + } + + @Override + public ITimeSyncedIterator getCopy() { + final ExpressionIterator ei = new ExpressionIterator(this); + return ei; + } + + @Override + public boolean hasNext(final int i) { + return iterator.hasNext(i); + } + + @Override + public void next(final int i) { + iterator.next(i); + + // set aside a couple of addresses for the variables + double val; + double result; + // this here is why life sucks. there MUST be a better way to bind variables + long ts = Long.MAX_VALUE; + for (final String variable : names) { + if (iteration_results.get(variable)[i] == null) { + context.set(variable, results.get(variable).getFillPolicy().getValue()); + } else { + if (iteration_results.get(variable)[i].timestamp() < ts) { + ts = iteration_results.get(variable)[i].timestamp(); + } + val = iteration_results.get(variable)[i].toDouble(); + if (Double.isNaN(val)) { + context.set(variable, results.get(variable).getFillPolicy().getValue()); + } else { + context.set(variable, val); + } + } + } + final Object output = expression.execute(context); + if (output instanceof Double) { + result = (Double) output; + } else if (output instanceof Boolean) { + result = (((Boolean) output) ? 1 : 0); + } else { + throw new IllegalStateException("Expression returned a result of type: " + + output.getClass().getName() + " for " + this); + } + dps[i].reset(ts, result); + } + +} diff --git a/src/query/expression/ExpressionReader.java b/src/query/expression/ExpressionReader.java new file mode 100644 index 0000000000..68805dd90f --- /dev/null +++ b/src/query/expression/ExpressionReader.java @@ -0,0 +1,156 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.expression; + +import java.util.NoSuchElementException; + +/** + * Parses a Graphite style expression. + * Please use {@link #isEOF()} before any method call. Otherwise the methods + * will throw a NoSuchElementException. + * @since 2.3 + */ +public class ExpressionReader { + /** The character array to parse */ + protected final char[] chars; + + /** The current index in the character array */ + private int mark = 0; + + /** + * Default ctor + * @param chars The characters to parse + */ + public ExpressionReader(final char[] chars) { + if (chars == null) { + throw new IllegalArgumentException("Character set cannot be null"); + } + this.chars = chars; + } + + /** @return the current index */ + public int getMark() { + return mark; + } + + /** @return the current character without advancing the index */ + public char peek() { + if (isEOF()) { + throw new NoSuchElementException("Index " + mark + " is out of bounds " + + chars.length); + } + return chars[mark]; + } + + /** @return the current character and advances the index */ + public char next() { + if (isEOF()) { + throw new NoSuchElementException("Index " + mark + " is out of bounds " + + chars.length); + } + return chars[mark++]; + } + + /** @param num the number of characters to skip */ + public void skip(final int num) { + if (num < 0) { + throw new UnsupportedOperationException("Skipping backwards is not allowed"); + } + mark += num; + } + + /** + * Checks to see if the next character matches the parameter + * @param c The character to check for + * @return True if they match, false if not + */ + public boolean isNextChar(final char c) { + return peek() == c; + } + + /** @return true if the given sequence appears next in the array. */ + public boolean isNextSeq(final CharSequence seq) { + if (seq == null) { + throw new IllegalArgumentException("Comparative sequence cannot be null"); + } + for (int i = 0; i < seq.length(); i++) { + if (mark + i >= chars.length) { + return false; + } + if (chars[mark + i] != seq.charAt(i)) { + return false; + } + } + + return true; + } + + /** @return the name of the function */ + public String readFuncName() { + // in case we get something like " function(foo)" consume a bit + skipWhitespaces(); + StringBuilder builder = new StringBuilder(); + while (peek() != '(' && !Character.isWhitespace(peek())) { + builder.append(next()); + } + skipWhitespaces(); // increment over whitespace after + return builder.toString(); + } + + /** @return Whether or not the index is at the end of the character array */ + public boolean isEOF() { + return mark >= chars.length; + } + + /** Increments the mark over white spaces */ + public void skipWhitespaces() { + for (int i = mark; i < chars.length; i++) { + if (Character.isWhitespace(chars[i])) { + mark++; + } else { + break; + } + } + } + + /** @return the next parameter from the expression + * TODO - may need some work */ + public String readNextParameter() { + final StringBuilder builder = new StringBuilder(); + int num_nested = 0; + while (!isEOF() && !Character.isWhitespace(peek())) { + final char ch = peek(); + if (ch == '(') { + num_nested++; + } else if (ch == ')') { + num_nested--; + } + + if (num_nested < 0) { + break; + } + if (num_nested <= 0 && isNextSeq(",,")) { + break; + } + builder.append(next()); + } + return builder.toString(); + } + + @Override + public String toString() { + // make a copy + return new String(chars); + } + +} diff --git a/src/query/expression/ExpressionTree.java b/src/query/expression/ExpressionTree.java new file mode 100644 index 0000000000..28bc5feaf2 --- /dev/null +++ b/src/query/expression/ExpressionTree.java @@ -0,0 +1,257 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.expression; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.IllegalDataException; +import net.opentsdb.core.TSQuery; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * A node in a tree of nested expressions. The tree may link to other nodes as + * sub expressions. Evaluating the tree evaluates all sub expressions. + *

    + * Before calling {@link evaluate} you MUST call a one or a combination of + * {@link addSubExpression}, {@link addSubMetricQuery} and optionally + * {@link addFunctionParameter} + *

    + * TODO(cl) - Cleanup needed. Tracking the indices can likely be done better + * and it would be good to have a ctor that sets the sub or metric query. + * @since 2.3 + */ +public class ExpressionTree { + /** Used for the toString() helpers */ + private static final Joiner DOUBLE_COMMA_JOINER = Joiner.on(",").skipNulls(); + + /** An enumerator of the different query types */ + enum Parameter { + SUB_EXPRESSION, + METRIC_QUERY + } + + /** The root expression for the tree */ + private final Expression expression; + /** The original time series query */ + private final TSQuery data_query; + /** An optional list of sub expressions */ + private List sub_expressions; + /** A list of parameters for the root expression */ + private List func_params; + /** A mapping of result indices to sub metric queries */ + private Map sub_metric_queries; + /** A mapping of query types to their result index */ + private Map parameter_index = Maps.newHashMap(); + + /** + * Creates a tree with a root and no children + * @param expression_name The name of the expression to lookup in the factory + * @param data_query The original query + * @throws UnsupportedOperationException if the expression is not implemented + */ + public ExpressionTree(final String expression_name, final TSQuery data_query) { + this(ExpressionFactory.getByName(expression_name), data_query); + } + + /** + * Creates a tree with a root and no children + * @param expression The expression to use + * @param data_query The original query + */ + public ExpressionTree(final Expression expression, final TSQuery data_query) { + this.expression = expression; + this.data_query = data_query; + } + + public void addSubExpression(final ExpressionTree child, final int param_index) { + if (child == null) { + throw new IllegalArgumentException("Cannot add a null child tree"); + } + if (child == this) { + throw new IllegalDataException("Recursive sub expression detected: " + + this); + } + if (param_index < 0) { + throw new IllegalArgumentException("Parameter index must be 0 or greater"); + } + if (sub_expressions == null) { + sub_expressions = Lists.newArrayList(); + } + sub_expressions.add(child); + parameter_index.put(param_index, Parameter.SUB_EXPRESSION); + } + + /** + * Sets the metric query key and index, setting the Parameter type to + * METRIC_QUERY + * @param metric_query The metric query id + * @param sub_query_index The index of the metric query + * @param param_index The index of the parameter (??) + */ + public void addSubMetricQuery(final String metric_query, + final int sub_query_index, + final int param_index) { + if (metric_query == null || metric_query.isEmpty()) { + throw new IllegalArgumentException("Metric query cannot be null or empty"); + } + if (sub_query_index < 0) { + throw new IllegalArgumentException("Sub query index must be 0 or greater"); + } + if (param_index < 0) { + throw new IllegalArgumentException("Parameter index must be 0 or greater"); + } + if (sub_metric_queries == null) { + sub_metric_queries = Maps.newHashMap(); + } + sub_metric_queries.put(sub_query_index, metric_query); + parameter_index.put(param_index, Parameter.METRIC_QUERY); + } + + /** + * Adds parameters for the root expression only. + * @param param The parameter to add, cannot be null or empty + * @throws IllegalArgumentException if the parameter is null or empty + */ + public void addFunctionParameter(final String param) { + if (param == null || param.isEmpty()) { + throw new IllegalArgumentException("Parameter cannot be null or empty"); + } + if (func_params == null) { + func_params = Lists.newArrayList(); + } + func_params.add(param); + } + + /** + * Processes the expression tree, including sub expressions, and returns the + * results. + * TODO(cl) - More tests around indices, etc. This can likely be cleaned up. + * @param query_results The result set to pass to the expressions + * @return The result set or an exception will bubble up if something wasn't + * configured properly. + */ + public DataPoints[] evaluate(final List query_results) { + // TODO - size the array + final List materialized = Lists.newArrayList(); + List metric_query_keys = null; + if (sub_metric_queries != null && sub_metric_queries.size() > 0) { + metric_query_keys = Lists.newArrayList(sub_metric_queries.keySet()); + Collections.sort(metric_query_keys); + } + + int metric_pointer = 0; + int sub_expression_pointer = 0; + for (int i = 0; i < parameter_index.size(); i++) { + final Parameter param = parameter_index.get(i); + + if (param == Parameter.METRIC_QUERY) { + if (metric_query_keys == null) { + throw new RuntimeException("Attempt to read metric " + + "results when none exist"); + } + + final int ix = metric_query_keys.get(metric_pointer++); + materialized.add(query_results.get(ix)); + } else if (param == Parameter.SUB_EXPRESSION) { + final ExpressionTree st = sub_expressions.get(sub_expression_pointer++); + materialized.add(st.evaluate(query_results)); + } else { + throw new IllegalDataException("Unknown parameter type: " + param + + " in tree: " + this); + } + } + + return expression.evaluate(data_query, materialized, func_params); + } + + @Override + public String toString() { + return writeStringField(); + } + + /** + * Helper to create the original expression (or at least a nested expression + * without the parameters included) + * @return A string representing the full expression. + */ + public String writeStringField() { + final List strs = Lists.newArrayList(); + if (sub_expressions != null) { + for (ExpressionTree sub : sub_expressions) { + strs.add(sub.toString()); + } + } + + if (sub_metric_queries != null) { + final String sub_metrics = clean(sub_metric_queries.values()); + if (sub_metrics != null && sub_metrics.length() > 0) { + strs.add(sub_metrics); + } + } + + final String inner_expression = DOUBLE_COMMA_JOINER.join(strs); + return expression.writeStringField(func_params, inner_expression); + } + + /** + * Helper to clean out some characters + * @param values The collection of strings to cleanup + * @return An empty string if values was empty or a cleaned up string + */ + private String clean(final Collection values) { + if (values == null || values.size() == 0) { + return ""; + } + + final List strs = Lists.newArrayList(); + for (String v : values) { + final String tmp = v.replaceAll("\\{.*\\}", ""); + final int ix = tmp.lastIndexOf(':'); + if (ix < 0) { + strs.add(tmp); + } else { + strs.add(tmp.substring(ix+1)); + } + } + + return DOUBLE_COMMA_JOINER.join(strs); + } + + @VisibleForTesting + List subExpressions() { + return sub_expressions; + } + + @VisibleForTesting + List funcParams() { + return func_params; + } + + @VisibleForTesting + Map subMetricQueries() { + return sub_metric_queries; + } + + @VisibleForTesting + Map parameterIndex() { + return parameter_index; + } +} \ No newline at end of file diff --git a/src/query/expression/Expressions.java b/src/query/expression/Expressions.java new file mode 100644 index 0000000000..e49943d6b6 --- /dev/null +++ b/src/query/expression/Expressions.java @@ -0,0 +1,166 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.expression; + +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; + +import net.opentsdb.core.TSQuery; +import net.opentsdb.query.expression.parser.ParseException; +import net.opentsdb.query.expression.parser.SyntaxChecker; + +/** + * Static class with helpers to parse and deal with expressions + * @since 2.3 + */ +public class Expressions { + + /** No instantiation for you! */ + private Expressions() { } + + /** + * Parses an expression into a tree + * @param expression The expression to parse (as a string) + * @param metric_queries A list to store the parsed metrics in + * @param data_query The time series query + * @return The parsed tree ready for evaluation + * @throws IllegalArgumentException if the expression was null, empty or + * invalid. + * @throws UnsupportedOperationException if the requested function couldn't + * be found. + */ + public static ExpressionTree parse(final String expression, + final List metric_queries, + final TSQuery data_query) { + if (expression == null || expression.isEmpty()) { + throw new IllegalArgumentException("Expression may not be null or empty"); + } + if (expression.indexOf('(') == -1 || expression.indexOf(')') == -1) { + throw new IllegalArgumentException("Invalid Expression: " + expression); + } + + final ExpressionReader reader = new ExpressionReader(expression.toCharArray()); + // consume any whitespace ahead of the expression + reader.skipWhitespaces(); + + final String function_name = reader.readFuncName(); + final Expression root_expression = ExpressionFactory.getByName(function_name); + + final ExpressionTree root = new ExpressionTree(root_expression, data_query); + reader.skipWhitespaces(); + + if (reader.peek() == '(') { + reader.next(); + parse(reader, metric_queries, root, data_query); + } + + return root; + } + + /** + * Parses a list of string expressions into the proper trees, adding the + * metrics to the {@code metric_queries} list. + * @param expressions A list of zero or more expressions (if empty, you get an + * empty tree list back) + * @param ts_query The original query with timestamps + * @param metric_queries The list to fill with metrics to fetch + */ + public static List parseExpressions( + final List expressions, + final TSQuery ts_query, + final List metric_queries) { + final List trees = + new ArrayList(expressions.size()); + for (final String expr: expressions) { + final SyntaxChecker checker = new SyntaxChecker(new StringReader(expr)); + checker.setMetricQueries(metric_queries); + checker.setTSQuery(ts_query); + try { + trees.add(checker.EXPRESSION()); + } catch (ParseException e) { + throw new IllegalArgumentException("Failed to parse " + expr, e); + } + } + return trees; + } + + /** + * Helper to parse out the function(s) and parameters + * @param reader The reader used for iterating over the expression + * @param metric_queries A list to store the parsed metrics in + * @param root The root tree + * @param data_query The time series query + */ + private static void parse(final ExpressionReader reader, + final List metric_queries, + final ExpressionTree root, + final TSQuery data_query) { + + int parameter_index = 0; + reader.skipWhitespaces(); + if (reader.peek() != ')') { + final String param = reader.readNextParameter(); + parseParam(param, metric_queries, root, data_query, parameter_index++); + } + + while (!reader.isEOF()) { + reader.skipWhitespaces(); + if (reader.peek() == ')') { + return; + } else if (reader.isNextSeq(",,")) { + reader.skip(2); //swallow the ",," delimiter + reader.skipWhitespaces(); + final String param = reader.readNextParameter(); + parseParam(param, metric_queries, root, data_query, parameter_index++); + } else { + throw new IllegalArgumentException("Invalid delimiter in parameter " + + "list at pos=" + reader.getMark() + ", expr=" + + reader.toString()); + } + } + } + + /** + * Helper that parses out the parameter from the expression + * @param param The parameter to parse + * @param metric_queries A list to store the parsed metrics in + * @param root The root tree + * @param data_query The time series query + * @param index Index of the parameter + */ + private static void parseParam(final String param, + final List metric_queries, + final ExpressionTree root, + final TSQuery data_query, + final int index) { + if (param == null || param.length() == 0) { + throw new IllegalArgumentException("Parameter cannot be null or empty"); + } + + if (param.indexOf('(') > 0 && param.indexOf(')') > 0) { + // sub expression + final ExpressionTree sub_tree = parse(param, metric_queries, data_query); + root.addSubExpression(sub_tree, index); + } else if (param.indexOf(':') >= 0) { + // metric query + metric_queries.add(param); + root.addSubMetricQuery(param, metric_queries.size() - 1, index); + } else { + // expression parameter + root.addFunctionParameter(param); + } + } + +} + diff --git a/src/query/expression/FirstDifference.java b/src/query/expression/FirstDifference.java new file mode 100644 index 0000000000..6dc75bc088 --- /dev/null +++ b/src/query/expression/FirstDifference.java @@ -0,0 +1,101 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.expression; + +import java.util.ArrayList; + +import java.util.List; + +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.IllegalDataException; +import net.opentsdb.core.MutableDataPoint; +import net.opentsdb.core.SeekableView; +import net.opentsdb.core.TSQuery; +import net.opentsdb.core.Aggregators.Interpolation; + +/** + * Implements a difference function, calculates the first difference of a given series + * + * @since 2.3 + */ +public class FirstDifference implements net.opentsdb.query.expression.Expression { + + @Override + public DataPoints[] evaluate(final TSQuery data_query, + final List query_results, final List params) { + if (data_query == null) { + throw new IllegalArgumentException("Missing time series query"); + } + if (query_results == null || query_results.isEmpty()) { + return new DataPoints[]{}; + } + + + int num_results = 0; + for (final DataPoints[] results : query_results) { + num_results += results.length; + } + final DataPoints[] results = new DataPoints[num_results]; + + int ix = 0; + // one or more sub queries (m=...&m=...&m=...) + for (final DataPoints[] sub_query_result : query_results) { + // group bys (m=sum:foo{host=*}) + for (final DataPoints dps : sub_query_result) { + results[ix++] = firstDiff(dps); + } + } + + return results; + + } + + /** + * return the first difference of datapoints + * + * @param points The data points to do difference + * @return The resulting data points + */ + private DataPoints firstDiff(final DataPoints points) { + final List dps = new ArrayList(); + final SeekableView view = points.iterator(); + List nums = new ArrayList(); + List times = new ArrayList(); + while (view.hasNext()) { + DataPoint pt = view.next(); + nums.add(pt.toDouble()); + times.add(pt.timestamp()); + } + List diff = new ArrayList(); + diff.add(0.0); + for (int j =0;j query_params, + final String inner_expression) { + return "firstDiff(" + inner_expression + ")"; + } + +} \ No newline at end of file diff --git a/src/query/expression/HighestCurrent.java b/src/query/expression/HighestCurrent.java new file mode 100644 index 0000000000..e6295206a3 --- /dev/null +++ b/src/query/expression/HighestCurrent.java @@ -0,0 +1,283 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.expression; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import net.opentsdb.core.AggregationIterator; +import net.opentsdb.core.Aggregator; +import net.opentsdb.core.Aggregators; +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.MutableDataPoint; +import net.opentsdb.core.SeekableView; +import net.opentsdb.core.TSQuery; +import net.opentsdb.core.Aggregators.Interpolation; +import net.opentsdb.query.expression.HighestMax.TopNSortingEntry; + +/** + * Implements top-n functionality by iterating over each of the time series, + * sorting and returning the top "n" time series with the highest current (or + * latest) value. + * @since 2.3 + */ +public class HighestCurrent implements Expression { + + @Override + public DataPoints[] evaluate(final TSQuery data_query, + final List query_results, final List params) { + if (data_query == null) { + throw new IllegalArgumentException("Missing time series query"); + } + if (query_results == null || query_results.isEmpty()) { + return new DataPoints[]{}; + } + // TODO(cl) - allow for empty top-n maybe? Just sort the results by max? + if (params == null || params.isEmpty()) { + throw new IllegalArgumentException("Need aggregation window for moving average"); + } + + String param = params.get(0); + if (param == null || param.length() == 0) { + throw new IllegalArgumentException("Missing top n value " + + "(number of series to return)"); + } + + int topn = 0; + if (param.matches("^[0-9]+$")) { + try { + topn = Integer.parseInt(param); + } catch (NumberFormatException nfe) { + throw new IllegalArgumentException( + "Invalid parameter, must be an integer", nfe); + } + } else { + throw new IllegalArgumentException("Unparseable top n value: " + param); + } + if (topn < 1) { + throw new IllegalArgumentException("Top n value must be greater " + + "than zero: " + topn); + } + + int num_results = 0; + for (DataPoints[] results: query_results) { + num_results += results.length; + } + + final PostAggregatedDataPoints[] post_agg_results = + new PostAggregatedDataPoints[num_results]; + int ix = 0; + // one or more sub queries (m=...&m=...&m=...) + for (final DataPoints[] sub_query_result : query_results) { + // group bys (m=sum:foo{host=*}) + for (final DataPoints dps : sub_query_result) { + // TODO(cl) - Avoid iterating and copying if we can help it. We should + // be able to pass the original DataPoints object to the seekable view + // and then iterate through it. + final List mutable_points = new ArrayList(); + for (final DataPoint point : dps) { + mutable_points.add(point.isInteger() ? + MutableDataPoint.ofLongValue(point.timestamp(), point.longValue()) + : MutableDataPoint.ofDoubleValue(point.timestamp(), point.doubleValue())); + } + // Because of AggregationIterator.hasNextValue() will skip empty item. + if (mutable_points.size() > 0) { + post_agg_results[ix++] = new PostAggregatedDataPoints(dps, + mutable_points.toArray(new DataPoint[mutable_points.size()])); + } + } + } + num_results = ix; + + final SeekableView[] views = new SeekableView[num_results]; + for (int i = 0; i < num_results; i++) { + views[i] = post_agg_results[i].iterator(); + } + + final MaxLatestAggregator aggregator = new + MaxLatestAggregator(Aggregators.Interpolation.LERP, + "maxLatest", num_results, data_query.startTime(), data_query.endTime()); + + final SeekableView view = (new AggregationIterator(views, + data_query.startTime(), data_query.endTime(), + aggregator, Aggregators.Interpolation.LERP, false)); + + // slurp all the points even though we aren't using them at this stage + while (view.hasNext()) { + final DataPoint mdp = view.next(); + @SuppressWarnings("unused") + final Object o = mdp.isInteger() ? mdp.longValue() : mdp.doubleValue(); + } + + final long[] max_longs = aggregator.getLongMaxes(); + final double[] max_doubles = aggregator.getDoubleMaxes(); + final TopNSortingEntry[] max_by_ts = + new TopNSortingEntry[num_results]; + if (aggregator.hasDoubles() && aggregator.hasLongs()) { + for (int i = 0; i < num_results; i++) { + max_by_ts[i] = new TopNSortingEntry( + Math.max((double)max_longs[i], max_doubles[i]), i); + } + } else if (aggregator.hasLongs() && !aggregator.hasDoubles()) { + for (int i = 0; i < num_results; i++) { + max_by_ts[i] = new TopNSortingEntry((double) max_longs[i], i); + } + } else if (aggregator.hasDoubles() && !aggregator.hasLongs()) { + for (int i = 0; i < num_results; i++) { + max_by_ts[i] = new TopNSortingEntry(max_doubles[i], i); + } + } + + Arrays.sort(max_by_ts); + + final int result_count = Math.min(topn, num_results); + final DataPoints[] results = new DataPoints[result_count]; + for (int i = 0; i < result_count; i++) { + results[i] = post_agg_results[max_by_ts[i].pos]; + } + + return results; + } + + @Override + public String writeStringField(final List query_params, + final String inner_expression) { + return "highestCurrent(" + inner_expression + ")"; + } + + /** + * Aggregator that stores only the latest value for each series so that they + * can be sorted on it + */ + public static class MaxLatestAggregator extends Aggregator { + /** The total number of series in the result set, including sub queries and + * group bys */ + private final int total_series; + /** An array of maximum integers by time series */ + private final long[] max_longs; + /** An array of maximum doubles by time series */ + private final double[] max_doubles; + /** Whether or not any of the series contain integers */ + private boolean has_longs = false; + /** Whether or not any of the series contain doubles */ + private boolean has_doubles = false; + /** Query start time in milliseconds for filtering */ + private long start; + /** Query end time in milliseconds for filtering */ + private long end; + /** The most recent timestamp in the different series */ + private long latest_ts = -1; + + /** + * An aggregator that keeps track of the maximum latest value for each series + * @param method The interpolation method (not used) + * @param name The name of the aggregator + * @param total_series The total number of series in the result set, + * including sub queries and group bys + * @param start Query start time in milliseconds for filtering + * @param end Query end time in milliseconds for filtering + */ + public MaxLatestAggregator(final Interpolation method, final String name, + final int total_series, final long start, final long end) { + super(method, name); + this.total_series = total_series; + this.start = start; + this.end = end; + this.max_longs = new long[total_series]; + this.max_doubles = new double[total_series]; + + for (int i = 0; i < total_series; i++) { + max_doubles[i] = Double.MIN_VALUE; + max_longs[i] = Long.MIN_VALUE; + } + } + + @Override + public long runLong(Longs values) { + // TODO(cl) - Can we get anything other than a DataPoint? + if (values instanceof DataPoint) { + long ts = ((DataPoint) values).timestamp(); + //data point falls outside required range + if (ts < start || ts > end) { + return 0; + } + } + + final long[] longs = new long[total_series]; + int ix = 0; + longs[ix++] = values.nextLongValue(); + while (values.hasNextValue()) { + longs[ix++] = values.nextLongValue(); + } + + if (values instanceof DataPoint) { + final long ts = ((DataPoint) values).timestamp(); + if (ts > latest_ts) { + System.arraycopy(longs, 0, max_longs, 0, total_series); + } + } + + has_longs = true; + return 0; + } + + @Override + public double runDouble(Doubles values) { + // TODO(cl) - Can we get anything other than a DataPoint? + if (values instanceof DataPoint) { + long ts = ((DataPoint) values).timestamp(); + //data point falls outside required range + if (ts < start || ts > end) { + return 0; + } + } + + // TODO(cl) - Properly handle NaNs here + final double[] doubles = new double[total_series]; + int ix = 0; + doubles[ix++] = values.nextDoubleValue(); + while (values.hasNextValue()) { + doubles[ix++] = values.nextDoubleValue(); + } + + if (values instanceof DataPoint) { + final long ts = ((DataPoint) values).timestamp(); + if (ts > latest_ts) { + System.arraycopy(doubles, 0, max_doubles, 0, total_series); + } + } + + has_doubles = true; + return 0; + } + + public long[] getLongMaxes() { + return max_longs; + } + + public double[] getDoubleMaxes() { + return max_doubles; + } + + public boolean hasLongs() { + return has_longs; + } + + public boolean hasDoubles() { + return has_doubles; + } + + } +} diff --git a/src/query/expression/HighestMax.java b/src/query/expression/HighestMax.java new file mode 100644 index 0000000000..0b942d5c1b --- /dev/null +++ b/src/query/expression/HighestMax.java @@ -0,0 +1,293 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2010-2012 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.expression; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; + +import net.opentsdb.core.AggregationIterator; +import net.opentsdb.core.Aggregator; +import net.opentsdb.core.Aggregators; +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.MutableDataPoint; +import net.opentsdb.core.SeekableView; +import net.opentsdb.core.TSQuery; +import net.opentsdb.core.Aggregators.Interpolation; + +/** + * Implements top-n functionality by iterating over each of the time series, + * finding the max value for each time series within the query time range, + * and up to "n" time series with the highest values, sorted in descending + * order. + * @since 2.3 + */ +public class HighestMax implements Expression { + + @Override + public DataPoints[] evaluate(final TSQuery data_query, + final List query_results, final List params) { + if (data_query == null) { + throw new IllegalArgumentException("Missing time series query"); + } + if (query_results == null || query_results.isEmpty()) { + return new DataPoints[]{}; + } + // TODO(cl) - allow for empty top-n maybe? Just sort the results by max? + if (params == null || params.isEmpty()) { + throw new IllegalArgumentException("Need aggregation window for moving average"); + } + + String param = params.get(0); + if (param == null || param.length() == 0) { + throw new IllegalArgumentException("Missing top n value " + + "(number of series to return)"); + } + + int topn = 0; + if (param.matches("^[0-9]+$")) { + try { + topn = Integer.parseInt(param); + } catch (NumberFormatException nfe) { + throw new IllegalArgumentException( + "Invalid parameter, must be an integer", nfe); + } + } else { + throw new IllegalArgumentException("Unparseable top n value: " + param); + } + if (topn < 1) { + throw new IllegalArgumentException("Top n value must be greater " + + "than zero: " + topn); + } + + int num_results = 0; + for (DataPoints[] results: query_results) { + num_results += results.length; + } + + final PostAggregatedDataPoints[] post_agg_results = + new PostAggregatedDataPoints[num_results]; + int ix = 0; + // one or more sub queries (m=...&m=...&m=...) + for (final DataPoints[] sub_query_result : query_results) { + // group bys (m=sum:foo{host=*}) + for (final DataPoints dps : sub_query_result) { + // TODO(cl) - Avoid iterating and copying if we can help it. We should + // be able to pass the original DataPoints object to the seekable view + // and then iterate through it. + final List mutable_points = new ArrayList(); + for (final DataPoint point : dps) { + mutable_points.add(point.isInteger() ? + MutableDataPoint.ofLongValue(point.timestamp(), point.longValue()) + : MutableDataPoint.ofDoubleValue(point.timestamp(), point.doubleValue())); + } + post_agg_results[ix++] = new PostAggregatedDataPoints(dps, + mutable_points.toArray(new DataPoint[mutable_points.size()])); + } + } + + final SeekableView[] views = new SeekableView[num_results]; + for (int i = 0; i < num_results; i++) { + views[i] = post_agg_results[i].iterator(); + } + + final MaxCacheAggregator aggregator = new MaxCacheAggregator( + Aggregators.Interpolation.LERP, "maxCache", num_results, + data_query.startTime(), data_query.endTime()); + + final SeekableView view = (new AggregationIterator(views, + data_query.startTime(), data_query.endTime(), + aggregator, Aggregators.Interpolation.LERP, false)); + + // slurp all the points even though we aren't using them at this stage + while (view.hasNext()) { + final DataPoint mdp = view.next(); + @SuppressWarnings("unused") + final Object o = mdp.isInteger() ? mdp.longValue() : mdp.doubleValue(); + } + + final long[] max_longs = aggregator.getLongMaxes(); + final double[] max_doubles = aggregator.getDoubleMaxes(); + final TopNSortingEntry[] max_by_ts = new TopNSortingEntry[num_results]; + if (aggregator.hasDoubles() && aggregator.hasLongs()) { + for (int i = 0; i < num_results; i++) { + max_by_ts[i] = new TopNSortingEntry( + Math.max((double)max_longs[i], max_doubles[i]), i); + } + } else if (aggregator.hasLongs() && !aggregator.hasDoubles()) { + for (int i = 0; i < num_results; i++) { + max_by_ts[i] = new TopNSortingEntry((double) max_longs[i], i); + } + } else if (aggregator.hasDoubles() && !aggregator.hasLongs()) { + for (int i = 0; i < num_results; i++) { + max_by_ts[i] = new TopNSortingEntry(max_doubles[i], i); + } + } + + Arrays.sort(max_by_ts); + + final int result_count = Math.min(topn, num_results); + final DataPoints[] results = new DataPoints[result_count]; + for (int i = 0; i < result_count; i++) { + results[i] = post_agg_results[max_by_ts[i].pos]; + } + + return results; + } + + /** + * Helper class for sorting the series. It will sort from highest to lowest. + */ + static class TopNSortingEntry implements Comparable { + final double val; + final int pos; + + public TopNSortingEntry(final double val, final int pos) { + this.val = val; + this.pos = pos; + } + + @Override + public String toString() { + return "{" + val + "," + pos + "}"; + } + + @Override + public int compareTo(final TopNSortingEntry o) { + return -1 * Double.compare(val, o.val); + } + } + + @Override + public String writeStringField(final List query_params, + final String inner_expression) { + return "highestMax(" + inner_expression + ")"; + } + + /** + * Aggregator that stores the overall maximum value for the entire series + */ + public static class MaxCacheAggregator extends Aggregator { + /** The total number of series in the result set, including sub queries and + * group bys */ + private final int total_series; + /** An array of maximum integers by time series */ + private final long[] max_longs; + /** An array of maximum doubles by time series */ + private final double[] max_doubles; + /** Whether or not any of the series contain integers */ + private boolean has_longs = false; + /** Whether or not any of the series contain doubles */ + private boolean has_doubles = false; + /** Query start time in milliseconds for filtering */ + private long start; + /** Query end time in milliseconds for filtering */ + private long end; + + /** + * An aggregator that keeps track of the maximum values for each time series + * in the result set. + * @param method The interpolation method (not used) + * @param name The name of the aggregator + * @param total_series The total number of series in the result set, + * including sub queries and group bys + * @param start Query start time in milliseconds for filtering + * @param end Query end time in milliseconds for filtering + */ + public MaxCacheAggregator(final Interpolation method, final String name, + final int total_series, final long start, final long end) { + super(method, name); + this.total_series = total_series; + this.start = start; + this.end = end; + this.max_longs = new long[total_series]; + this.max_doubles = new double[total_series]; + + for (int i = 0; i < total_series; i++) { + max_doubles[i] = Double.MIN_VALUE; + max_longs[i] = Long.MIN_VALUE; + } + } + + @Override + public long runLong(final Longs values) { + // TODO(cl) - Can we get anything other than a DataPoint? + if (values instanceof DataPoint) { + long ts = ((DataPoint) values).timestamp(); + //data point falls outside required range + if (ts < start || ts > end) { + return 0; + } + } + + final long[] longs = new long[total_series]; + int ix = 0; + longs[ix++] = values.nextLongValue(); + while (values.hasNextValue()) { + longs[ix++] = values.nextLongValue(); + } + + for (int i = 0; i < total_series;i++) { + max_longs[i] = Math.max(max_longs[i], longs[i]); + } + + has_longs = true; + return 0; + } + + @Override + public double runDouble(Doubles values) { + // TODO(cl) - Can we get anything other than a DataPoint? + if (values instanceof DataPoint) { + long ts = ((DataPoint) values).timestamp(); + //data point falls outside required range + if (ts < start || ts > end) { + return 0; + } + } + + final double[] doubles = new double[total_series]; + int ix = 0; + doubles[ix++] = values.nextDoubleValue(); + while (values.hasNextValue()) { + doubles[ix++] = values.nextDoubleValue(); + } + for (int i = 0; i < total_series;i++) { + // TODO(cl) - Properly handle NaNs here + max_doubles[i] = Math.max(max_doubles[i], doubles[i]); + } + + has_doubles = true; + return 0; + } + + public long[] getLongMaxes() { + return max_longs; + } + + public double[] getDoubleMaxes() { + return max_doubles; + } + + public boolean hasLongs() { + return has_longs; + } + + public boolean hasDoubles() { + return has_doubles; + } + + } +} diff --git a/src/query/expression/ITimeSyncedIterator.java b/src/query/expression/ITimeSyncedIterator.java new file mode 100644 index 0000000000..77aaec3eae --- /dev/null +++ b/src/query/expression/ITimeSyncedIterator.java @@ -0,0 +1,89 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.expression; + +import net.opentsdb.utils.ByteSet; + +/** + * An interface for expressions or queries that operate across time series + * and require point-by-point timestamp synchronization. + * @since 2.3 + */ +public interface ITimeSyncedIterator { + + /** @return true if any of the series in the set has another value */ + public boolean hasNext(); + + /** + * @param timestamp The timestamp to fastforward to + * @return The data point array for the given timestamp. Implementations + * may throw an exception if the timestamp is invliad or they may return an + * empty array. + */ + public ExpressionDataPoint[] next(final long timestamp); + + /** + * Determines whether the individual series in the {@link values} array has + * another value. This may be used for non-synchronous iteration. + * @param index The index of the series in the values array to check for + * @return True if the series has another value, false if not + */ + public boolean hasNext(final int index); + + /** + * Fetches the next value for an individual series in the {@link values} array. + * @param index The index of the series in the values array to advance + */ + public void next(final int index); + + /** + * @return the next timestamp available in this set. + */ + public long nextTimestamp(); + + /** @return the number of series in this set */ + public int size(); + + /** @return an array of the emitters populated during iteration */ + public ExpressionDataPoint[] values(); + + /** @param index the index to null. Nulls the given object so we don't use it + * in timestamps. + */ + public void nullIterator(final int index); + + /** @return the index in the ExpressionIterator */ + public int getIndex(); + + /** @param index the index in the ExpressionIterator */ + public void setIndex(final int index); + + /** @return the ID of this set given by the user */ + public String getId(); + + /** @return a set of unique tag key UIDs from the filter list. If no filters + * were defined then the set may be empty. */ + public ByteSet getQueryTagKs(); + + /** @param policy A fill policy for the iterator. Iterators should implement a default */ + public void setFillPolicy(final NumericFillPolicy policy); + + /** @return the fill policy for the iterator */ + public NumericFillPolicy getFillPolicy(); + + /** @return a copy of the iterator. This should return references to + * underlying data objects but not necessarily copy all of the underlying + * data (to avoid memory explosions). This is useful for creating other + * iterators that operate over the same data. */ + public ITimeSyncedIterator getCopy(); +} diff --git a/src/query/expression/IntersectionIterator.java b/src/query/expression/IntersectionIterator.java new file mode 100644 index 0000000000..0c5c4213e3 --- /dev/null +++ b/src/query/expression/IntersectionIterator.java @@ -0,0 +1,521 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.expression; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; + +import net.opentsdb.core.IllegalDataException; +import net.opentsdb.core.TSDB; +import net.opentsdb.utils.ByteSet; + +import org.hbase.async.Bytes; +import org.hbase.async.Bytes.ByteMap; +import org.hbase.async.HBaseClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import sun.reflect.generics.reflectiveObjects.NotImplementedException; + +/** + * This class handles taking a set of queries and their results and iterates + * over each series in each set with time alignment after computing the + * intersection of all sets. + *

    + * The iterator performs the following: + * - calculates the intersection of all queries based on the tags or query tags + * and optionally the aggregated tags. + * - any series that are not members of ever set are kicked out (and logged). + * - series are aligned across queries so that expressions can operate over them. + * - series are also time aligned and maintain alignment during iteration. + *

    + * The {@link #current_values} map will map the expression "variables" to the + * proper iterator for each serie's array. E.g. + * <"A", [1, 2, 3, 4]> + * <"B", [1, 2, 3, 4]> + *

    + * So to use it's you simply fetch the result map, call {@link #hasNext()} and + * {@link #next()} to iterate and in a for loop, iterate {@link #getSeriesSize()} + * times to get all of the current values. + * For efficiency, call {@link #getResults()} once before iterating, then on + * each call to {@link #next()} you can just iterate over the same result map + * again as the values will be updated. + * @since 2.3 + */ +public class IntersectionIterator implements ITimeSyncedIterator, VariableIterator { + private static final Logger LOG = LoggerFactory.getLogger(IntersectionIterator.class); + + /** The queries compiled and fetched from storage */ + private final Map queries; + + /** A list of the current values for each series post intersection */ + private final Map current_values; + + /** A map of the sub query index to their names for intersection computation */ + private final String[] index_to_names; + + /** Whether or not to intersect on the query tagks instead of the result set + * tagks */ + private final boolean intersect_on_query_tagks; + + /** Whether or not to include the aggregated tags in the result set */ + private final boolean include_agg_tags; + + /** The start/current timestamp for the iterator in ms */ + private long timestamp; + + /** Post intersection number of time series */ + private int series_size; + + /** The ID of this iterator */ + private final String id; + + /** The index of this iterator in a list of iterators */ + private int index; + + /** + * Ctor to create the expression lock-step iterator from a set of query results. + * If the results map is empty, then the ctor will complete but the results map + * will be empty and calls to {@link #hasNext()} will always return false. + * @param results The query results to store + * @param intersect_on_query_tagks Whether or not to include only the query + * specified tags during intersection + * @param include_agg_tags Whether or not to include aggregated tags during + * intersection + * @throws IllegalDataException if, after computing the intersection, no results + * would be left. + */ + public IntersectionIterator(final String id, final Map results, + final boolean intersect_on_query_tagks, final boolean include_agg_tags) { + this.id = id; + this.intersect_on_query_tagks = intersect_on_query_tagks; + this.include_agg_tags = include_agg_tags; + timestamp = Long.MAX_VALUE; + queries = new HashMap(results.size()); + current_values = new HashMap(results.size()); + index_to_names = new String[results.size()]; + + int max_series = 0; + int i = 0; + for (final Map.Entry entry : results.entrySet()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Adding iterator " + entry.getValue()); + } + queries.put(entry.getKey(), entry.getValue()); + entry.getValue().setIndex(i); + index_to_names[i] = entry.getKey(); + if (entry.getValue().values().length > max_series) { + max_series = entry.getValue().values().length; + } + ++i; + } + + if (max_series < 1) { + // we don't want to throw an exception here, just set it up so that the + // call to {@link #hasNext()} will be false. + LOG.debug("No series in the result sets"); + return; + } + + computeIntersection(); + + // calculate the starting timestamp from the various iterators + for (final ITimeSyncedIterator it : queries.values()) { + final long ts = it.nextTimestamp(); + if (ts < timestamp) { + timestamp = ts; + } + } + } + + /** + * A sort of copy constructor that populates the iterator from an existing + * iterator, copying all child iterators. + * @param iterator The iterator to copy from. + */ + private IntersectionIterator(final IntersectionIterator iterator) { + id = iterator.id; + intersect_on_query_tagks = iterator.intersect_on_query_tagks; + include_agg_tags = iterator.include_agg_tags; + timestamp = Long.MAX_VALUE; + queries = new HashMap(iterator.queries.size()); + current_values = new HashMap(queries.size()); + index_to_names = new String[queries.size()]; + + int max_series = 0; + int i = 0; + for (final Entry entry : iterator.queries.entrySet()) { + queries.put(entry.getKey(), entry.getValue().getCopy()); + entry.getValue().setIndex(i); + index_to_names[i] = entry.getKey(); + if (entry.getValue().values().length > max_series) { + max_series = entry.getValue().values().length; + } + ++i; + } + + if (max_series < 1) { + // we don't want to throw an exception here, just set it up so that the + // call to {@link #hasNext()} will be false. + LOG.debug("No series in the result sets"); + return; + } + + computeIntersection(); + + // calculate the starting timestamp from the various iterators + for (final ITimeSyncedIterator it : queries.values()) { + final long ts = it.nextTimestamp(); + if (ts < timestamp) { + timestamp = ts; + } + } + } + + @Override + public String toString() { + final StringBuilder buf = new StringBuilder(); + buf.append("IntersectionIterator(id=") + .append(id) + .append(", useQueryTags=") + .append(intersect_on_query_tagks) + .append(", includeAggTags=") + .append(include_agg_tags) + .append(", index=") + .append(index) + .append(", queries=") + .append(queries); + return buf.toString(); + } + + @Override + public boolean hasNext() { + for (final ITimeSyncedIterator sub : queries.values()) { + if (sub.hasNext()) { + return true; + } + } + return false; + } + + /** fetch the next set of time aligned results for all series */ + @Override + public void next() { + if (!hasNext()) { + throw new IllegalDataException("No more data"); + } + for (final ITimeSyncedIterator sub : queries.values()) { + sub.next(timestamp); + } + timestamp = nextTimestamp(); + } + + /** @return a map of values that will change on each iteration */ + @Override + public Map getResults() { + return current_values; + } + + /** @return the number of series in each map of the result set */ + @Override + public int getSeriesSize() { + return series_size; + } + + /** @return the next timestamp calculated from all series in the set */ + public long nextTimestamp() { + long ts = Long.MAX_VALUE; + for (final ITimeSyncedIterator sub : queries.values()) { + if (sub != null) { + final long t = sub.nextTimestamp(); + if (t < ts) { + ts = t; + } + } + } + return ts; + } + + /** + * A super ugly messy way to compute the intersection of the various sets of + * time series returned from the sub queries. + *

    + * The process is: + * - Iterate over each query set + * - For the first set, flatten each series' tag and (optionally) aggregated tag + * set into a single byte array for use as an ID. + * - Populate a map with the IDs and references to the series iterator for the + * first query set. + * - For each additional set, flatten the tags and if the tag set ID isn't in + * the intersection map, kick it out. + * - For each key in the intersection map, if it doesn't appear in the current + * query set, kick it out. + * - Once all sets are finished, align the resulting series iterators in the + * {@link #current_values} map which is then prepped for expression processing. + * @throws IllegalDataException if more than one series was supplied and + * the resulting intersection failed to produce any series + */ + private void computeIntersection() { + final ByteMap ordered_intersection = + new ByteMap(); + final Iterator it = queries.values().iterator(); + + // assume we have at least on query in our set + ITimeSyncedIterator sub = it.next(); + Map> flattened_tags = + new HashMap>(queries.size()); + ByteMap tags = new ByteMap(); + flattened_tags.put(sub.getId(), tags); + ExpressionDataPoint[] dps = sub.values(); + + for (int i = 0; i < sub.size(); i++) { + final byte[] tagks = flattenTags(intersect_on_query_tagks, include_agg_tags, + dps[i].tags(), dps[i].aggregatedTags(), sub); + tags.put(tagks, i); + + final ExpressionDataPoint[] idps = new ExpressionDataPoint[queries.size()]; + idps[sub.getIndex()] = dps[i]; + ordered_intersection.put(tagks, idps); + } + + if (!it.hasNext()) { + setCurrentAndMeta(ordered_intersection); + return; + } + + while (it.hasNext()) { + sub = it.next(); + tags = new ByteMap(); + flattened_tags.put(sub.getId(), tags); + dps = sub.values(); + + // loop through the series in the sub iterator, compute the flattened tag + // ids, then kick out any that are NOT in the existing intersection map. + for (int i = 0; i < sub.size(); i++) { + final byte[] tagks = flattenTags(intersect_on_query_tagks, include_agg_tags, + dps[i].tags(), dps[i].aggregatedTags(), sub); + tags.put(tagks, i); + + final ExpressionDataPoint[] idps = ordered_intersection.get(tagks); + if (idps == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Kicking out " + Bytes.pretty(tagks) + " from " + sub.getId()); + } + sub.nullIterator(i); + continue; + } + idps[sub.getIndex()] = dps[i]; + } + + // gotta go backwards now to complete the intersection by kicking + // any series that appear in other sets but not HERE + final Iterator> reverse_it = + ordered_intersection.iterator(); + while (reverse_it.hasNext()) { + Entry e = reverse_it.next(); + if (!tags.containsKey(e.getKey())) { + if (LOG.isDebugEnabled()) { + LOG.debug("Kicking out " + Bytes.pretty(e.getKey()) + + " from the main list since the query for " + sub.getId() + + " didn't have it"); + } + + // null the iterators for the other sets + for (final Map.Entry> entry : + flattened_tags.entrySet()) { + if (entry.getKey().equals(sub.getId())) { + continue; + } + final Integer index = entry.getValue().get(e.getKey()); + if (index != null) { + queries.get(entry.getKey()).nullIterator(index); + } + } + + reverse_it.remove(); + } + } + } + + // now set our properly condensed and ordered values + if (ordered_intersection.size() < 1) { + // TODO - is it best to toss an exception here or return an empty result? + throw new IllegalDataException("No intersections found: " + this); + } + + setCurrentAndMeta(ordered_intersection); + } + + /** + * Takes the resulting intersection and builds the {@link #current_values} + * and {@link #meta} maps. + * @param ordered_intersection The intersection to build from. + */ + private void setCurrentAndMeta(final ByteMap + ordered_intersection) { + for (final String id : queries.keySet()) { + current_values.put(id, new ExpressionDataPoint[ordered_intersection.size()]); + } + + int i = 0; + for (final ExpressionDataPoint[] idps : ordered_intersection.values()) { + for (int x = 0; x < idps.length; x++) { + final ExpressionDataPoint[] current_dps = + current_values.get(index_to_names[x]); + current_dps[i] = idps[x]; + } + ++i; + } + series_size = ordered_intersection.size(); + } + + /** + * Flattens the appropriate tags into a single byte array + * @param use_query_tags Whether or not to include tags returned with the + * results or just use those group by'd in the query + * @param include_agg_tags Whether or not to include the aggregated tags in + * the identifier + * @param tags The map of tags from the result set + * @param agg_tags The list of aggregated tags + * @param sub The sub query iterator + * @return A byte array with the flattened tag keys and values. Note that + * if the tags set is empty, this may return an empty array (but not a null + * array) + */ + static byte[] flattenTags(final boolean use_query_tags, + final boolean include_agg_tags, final ByteMap tags, + final ByteSet agg_tags, final ITimeSyncedIterator sub) { + if (tags.isEmpty()) { + return HBaseClient.EMPTY_ARRAY; + } + final ByteSet query_tagks; + // NOTE: We MAY need the agg tags but I'm not sure yet + final int tag_size; + if (use_query_tags) { + int i = 0; + if (sub.getQueryTagKs() != null && !sub.getQueryTagKs().isEmpty()) { + query_tagks = sub.getQueryTagKs(); + for (final Map.Entry pair : tags.entrySet()) { + if (query_tagks.contains(pair.getKey())) { + i++; + } + } + } else { + query_tagks = new ByteSet(); + } + tag_size = i; + } else { + query_tagks = new ByteSet(); + tag_size = tags.size(); + } + + int len = (tag_size * (TSDB.tagk_width() + TSDB.tagv_width())) + + (include_agg_tags ? (agg_tags.size() * TSDB.tagk_width()) : 0); + final byte[] tagks = new byte[len]; + int i = 0; + for (final Map.Entry pair : tags.entrySet()) { + if (use_query_tags && !query_tagks.contains(pair.getKey())) { + continue; + } + System.arraycopy(pair.getKey(), 0, tagks, i, TSDB.tagk_width()); + i += TSDB.tagk_width(); + System.arraycopy(pair.getValue(), 0, tagks, i, TSDB.tagv_width()); + i += TSDB.tagv_width(); + } + if (include_agg_tags) { + for (final byte[] tagk : agg_tags) { + System.arraycopy(tagk, 0, tagks, i, TSDB.tagk_width()); + i += TSDB.tagk_width(); + } + } + return tagks; + } + + @Override + public ExpressionDataPoint[] next(long timestamp) { + throw new NotImplementedException(); + } + + @Override + public int size() { + throw new NotImplementedException(); + } + + @Override + public ExpressionDataPoint[] values() { + throw new NotImplementedException(); + } + + @Override + public void nullIterator(int index) { + throw new NotImplementedException(); + } + + @Override + public int getIndex() { + return index; + } + + @Override + public void setIndex(int index) { + this.index = index; + } + + @Override + public String getId() { + return id; + } + + @Override + public ByteSet getQueryTagKs() { + throw new NotImplementedException(); + } + + @Override + public void setFillPolicy(NumericFillPolicy policy) { + throw new NotImplementedException(); + } + + @Override + public NumericFillPolicy getFillPolicy() { + throw new NotImplementedException(); + } + + @Override + public ITimeSyncedIterator getCopy() { + return new IntersectionIterator(this); + } + + @Override + public boolean hasNext(int index) { + for (final ITimeSyncedIterator sub : queries.values()) { + if (sub.hasNext(index)) { + return true; + } + } + return false; + } + + @Override + public void next(int index) { + if (!hasNext()) { + throw new IllegalDataException("No more data"); + } + for (final ITimeSyncedIterator sub : queries.values()) { + sub.next(index); + } + } + +} diff --git a/src/query/expression/MovingAverage.java b/src/query/expression/MovingAverage.java new file mode 100644 index 0000000000..5a86b90c7c --- /dev/null +++ b/src/query/expression/MovingAverage.java @@ -0,0 +1,345 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.expression; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import net.opentsdb.core.AggregationIterator; +import net.opentsdb.core.Aggregator; +import net.opentsdb.core.Aggregators; +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.IllegalDataException; +import net.opentsdb.core.MutableDataPoint; +import net.opentsdb.core.SeekableView; +import net.opentsdb.core.TSQuery; +import net.opentsdb.core.Aggregators.Interpolation; + +/** + * Implements a moving average function windowed on either the number of + * data points or a unit of time. + * @since 2.3 + */ +public class MovingAverage implements Expression { + + @Override + public DataPoints[] evaluate(final TSQuery data_query, + final List query_results, final List params) { + if (data_query == null) { + throw new IllegalArgumentException("Missing time series query"); + } + if (query_results == null || query_results.isEmpty()) { + return new DataPoints[]{}; + } + if (params == null || params.isEmpty()) { + throw new IllegalArgumentException("Missing moving average window size"); + } + + String param = params.get(0); + if (param == null || param.isEmpty()) { + throw new IllegalArgumentException("Missing moving average window size"); + } + param = param.trim(); + + long condition = -1; + boolean is_time_unit = false; + if (param.matches("^[0-9]+$")) { + try { + condition = Integer.parseInt(param); + } catch (NumberFormatException nfe) { + throw new IllegalArgumentException( + "Invalid parameter, must be an integer", nfe); + } + } else if (param.startsWith("'") && param.endsWith("'")) { + condition = parseParam(param); + is_time_unit = true; + } else { + throw new IllegalArgumentException("Unparseable window size: " + param); + } + if (condition <= 0) { + throw new IllegalArgumentException("Moving average window must be an " + + "integer greater than zero"); + } + + int num_results = 0; + for (final DataPoints[] results : query_results) { + num_results += results.length; + } + + final PostAggregatedDataPoints[] post_agg_results = + new PostAggregatedDataPoints[num_results]; + int ix = 0; + // one or more queries (m=...&m=...&m=...) + for (final DataPoints[] sub_query_result : query_results) { + // group bys (m=sum:foo{host=*}) + for (final DataPoints dps: sub_query_result) { + // TODO(cl) - Avoid iterating and copying if we can help it. We should + // be able to pass the original DataPoints object to the seekable view + // and then iterate through it. + final List mutable_points = new ArrayList(); + for (final DataPoint point: dps) { + // avoid flip-flopping between integers and floats, always use double + // for average. + mutable_points.add( + MutableDataPoint.ofDoubleValue(point.timestamp(), point.toDouble())); + } + + post_agg_results[ix++] = new PostAggregatedDataPoints(dps, + mutable_points.toArray(new DataPoint[mutable_points.size()])); + } + } + + final DataPoints[] results = new DataPoints[num_results]; + for (int i = 0; i < num_results; i++) { + final Aggregator moving_average = new MovingAverageAggregator( + Aggregators.Interpolation.LERP, "movingAverage", + condition, is_time_unit); + final SeekableView[] metrics_groups = new SeekableView[] { + post_agg_results[i].iterator() }; + final SeekableView view = new AggregationIterator(metrics_groups, + data_query.startTime(), data_query.endTime(), + moving_average, + Aggregators.Interpolation.LERP, false); + final List points = new ArrayList(); + while (view.hasNext()) { + final DataPoint mdp = view.next(); + points.add(MutableDataPoint.ofDoubleValue(mdp.timestamp(), mdp.toDouble())); + } + results[i] = new PostAggregatedDataPoints(post_agg_results[i], + points.toArray(new DataPoint[points.size()])); + } + return results; + } + + /** + * Parses the parameter string to fetch the window size + *

    + * Package private for UTs + * @param param The string to parse + * @return The window size (number of points or a unit of time in ms) + */ + long parseParam(final String param) { + if (param == null || param.isEmpty()) { + throw new IllegalArgumentException( + "Window parameter may not be null or empty"); + } + final char[] chars = param.toCharArray(); + int idx = 0; + for (int c = 1; c < chars.length; c++) { + if (Character.isDigit(chars[c])) { + idx++; + } else { + break; + } + } + if (idx < 1) { + throw new IllegalArgumentException("Invalid moving window parameter: " + + param); + } + + try { + final int time = Integer.parseInt(param.substring(1, idx + 1)); + final String unit = param.substring(idx + 1, param.length() - 1); + + // TODO(CL) - add a Graphite unit parser to DateTime for this kind of conversion + if ("day".equals(unit) || "d".equals(unit)) { + return TimeUnit.MILLISECONDS.convert(time, TimeUnit.DAYS); + } else if ("hr".equals(unit) || "hour".equals(unit) || "h".equals(unit)) { + return TimeUnit.MILLISECONDS.convert(time, TimeUnit.HOURS); + } else if ("min".equals(unit) || "m".equals(unit)) { + return TimeUnit.MILLISECONDS.convert(time, TimeUnit.MINUTES); + } else if ("sec".equals(unit) || "s".equals(unit)) { + return TimeUnit.MILLISECONDS.convert(time, TimeUnit.SECONDS); + } else { + throw new IllegalArgumentException("Unknown time unit=" + unit + + " in window=" + param); + } + } catch (NumberFormatException nfe) { + throw new IllegalArgumentException("Unable to parse moving window " + + "parameter: " + param, nfe); + } + } + + @Override + public String writeStringField(final List query_params, + final String inner_expression) { + return "movingAverage(" + inner_expression + ")"; + } + + /** + * An aggregator that expects a single data point for each iteration. The + * values are prepended to a linked list. Next it iterates over the list until + * it either runs out of values (and returns a 0 with the proper timestamp) or + * returns the average of all values in the given window (time or number based). + *

    + * Package private for unit testing + */ + static final class MovingAverageAggregator extends Aggregator { + /** The individual values in the window */ + private final LinkedList accumulation; + /** The condition to satisfy, either a time unit or # of data points */ + private final long condition; + /** Whether or not the condition is a time unit or the # of data points */ + private final boolean is_time_unit; + /** Sentinel used to kick out the first timed window value */ + private boolean window_started; + + /** + * Ctor for this implementation + * @param method The interpolation method to use (ignored) + * @param name The name of this aggregator + * @param condition The windowing condition + * @param is_time_unit Whether or not the condition is a time unit or + * the # of data points + */ + public MovingAverageAggregator(final Interpolation method, final String name, + final long condition, final boolean is_time_unit) { + super(method, name); + this.condition = condition; + this.is_time_unit = is_time_unit; + accumulation = new LinkedList(); + } + + @Override + public long runLong(final Longs values) { + final long value = values.nextLongValue(); + if (values.hasNextValue()) { + throw new IllegalDataException( + "There should only be one value in " + values); + } + final long ts = ((DataPoint) values).timestamp(); + accumulation.addFirst(MutableDataPoint.ofLongValue(ts, value)); + + // for timed windows we need to skip the first data point in the series + // as we have no idea what the previous value's timestamp was. + if (is_time_unit && !window_started) { + window_started = true; + return 0; + } + + long sum = 0; + int count = 0; + final Iterator iter = accumulation.iterator(); + boolean condition_met = false; + long time_window_cumulation = 0; // how many ms are in our window + long last_ts = -1; // the timestamp of the previous dp + + // now sum up the preceding points + while(iter.hasNext()) { + final DataPoint dp = iter.next(); + if (is_time_unit) { + if (last_ts < 0) { + last_ts = dp.timestamp(); + } else { + time_window_cumulation += last_ts - dp.timestamp(); + last_ts = dp.timestamp(); + if (time_window_cumulation >= condition) { + condition_met = true; + break; + } + } + } + // cast to long if we dumped a double in there + sum += dp.isInteger() ? dp.longValue() : dp.doubleValue(); + count++; + if (!is_time_unit && count >= condition) { + condition_met = true; + break; + } + } + while (iter.hasNext()) { + // should drop the last entry in the linked list to avoid accumulating + // everything in memory + iter.next(); + iter.remove(); + } + + if (!condition_met || count == 0) { + return 0; + } + return sum / count; + } + + @Override + public double runDouble(Doubles values) { + final double value = values.nextDoubleValue(); + if (values.hasNextValue()) { + throw new IllegalDataException( + "There should only be one value in " + values); + } + final long ts = ((DataPoint) values).timestamp(); + accumulation.addFirst(MutableDataPoint.ofDoubleValue(ts, value)); + + // for timed windows we need to skip the first data point in the series + // as we have no idea what the previous value's timestamp was. + if (is_time_unit && !window_started) { + window_started = true; + return 0; + } + + double sum = 0; + int count = 0; + final Iterator iter = accumulation.iterator(); + boolean condition_met = false; + long time_window_cumulation = 0; // how many ms are in our window + long last_ts = -1; // the timestamp of the previous dp + + // now sum up the preceding points + while(iter.hasNext()) { + final DataPoint dp = iter.next(); + + if (is_time_unit) { + if (last_ts < 0) { + last_ts = dp.timestamp(); + } else { + time_window_cumulation += last_ts - dp.timestamp(); + last_ts = dp.timestamp(); + if (time_window_cumulation >= condition) { + condition_met = true; + break; + } + } + } + + // cast to double if we dumped a long in there + final double v = dp.isInteger() ? dp.longValue() : dp.doubleValue(); + if (!Double.isNaN(v)) { + // skip NaNs to avoid NaNing everything in the window. + sum += v; + count++; + } + + if (!is_time_unit && count >= condition) { + condition_met = true; + break; + } + } + + while (iter.hasNext()) { + // should drop the last entry in the linked list to avoid accumulating + // everything in memory + iter.next(); + iter.remove(); + } + + if (!condition_met || count == 0) { + return 0; + } + return sum/count; + } + } +} diff --git a/src/query/expression/MultiplySeries.java b/src/query/expression/MultiplySeries.java new file mode 100644 index 0000000000..34cbb251d8 --- /dev/null +++ b/src/query/expression/MultiplySeries.java @@ -0,0 +1,86 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.expression; + +import java.util.List; + +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.TSDB; +import net.opentsdb.core.TSQuery; +import net.opentsdb.query.expression.VariableIterator.SetOperator; + +/** + * Performs a UNION set join on up to 26 metric query results and returns the + * product. + */ +public class MultiplySeries implements Expression { + /** The TSDB used for UID to name lookups */ + final TSDB tsdb; + + /** + * Default ctor. + * @param tsdb The TSDB used for UID to name lookups + */ + public MultiplySeries(final TSDB tsdb) { + this.tsdb = tsdb; + } + + @Override + public DataPoints[] evaluate(final TSQuery data_query, + final List query_results, final List params) { + if (data_query == null) { + throw new IllegalArgumentException("Missing time series query"); + } + if (query_results == null || query_results.isEmpty()) { + return new DataPoints[]{}; + } + + if (query_results.size() < 2 || query_results.size() > 26) { + throw new IllegalArgumentException("Must have 2 to 26 series, got " + + query_results.size() + " instead"); + } + + final StringBuilder buf = new StringBuilder(); + char v = 'a'; + for (int i = 0; i < query_results.size(); i++) { + buf.append(v++); + if (i < query_results.size() - 1) { + buf.append(" * "); + } + } + + final ExpressionIterator expression = new ExpressionIterator("multiplySeries", + buf.toString(), SetOperator.UNION, false, false); + v = 'a'; + + for (final DataPoints[] dps : query_results) { + final TimeSyncedIterator it = new TimeSyncedIterator( + Character.toString(v++), null, dps); + expression.addResults(it.getId(), it); + } + expression.compile(); + + final DataPoints[] results = new DataPoints[expression.values().length]; + for (int i = 0; i < expression.values().length; i++) { + results[i] = new EDPtoDPS(tsdb, i, expression); + } + return results; + } + + @Override + public String writeStringField(final List query_params, + final String inner_expression) { + return "multiplySeries(" + inner_expression + ")"; + } + +} diff --git a/src/query/expression/NumericFillPolicy.java b/src/query/expression/NumericFillPolicy.java new file mode 100644 index 0000000000..1ea9fa9ba6 --- /dev/null +++ b/src/query/expression/NumericFillPolicy.java @@ -0,0 +1,176 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.expression; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; +import com.google.common.base.Objects; + +import net.opentsdb.core.FillPolicy; + +/** + * POJO for serdes of fill policies. It allows the user to pick either policies + * with default values or a scalar that can be supplied with any number. + * @since 2.3 + */ +@JsonDeserialize(builder = NumericFillPolicy.Builder.class) +public class NumericFillPolicy { + + /** The fill policy to use. This is required */ + private FillPolicy policy; + + /** The value to store with the fill policy */ + private double value; + + /** + * CTor to set the policy. Also calls {@link #validate()} + * @param policy The policy to set. + */ + public NumericFillPolicy(final FillPolicy policy) { + this.policy = policy; + validate(); + } + + /** + * CTor to set the policy and value. Also calls {@link #validate()} + * @param policy The name of the fill policy + * @param value The value to use when filling + * @throws IllegalArgumentException if the policy and value don't gel together + */ + public NumericFillPolicy(final FillPolicy policy, final double value) { + this.policy = policy; + this.value = value; + validate(); + } + + @Override + public String toString() { + return "policy=" + policy + ", value=" + value; + } + + /** @return a NumericFillPolicy builder */ + public static Builder Builder() { + return new Builder(); + } + + /** + * A builder class for deserialization via Jackson + */ + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonPOJOBuilder(buildMethodName = "build", withPrefix = "") + public static final class Builder { + @JsonProperty + private FillPolicy policy; + @JsonProperty + private double value; + + public Builder setPolicy(FillPolicy policy) { + this.policy = policy; + return this; + } + + public Builder setValue(double value) { + this.value = value; + return this; + } + + public NumericFillPolicy build() { + return new NumericFillPolicy(policy, value); + } + } + + @Override + public int hashCode() { + return Objects.hashCode(policy, value); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj == this) { + return true; + } + if (!(obj instanceof NumericFillPolicy)) { + return false; + } + final NumericFillPolicy nfp = (NumericFillPolicy)obj; + return Objects.equal(policy, nfp.policy) && + Objects.equal(value, nfp.value); + } + + /** @return the fill policy */ + public FillPolicy getPolicy() { + return policy; + } + + /** @param policy the fill policy to use */ + public void setPolicy(final FillPolicy policy) { + this.policy = policy; + } + + /** @return the value to use when filling */ + public double getValue() { + return value; + } + + /** @param value the value to use when filling */ + public void setValue(final double value) { + this.value = value; + } + + /** + * Makes sure the policy name and value are a suitable combination. If one + * or the other is missing then we set the other with the proper value. + * @throws IllegalArgumentException if the combination is bad + */ + public void validate() { + if (policy == null) { + if (value == 0) { + policy = FillPolicy.ZERO; + } else if (Double.isNaN(value)) { + policy = FillPolicy.NOT_A_NUMBER; + } else { + policy = FillPolicy.SCALAR; + } + } else { + switch (policy) { + case NONE: + case NOT_A_NUMBER: + if (value != 0 && !Double.isNaN(value)) { + throw new IllegalArgumentException( + "The value for NONE and NAN must be NaN"); + } + value = Double.NaN; + break; + case ZERO: + if (value != 0) { + throw new IllegalArgumentException("The value for ZERO must be 0"); + } + value = 0; + break; + case NULL: + if (value != 0 && !Double.isNaN(value)) { + throw new IllegalArgumentException("The value for NULL must be 0"); + } + value = Double.NaN; + break; + case SCALAR: // it CAN be zero + break; + } + } + } +} diff --git a/src/query/expression/PostAggregatedDataPoints.java b/src/query/expression/PostAggregatedDataPoints.java new file mode 100644 index 0000000000..c54a1c2c00 --- /dev/null +++ b/src/query/expression/PostAggregatedDataPoints.java @@ -0,0 +1,262 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.expression; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.NoSuchElementException; + +import org.hbase.async.Bytes.ByteMap; + +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.SeekableView; +import net.opentsdb.meta.Annotation; + +import com.stumbleupon.async.Callback; +import com.stumbleupon.async.Deferred; + +/** + * A class to store an array of data points processed through expressions along + * with the original meta data of the result set (metric, tags, etc). + * @since 2.3 + */ +public class PostAggregatedDataPoints implements DataPoints { + + /** The original results from storage, used for fetching meta data */ + private final DataPoints base_data_points; + + /** The results of the expression calculation */ + private final DataPoint[] points; + + /** An optional alias for the results */ + private String alias = null; + + /** + * Default ctor + * @param base_data_points The original results from storage for fetching meta + * @param points The results of the expression calculation + */ + public PostAggregatedDataPoints(final DataPoints base_data_points, + final DataPoint[] points) { + if (base_data_points == null) { + throw new IllegalArgumentException("base_data_points cannot be null"); + } + if (points == null) { + throw new IllegalArgumentException("points cannot be null"); + } + this.base_data_points = base_data_points; + this.points = points; + } + + @Override + public String metricName() { + try { + return metricNameAsync().join(); + } catch (Exception e) { + throw new RuntimeException("Unexpected exception waiting for " + + "name resolution", e); + } + } + + @Override + public Deferred metricNameAsync() { + if (alias != null) { + if (alias.contains("@") && getTagUids().size() > 0) { + // need to resolve the tag UIDs for the templating feature + + class TemplateFill implements Callback, + Map> { + @Override + public Deferred call(final Map tags) + throws Exception { + for (final Entry pair : tags.entrySet()) { + alias = alias.replace("@" + pair.getKey(), pair.getValue()); + } + return Deferred.fromResult(alias); + } + } + return base_data_points.getTagsAsync() + .addCallbackDeferring(new TemplateFill()); + } + return Deferred.fromResult(alias); + } + return base_data_points.metricNameAsync(); + } + + @Override + public byte[] metricUID() { + if (alias != null) { + return new byte[] { }; + } + return base_data_points.metricUID(); + } + + @Override + public Map getTags() { + if (alias != null) { + return Collections.emptyMap(); + } else { + return base_data_points.getTags(); + } + } + + @Override + public Deferred> getTagsAsync() { + if (alias != null) { + return Deferred.fromResult(Collections.emptyMap()); + } + return base_data_points.getTagsAsync(); + } + + @Override + public List getAggregatedTags() { + if (alias != null) { + return Collections.emptyList(); + } + return base_data_points.getAggregatedTags(); + } + + @Override + public Deferred> getAggregatedTagsAsync() { + if (alias != null) { + return Deferred.fromResult(Collections.emptyList()); + } + return base_data_points.getAggregatedTagsAsync(); + } + + @Override + public List getAggregatedTagUids() { + if (alias != null) { + return Collections.emptyList(); + } + return base_data_points.getAggregatedTagUids(); + } + + @Override + public List getTSUIDs() { + return base_data_points.getTSUIDs(); + } + + @Override + public List getAnnotations() { + return base_data_points.getAnnotations(); + } + + @Override + public ByteMap getTagUids() { + return base_data_points.getTagUids(); + } + + @Override + public int getQueryIndex() { + return base_data_points.getQueryIndex(); + } + + @Override + public int size() { + return points.length; + } + + @Override + public int aggregatedSize() { + return points.length; + } + + @Override + public SeekableView iterator() { + return new SeekableViewImpl(points); + } + + @Override + public long timestamp(int i) { + return points[i].timestamp(); + } + + @Override + public boolean isInteger(int i) { + return points[i].isInteger(); + } + + @Override + public long longValue(int i) { + return points[i].longValue(); + } + + @Override + public double doubleValue(int i) { + return points[i].doubleValue(); + } + + @Override + public boolean isPercentile() { + return false; + } + + @Override + public float getPercentile() { + throw new UnsupportedOperationException("getPercentile not supported"); + } + + /** + * An iterator working over the data points resulting from the expression + * calculation. + */ + static class SeekableViewImpl implements SeekableView { + + private int pos = 0; + private final DataPoint[] dps; + + SeekableViewImpl(final DataPoint[] dps) { + this.dps = dps; + } + + @Override + public boolean hasNext() { + return pos < dps.length; + } + + @Override + public DataPoint next() { + if (hasNext()) { + return dps[pos++]; + } else { + throw new NoSuchElementException("no more elements"); + } + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + @Override + public void seek(long timestamp) { + for (int i = pos; i < dps.length; i++) { + if (dps[i].timestamp() >= timestamp) { + break; + } else { + pos++; + } + } + } + } + + /** @param alias The alias to set for the time series. Used in place of + * the metric and nulls out all tags. */ + public void setAlias(String alias) { + this.alias = alias; + } +} diff --git a/src/query/expression/Scale.java b/src/query/expression/Scale.java new file mode 100644 index 0000000000..2a7bf3649f --- /dev/null +++ b/src/query/expression/Scale.java @@ -0,0 +1,110 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.expression; + +import java.util.ArrayList; +import java.util.List; + +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.MutableDataPoint; +import net.opentsdb.core.SeekableView; +import net.opentsdb.core.TSQuery; + +/** + * Multiplies each data point in the series by the given factor. + * @since 2.3 + */ +public class Scale implements Expression { + + @Override + public DataPoints[] evaluate(final TSQuery data_query, + final List query_results, final List params) { + if (data_query == null) { + throw new IllegalArgumentException("Missing time series query"); + } + if (query_results == null || query_results.isEmpty()) { + return new DataPoints[]{}; + } + if (params == null || params.isEmpty()) { + throw new IllegalArgumentException("Missing scaling factor"); + } + + double scale_factor = 0; // zero is fine, if useless *shrug* + final String factor = params.get(0); + if (factor != null && factor.matches("^[-0-9\\.]+$")) { + try { + scale_factor = Double.parseDouble(factor); + } catch (NumberFormatException nfe) { + throw new IllegalArgumentException( + "Invalid parameter, must be an integer or floating point", nfe); + } + } else { + throw new IllegalArgumentException("Unparseable scale factor value: " + + scale_factor); + } + + int num_results = 0; + for (DataPoints[] results: query_results) { + num_results += results.length; + } + + final DataPoints[] results = new DataPoints[num_results]; + int ix = 0; + // one or more sub queries (m=...&m=...&m=...) + for (final DataPoints[] sub_query_result : query_results) { + // group bys (m=sum:foo{host=*}) + for (final DataPoints dps : sub_query_result) { + results[ix++] = scale(dps, scale_factor); + } + } + return results; + } + + /** + * Multiplies each data point in the series by the scale factor, maintaining + * integers if both the data point and scale are integers. + * @param points The data points to factor + * @param scale_factor The factor to multiply by + * @return The resulting data points + */ + private DataPoints scale(final DataPoints points, final double scale_factor) { + // TODO(cl) - Using an array as the size function may not return the exact + // results and we should figure a way to avoid copying data anyway. + final List dps = new ArrayList(); + final boolean scale_is_int = (scale_factor == Math.floor(scale_factor)) && + !Double.isInfinite(scale_factor); + final SeekableView view = points.iterator(); + while (view.hasNext()) { + DataPoint pt = view.next(); + if (pt.isInteger() && scale_is_int) { + dps.add(MutableDataPoint.ofLongValue(pt.timestamp(), + (long)scale_factor * pt.longValue())); + } else { + // NaNs are fine here, they'll just be re-computed as NaN + dps.add(MutableDataPoint.ofDoubleValue(pt.timestamp(), + scale_factor * pt.toDouble())); + } + } + final DataPoint[] results = new DataPoint[dps.size()]; + dps.toArray(results); + return new PostAggregatedDataPoints(points, results); + } + + @Override + public String writeStringField(final List query_params, + final String inner_expression) { + return "scale(" + inner_expression + ")"; + } + +} diff --git a/src/query/expression/SumSeries.java b/src/query/expression/SumSeries.java new file mode 100644 index 0000000000..3241901bcc --- /dev/null +++ b/src/query/expression/SumSeries.java @@ -0,0 +1,85 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.expression; + +import java.util.List; + +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.TSDB; +import net.opentsdb.core.TSQuery; +import net.opentsdb.query.expression.VariableIterator.SetOperator; + +/** + * Performs a UNION set join on x metric query results and returns the results. + */ +public class SumSeries implements Expression { + /** The TSDB used for UID to name lookups */ + final TSDB tsdb; + + /** + * Default ctor. + * @param tsdb The TSDB used for UID to name lookups + */ + public SumSeries(final TSDB tsdb) { + this.tsdb = tsdb; + } + + @Override + public DataPoints[] evaluate(final TSQuery data_query, + final List query_results, final List params) { + if (data_query == null) { + throw new IllegalArgumentException("Missing time series query"); + } + if (query_results == null || query_results.isEmpty()) { + return new DataPoints[]{}; + } + + if (query_results.size() < 2 || query_results.size() > 26) { + throw new IllegalArgumentException("Must have 2 to 26 series, got " + + query_results.size() + " instead"); + } + + final StringBuilder buf = new StringBuilder(); + char v = 'a'; + for (int i = 0; i < query_results.size(); i++) { + buf.append(v++); + if (i < query_results.size() - 1) { + buf.append(" + "); + } + } + System.out.println("Expression: [" + buf.toString() + "]"); + final ExpressionIterator expression = new ExpressionIterator("sumSeries", + buf.toString(), SetOperator.UNION, false, false); + v = 'a'; + + for (final DataPoints[] dps : query_results) { + final TimeSyncedIterator it = new TimeSyncedIterator( + Character.toString(v++), null, dps); + expression.addResults(it.getId(), it); + } + expression.compile(); + + final DataPoints[] results = new DataPoints[expression.values().length]; + for (int i = 0; i < expression.values().length; i++) { + results[i] = new EDPtoDPS(tsdb, i, expression); + } + return results; + } + + @Override + public String writeStringField(final List query_params, + final String inner_expression) { + return "sumSeries(" + inner_expression + ")"; + } + +} diff --git a/src/query/expression/TimeShift.java b/src/query/expression/TimeShift.java new file mode 100644 index 0000000000..c157f0c5d3 --- /dev/null +++ b/src/query/expression/TimeShift.java @@ -0,0 +1,144 @@ +package net.opentsdb.query.expression; +/** + * Copyright 2015 The opentsdb Authors + *

    + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import net.opentsdb.core.*; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class TimeShift implements Expression { + /** + * in place modify of TsdbResult array to increase timestamps by timeshift + * + * @param data_query + * @param results + * @param params + * @return + */ + @Override + public DataPoints[] evaluate(TSQuery data_query, List results, List params) { + //not 100% sure what to do here -> do I need to think of the case where I have no data points + if (results == null || results.isEmpty()) { + return new DataPoints[]{}; + } + if (params == null || params.isEmpty()) { + throw new IllegalArgumentException("Need amount of timeshift to perform timeshift"); + } + + String param = params.get(0); + if (param == null || param.length() == 0) { + throw new IllegalArgumentException("Invalid timeshift='" + param + "'"); + } + + param = param.trim(); + + long timeshift = -1; + if (param.startsWith("'") && param.endsWith("'")) { + timeshift = parseParam(param); + } else { + throw new RuntimeException("Invalid timeshift parameter: eg '10min'"); + } + + if (timeshift <= 0) { + throw new RuntimeException("timeshift <= 0"); + } + + return performShift(results.get(0), timeshift); + } + + private static Boolean timeshiftIsInt(final long timeshift) { + return (timeshift == Math.floor(timeshift)) && + !Double.isInfinite(timeshift); + } + + DataPoints[] performShift(DataPoints[] inputPoints, long timeshift) { + DataPoints[] outputPoints = new DataPoints[inputPoints.length]; + for (int n = 0; n < inputPoints.length; n++) { + outputPoints[n] = shift(inputPoints[n], timeshift); + } + return outputPoints; + } + + long parseParam(String param) { + char[] chars = param.toCharArray(); + int tuIndex = 0; + for (int c = 1; c < chars.length; c++) { + if (Character.isDigit(chars[c])) { + tuIndex++; + } else { + break; + } + } + + if (tuIndex == 0) { + throw new RuntimeException("Invalid Parameter: " + param); + } + + int time = Integer.parseInt(param.substring(1, tuIndex + 1)); + String unit = param.substring(tuIndex + 1, param.length()).trim(); + if ("sec".equals(unit)) { + return TimeUnit.MILLISECONDS.convert(time, TimeUnit.SECONDS); + } else if ("min".equals(unit)) { + return TimeUnit.MILLISECONDS.convert(time, TimeUnit.MINUTES); + } else if ("hr".equals(unit)) { + return TimeUnit.MILLISECONDS.convert(time, TimeUnit.HOURS); + } else if ("day".equals(unit) || "days".equals(unit)) { + return TimeUnit.MILLISECONDS.convert(time, TimeUnit.DAYS); + } else if ("week".equals(unit) || "weeks".equals(unit)) { + //didn't have week so small cheat here + return TimeUnit.MILLISECONDS.convert(time * 7, TimeUnit.DAYS); + } else { + throw new RuntimeException("unknown time unit=" + unit); + } + } + + /** + * Adjusts the timestamp of each datapoint by timeshift + * + * @param points The data points to factor + * @param timeshift The factor to multiply by + * @return The resulting data points + */ + DataPoints shift(final DataPoints points, final long timeshift) { + // TODO(cl) - Using an array as the size function may not return the exact + // results and we should figure a way to avoid copying data anyway. + final List dps = new ArrayList(); + + for (DataPoint pt : points) { + dps.add(shift(pt, timeshift)); + } + + final DataPoint[] results = new DataPoint[dps.size()]; + dps.toArray(results); + return new PostAggregatedDataPoints(points, results); + } + + DataPoint shift(final DataPoint pt, final long timeshift) { + if (timeshiftIsInt(timeshift)) { + return MutableDataPoint.ofLongValue(pt.timestamp() + timeshift, pt.longValue()); + } else { + // NaNs are fine here, they'll just be re-computed as NaN + return MutableDataPoint.ofDoubleValue(pt.timestamp() + timeshift, timeshift * pt.toDouble()); + } + } + + @Override + public String writeStringField(List params, String inner_expression) { + return "timeshift(" + inner_expression + ")"; + } +} diff --git a/src/query/expression/TimeSyncedIterator.java b/src/query/expression/TimeSyncedIterator.java new file mode 100644 index 0000000000..451e7b96c2 --- /dev/null +++ b/src/query/expression/TimeSyncedIterator.java @@ -0,0 +1,248 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.expression; + +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.FillPolicy; +import net.opentsdb.core.SeekableView; +import net.opentsdb.utils.ByteSet; + +/** + * Holds the results of a sub query (a single metric) and iterates over each + * resultant series in lock-step for expression evaluation. + * @since 2.3 + */ +public class TimeSyncedIterator implements ITimeSyncedIterator { + + /** The name of this sub query given by the user */ + private final String id; + + /** The set of tag keys issued with the query */ + private final ByteSet query_tagks; + + /** The data point interfaces fetched from storage */ + private final DataPoints[] dps; + + /** The current value used for iterating */ + private final DataPoint[] current_values; + + /** References to the MutableDataObjects the ExpressionIterator will read */ + private final ExpressionDataPoint[] emitter_values; + + /** A list of the iterators used for fetching the next value */ + private final SeekableView[] iterators; + + /** Set by the ExpressionIterator when it computes the intersection */ + private int index; + + /** A policy to use for emitting values when a timestamp is missing data */ + private NumericFillPolicy fill_policy; + + /** + * Instantiates an iterator based on the results of a TSSubQuery. + * This will setup the emitters so it's safe to call {@link #values()} + * @param id The name of the query. + * @param query_tagks The set of tags used in filters on the query. + * @param dps The data points fetched from storage. + * @throws IllegalArgumentException if one of the parameters is null or the ID + * is empty + */ + public TimeSyncedIterator(final String id, final ByteSet query_tagks, + final DataPoints[] dps) { + if (id == null || id.isEmpty()) { + throw new IllegalArgumentException("Missing ID string"); + } + if (dps == null) { + // it's ok for these to be empty, but they canna be null ya ken? + throw new IllegalArgumentException("Missing data points"); + } + this.id = id; + this.query_tagks = query_tagks; + this.dps = dps; + // TODO - load from a default or something + fill_policy = new NumericFillPolicy(FillPolicy.ZERO); + current_values = new DataPoint[dps.length]; + emitter_values = new ExpressionDataPoint[dps.length]; + iterators = new SeekableView[dps.length]; + setupEmitters(); + } + + /** + * A copy constructor that loads from an existing iterator. + * @param iterator The iterator to load from + */ + private TimeSyncedIterator(final TimeSyncedIterator iterator) { + id = iterator.id; + query_tagks = iterator.query_tagks; // sharing is ok here + dps = iterator.dps; // TODO ?? OK? + fill_policy = iterator.fill_policy; + current_values = new DataPoint[dps.length]; + emitter_values = new ExpressionDataPoint[dps.length]; + iterators = new SeekableView[dps.length]; + setupEmitters(); + } + + @Override + public String toString() { + final StringBuilder buf = new StringBuilder(); + buf.append("TimeSyncedIterator(id=") + .append(id) + .append(", index=") + .append(index) + .append(", dpsSize=") + .append(dps.length) + .append(")"); + return buf.toString(); + } + + @Override + public int size() { + return dps.length; + } + + @Override + public boolean hasNext() { + for (final DataPoint dp : current_values) { + if (dp != null) { + return true; + } + } + return false; + } + + @Override + public ExpressionDataPoint[] next(final long timestamp) { + for (int i = 0; i < current_values.length; i++) { + if (current_values[i] == null) { + emitter_values[i].reset(timestamp, fill_policy.getValue()); + continue; + } + + if (current_values[i].timestamp() > timestamp) { + emitter_values[i].reset(timestamp, fill_policy.getValue()); + } else { + emitter_values[i].reset(current_values[i]); + if (!iterators[i].hasNext()) { + current_values[i] = null; + } else { + current_values[i] = iterators[i].next(); + } + } + } + return emitter_values; + } + + @Override + public long nextTimestamp() { + long ts = Long.MAX_VALUE; + for (final DataPoint dp : current_values) { + if (dp != null) { + long t = dp.timestamp(); + if (t < ts) { + ts = t; + } + } + } + return ts; + } + + @Override + public void next(final int i) { + if (current_values[i] == null) { + throw new RuntimeException("No more elements"); + } + emitter_values[i].reset(current_values[i]); + if (iterators[i].hasNext()) { + current_values[i] = iterators[i].next(); + } else { + current_values[i] = null; + } + } + + @Override + public boolean hasNext(final int i) { + return current_values[i] != null; + } + + @Override + public int getIndex() { + return index; + } + + @Override + public void setIndex(final int index) { + this.index = index; + } + + @Override + public String getId() { + return id; + } + + /** @return the set of data points */ + public DataPoints[] getDataPoints() { + return dps; + } + + @Override + public void nullIterator(final int index) { + if (index < 0 || index > current_values.length) { + throw new IllegalArgumentException("Index out of range: " + index); + } + current_values[index] = null; + } + + @Override + public ExpressionDataPoint[] values() { + return emitter_values; + } + + @Override + public ByteSet getQueryTagKs() { + return query_tagks; + } + + @Override + public void setFillPolicy(final NumericFillPolicy policy) { + fill_policy = policy; + } + + @Override + public NumericFillPolicy getFillPolicy() { + return fill_policy; + } + + @Override + public ITimeSyncedIterator getCopy() { + return new TimeSyncedIterator(this); + } + + /** + * Iterates over the values and sets up the current and emitter values + */ + private void setupEmitters() { + // set the iterators + for (int i = 0; i < dps.length; i++) { + iterators[i] = dps[i].iterator(); + if (!iterators[i].hasNext()) { + current_values[i] = null; + emitter_values[i] = null; + } else { + current_values[i] = iterators[i].next(); + emitter_values[i] = new ExpressionDataPoint(dps[i]); + emitter_values[i].setIndex(i); + } + } + } +} diff --git a/src/query/expression/UnionIterator.java b/src/query/expression/UnionIterator.java new file mode 100644 index 0000000000..54cd70869a --- /dev/null +++ b/src/query/expression/UnionIterator.java @@ -0,0 +1,455 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.expression; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; + +import net.opentsdb.core.FillPolicy; +import net.opentsdb.core.IllegalDataException; +import net.opentsdb.core.TSDB; +import net.opentsdb.utils.ByteSet; + +import org.hbase.async.HBaseClient; +import org.hbase.async.Bytes.ByteMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import sun.reflect.generics.reflectiveObjects.NotImplementedException; + +/** + * An iterator that computes the union of all series in the result sets. This + * means we match every series with it's corresponding series in the other sets. + * If one or more set lacks the matching series, then a {@code null} is stored + * and when the caller iterates over the results, the need to detect the null + * and substitute a fill value. + * @since 2.3 + */ +public class UnionIterator implements ITimeSyncedIterator, VariableIterator { + private static final Logger LOG = LoggerFactory.getLogger(UnionIterator.class); + + /** The queries compiled and fetched from storage */ + private final Map queries; + + /** A list of the current values for each series post intersection */ + private final Map current_values; + + /** A map used for single series iteration where the array is the index */ + private final Map single_series_matrix; + + /** A map of the sub query index to their names for intersection computation */ + private final String[] index_to_names; + + /** Whether or not to intersect on the query tagks instead of the result set + * tagks */ + private final boolean union_on_query_tagks; + + /** Whether or not to include the aggregated tags in the result set */ + private final boolean include_agg_tags; + + /** The start/current timestamp for the iterator in ms */ + private long timestamp; + + /** Post intersection number of time series */ + private int series_size; + + /** The ID of this iterator */ + private final String id; + + /** The index of this iterator in a list of iterators */ + private int index; + + /** The fill policy to use when a series is missing from one of the sets. + * Default is zero. */ + private NumericFillPolicy fill_policy; + + /** A data point used for filling missing time series */ + private ExpressionDataPoint fill_dp; + + /** + * Default ctor + * @param id The variable ID for this iterator + * @param results Upstream iterators + * @param union_on_query_tagks Whether or not to flatten and join on only + * the tags from the query or those returned in the results. + * @param include_agg_tags Whether or not to include the flattened aggregated + * tag keys in the join. + */ + public UnionIterator(final String id, final Map results, + final boolean union_on_query_tagks, final boolean include_agg_tags) { + this.id = id; + this.union_on_query_tagks = union_on_query_tagks; + this.include_agg_tags = include_agg_tags; + timestamp = Long.MAX_VALUE; + queries = new HashMap(results.size()); + current_values = new HashMap(results.size()); + single_series_matrix = new HashMap(results.size()); + index_to_names = new String[results.size()]; + fill_policy = new NumericFillPolicy(FillPolicy.ZERO); + fill_dp = new ExpressionDataPoint(); + + int i = 0; + for (final Map.Entry entry : results.entrySet()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Adding iterator " + entry.getValue()); + } + queries.put(entry.getKey(), entry.getValue()); + entry.getValue().setIndex(i); + index_to_names[i] = entry.getKey(); + ++i; + } + + computeUnion(); + + // calculate the starting timestamp from the various iterators + for (final ITimeSyncedIterator it : queries.values()) { + final long ts = it.nextTimestamp(); + if (ts < timestamp) { + timestamp = ts; + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Computed union: " + this); + } + } + + /** + * Private copy constructor that copies references and sets up new collections + * without copying results. + * @param iterator The iterator to copy from. + */ + private UnionIterator(final UnionIterator iterator) { + id = iterator.id; + union_on_query_tagks = iterator.union_on_query_tagks; + include_agg_tags = iterator.include_agg_tags; + timestamp = Long.MAX_VALUE; + queries = new HashMap(iterator.queries.size()); + current_values = new HashMap(queries.size()); + single_series_matrix = new HashMap(queries.size()); + index_to_names = new String[queries.size()]; + fill_policy = iterator.fill_policy; + + int i = 0; + for (final Map.Entry entry : iterator.queries.entrySet()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Adding iterator " + entry.getValue()); + } + queries.put(entry.getKey(), entry.getValue()); + entry.getValue().setIndex(i); + index_to_names[i] = entry.getKey(); + ++i; + } + + computeUnion(); + + // calculate the starting timestamp from the various iterators + for (final ITimeSyncedIterator it : queries.values()) { + final long ts = it.nextTimestamp(); + if (ts < timestamp) { + timestamp = ts; + } + } + } + + /** + * Computes the union of all sets, matching on tags and optionally the + * aggregated tags across each variable. + */ + private void computeUnion() { + // key = flattened tags, array of queries.size() + final ByteMap ordered_union = + new ByteMap(); + + final Iterator it = queries.values().iterator(); + while (it.hasNext()) { + final ITimeSyncedIterator sub = it.next(); + final ExpressionDataPoint[] dps = sub.values(); + final ByteMap local_tags = new ByteMap(); + + for (int i = 0; i < sub.size(); i++) { + final byte[] key = flattenTags(union_on_query_tagks, include_agg_tags, + dps[i], sub); + local_tags.put(key, i); + ExpressionDataPoint[] udps = ordered_union.get(key); + if (udps == null) { + udps = new ExpressionDataPoint[queries.size()]; + ordered_union.put(key, udps); + } + udps[sub.getIndex()] = dps[i]; + } + } + + if (ordered_union.size() < 1) { + // if no data, just stop here + return; + } + + setCurrentAndMeta(ordered_union); + } + + /** + * Takes the resulting union and builds the {@link #current_values} + * and {@link #meta} maps. + * @param ordered_union The union to build from. + */ + private void setCurrentAndMeta(final ByteMap + ordered_union) { + for (final String id : queries.keySet()) { + current_values.put(id, new ExpressionDataPoint[ordered_union.size()]); + // TODO - blech. Fill with a sentinel value to reflect "no data here!" + final int[] m = new int[ordered_union.size()]; + for (int i = 0; i < m.length; i++) { + m[i] = -1; + } + single_series_matrix.put(id, m); + } + + int i = 0; + for (final Entry entry : ordered_union.entrySet()) { + final ExpressionDataPoint[] idps = entry.getValue(); + for (int x = 0; x < idps.length; x++) { + final ExpressionDataPoint[] current_dps = + current_values.get(index_to_names[x]); + current_dps[i] = idps[x]; + final int[] m = single_series_matrix.get(index_to_names[x]); + if (idps[x] != null) { + m[i] = idps[x].getIndex(); + } + } + ++i; + } + + // set fills on nulls + for (final ExpressionDataPoint[] idps : current_values.values()) { + for (i = 0; i < idps.length; i++) { + if (idps[i] == null) { + idps[i] = fill_dp; + } + } + } + series_size = ordered_union.size(); + } + + /** + * Creates a key based on the concatenation of the tag pairs then the agg + * tag keys. + * @param use_query_tags Whether or not to include tags returned with the + * results or just use those group by'd in the query + * @param include_agg_tags Whether or not to include the aggregated tags in + * the identifier + * @param dp The current expression data point + * @param sub The sub query iterator + * @return A byte array with the flattened tag keys and values. Note that + * if the tags set is empty, this may return an empty array (but not a null + * array) + */ + static byte[] flattenTags(final boolean use_query_tags, + final boolean include_agg_tags, final ExpressionDataPoint dp, + final ITimeSyncedIterator sub) { + if (dp.tags() == null || dp.tags().isEmpty()) { + return HBaseClient.EMPTY_ARRAY; + } + final int tagk_width = TSDB.tagk_width(); + final int tagv_width = TSDB.tagv_width(); + + final ByteSet query_tagks; + // NOTE: We MAY need the agg tags but I'm not sure yet + final int tag_size; + if (use_query_tags) { + int i = 0; + if (sub.getQueryTagKs() != null && !sub.getQueryTagKs().isEmpty()) { + query_tagks = sub.getQueryTagKs(); + for (final Map.Entry pair : dp.tags().entrySet()) { + if (query_tagks.contains(pair.getKey())) { + i++; + } + } + } else { + query_tagks = new ByteSet(); + } + tag_size = i; + } else { + query_tagks = new ByteSet(); + tag_size = dp.tags().size(); + } + + final int length = (tag_size * (tagk_width + tagv_width)) + + (include_agg_tags ? (dp.aggregatedTags().size() * tagk_width) : 0); + final byte[] key = new byte[length]; + int idx = 0; + for (final Entry pair : dp.tags().entrySet()) { + if (use_query_tags && !query_tagks.contains(pair.getKey())) { + continue; + } + System.arraycopy(pair.getKey(), 0, key, idx, tagk_width); + idx += tagk_width; + System.arraycopy(pair.getValue(), 0, key, idx, tagv_width); + idx += tagv_width; + } + if (include_agg_tags) { + for (final byte[] tagk : dp.aggregatedTags()) { + System.arraycopy(tagk, 0, key, idx, tagk_width); + idx += tagk_width; + } + } + return key; + } + + @Override + public String toString() { + final StringBuilder buf = new StringBuilder(); + buf.append("UnionIterator(id=") + .append(id) + .append(", useQueryTags=") + .append(union_on_query_tagks) + .append(", includeAggTags=") + .append(include_agg_tags) + .append(", index=") + .append(index) + .append(", queries=") + .append(queries); + return buf.toString(); + } + + // Iterator implementations + + @Override + public boolean hasNext() { + for (final ITimeSyncedIterator sub : queries.values()) { + if (sub.hasNext()) { + return true; + } + } + return false; + } + + @Override + public ExpressionDataPoint[] next(long timestamp) { + throw new NotImplementedException(); + } + + @Override + public long nextTimestamp() { + long ts = Long.MAX_VALUE; + for (final ITimeSyncedIterator sub : queries.values()) { + if (sub != null) { + final long t = sub.nextTimestamp(); + if (t < ts) { + ts = t; + } + } + } + return ts; + } + + @Override + public int size() { + throw new NotImplementedException(); + } + + @Override + public ExpressionDataPoint[] values() { + throw new NotImplementedException(); + } + + @Override + public void nullIterator(int index) { + throw new NotImplementedException(); + } + + @Override + public int getIndex() { + return index; + } + + @Override + public void setIndex(int index) { + this.index = index; + } + + @Override + public String getId() { + return id; + } + + @Override + public ByteSet getQueryTagKs() { + throw new NotImplementedException(); + } + + @Override + public void setFillPolicy(NumericFillPolicy policy) { + this.fill_policy = policy; + } + + @Override + public NumericFillPolicy getFillPolicy() { + return fill_policy; + } + + @Override + public ITimeSyncedIterator getCopy() { + return new UnionIterator(this); + } + + @Override + public void next() { + if (!hasNext()) { + throw new IllegalDataException("No more data"); + } + for (final ITimeSyncedIterator sub : queries.values()) { + sub.next(timestamp); + } + // reset the fill data point + fill_dp.reset(timestamp, fill_policy.getValue()); + timestamp = nextTimestamp(); + } + + @Override + public Map getResults() { + return current_values; + } + + @Override + public int getSeriesSize() { + return series_size; + } + + @Override + public boolean hasNext(int index) { + for (final Entry entry : single_series_matrix.entrySet()) { + final int idx = entry.getValue()[index]; + if (idx >= 0 && queries.get(entry.getKey()).hasNext(idx)) { + return true; + } + } + return false; + } + + @Override + public void next(int index) { + if (!hasNext()) { + throw new IllegalDataException("No more data"); + } + for (final Entry entry : single_series_matrix.entrySet()) { + final int idx = entry.getValue()[index]; + if (idx >= 0) { + queries.get(entry.getKey()).next(idx); + } + } + } + +} diff --git a/src/query/expression/VariableIterator.java b/src/query/expression/VariableIterator.java new file mode 100644 index 0000000000..82fedcfce6 --- /dev/null +++ b/src/query/expression/VariableIterator.java @@ -0,0 +1,115 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.expression; + +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; + +/** + * An interface that helps merge different time series sets (e.g. different + * metrics with a group by operator). The implementations handle joining the + * two sets according to the {@link SetOperator}. + * @since 2.3 + */ +public interface VariableIterator { + + /** An operator that determines how to sets of time series are merged via + * expression. */ + public enum SetOperator { + /** A union, meaning results from all sets will appear, using FillPolicies + * for missing series */ + UNION("union"), + + /** Computes the intersection, returning results only for series that appear + * in all sets */ + INTERSECTION("intersection"); + + /** The user-friendly name of this operator. */ + private final String name; + + /** @param the readable name of the operator */ + SetOperator(final String name) { + this.name = name; + } + + /** @return the readable name of the operator */ + @JsonValue + public String getName() { + return name; + } + + /** + * Converts a string to lower case then looks up the operator + * @param name The name to find an operator for + * @return The operator if found. + * @throws IllegalArgumentException if the operator wasn't found + */ + @JsonCreator + public static SetOperator fromString(final String name) { + for (final SetOperator operator : SetOperator.values()) { + if (operator.name.equalsIgnoreCase(name)) { + return operator; + } + } + throw new IllegalArgumentException("Unrecognized set operator: " + name); + } + } + + /** + * Whether or not another set of results are available. Always call this + * before calling next. + * @return True if more results are available, false if not. + */ + public boolean hasNext(); + + /** + * Iterates the {@link getResults()} to the next set of results. If there + * aren't any results left, the implementation may throw an exception. Always + * call {@link hasNext()} first. + */ + public void next(); + + /** + * Determines whether the individual series in the {@code values} array has + * another value. This may be used for non-synchronous iteration. + * @param index The index of the series in the values array to check for + * @return True if the series has another value, false if not + */ + public boolean hasNext(final int index); + + /** + * Fetches the next value for an individual series in the {@code values} array. + * @param index The index of the series in the values array to advance + */ + public void next(final int index); + + /** + * Returns a map of variable names to result series. You can maintain the + * reference returned without having to call getResults() on every iteration. + * Calling {@link next()} will simply update the ExpressionDataPoint array. + * The implementation may return a null map if there weren't any results + * available. Always all {@link hasNext()} before getting the results. + * @return A map with results to read from. + */ + public Map getResults(); + + /** @return The number of time series after the join. This should match the + * number of entries in the results data point array, not the number of + * variables in the results map. */ + public int getSeriesSize(); + + /** @return the next timestamp for all results without iterating */ + public long nextTimestamp(); +} diff --git a/src/query/filter/TagVFilter.java b/src/query/filter/TagVFilter.java new file mode 100644 index 0000000000..94c38f5d2f --- /dev/null +++ b/src/query/filter/TagVFilter.java @@ -0,0 +1,644 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.filter; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import org.hbase.async.Bytes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import net.opentsdb.core.TSDB; +import net.opentsdb.uid.UniqueId.UniqueIdType; +import net.opentsdb.utils.Config; +import net.opentsdb.utils.Pair; +import net.opentsdb.utils.PluginLoader; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; +import com.stumbleupon.async.Callback; +import com.stumbleupon.async.Deferred; + +/** + * A base class for tag value filters that may execute against rows that + * come out of a scanner to determine if we should include them in the results + * or not. The filters should be prefixed with something to differentiate them + * from literal values. + * + * Every filter must be associated with a tag key. During scanning, each time + * a new TSUID is encountered, the map will be passed to {@link match} for + * matching. + * + * Plugins implementing the filter must include the following: + * + * - {@code public static final String FILTER_NAME;} + * A short, unique name without spaces or odd characters that is used to + * invoke the filter. + * - {@code public static String description();} + * A method that returns a description of what the filter does. + * - {@code public static String examples();} + * A method that returns a string with some examples of how to use the filter. + * + * This class also contains the list of configured filters as well as a method + * to load filters from plugin Jars. + * @since 2.2 + */ +@JsonDeserialize(builder = TagVFilter.Builder.class) +public abstract class TagVFilter implements Comparable { + private static final Logger LOG = LoggerFactory.getLogger(TagVFilter.class); + + /** A map of configured filters for use in querying */ + private static Map, Constructor>> + tagv_filter_map = new HashMap, Constructor>>(); + static { + try { + tagv_filter_map.put(TagVLiteralOrFilter.FILTER_NAME, + new Pair, Constructor>(TagVLiteralOrFilter.class, + TagVLiteralOrFilter.class.getDeclaredConstructor(String.class, String.class))); + tagv_filter_map.put(TagVLiteralOrFilter.TagVILiteralOrFilter.FILTER_NAME, + new Pair, Constructor>(TagVLiteralOrFilter.TagVILiteralOrFilter.class, + TagVLiteralOrFilter.TagVILiteralOrFilter.class.getDeclaredConstructor(String.class, String.class))); + tagv_filter_map.put(TagVNotLiteralOrFilter.FILTER_NAME, + new Pair, Constructor>(TagVNotLiteralOrFilter.class, + TagVNotLiteralOrFilter.class.getDeclaredConstructor(String.class, String.class))); + tagv_filter_map.put(TagVNotLiteralOrFilter.TagVNotILiteralOrFilter.FILTER_NAME, + new Pair, Constructor>(TagVNotLiteralOrFilter.TagVNotILiteralOrFilter.class, + TagVNotLiteralOrFilter.TagVNotILiteralOrFilter.class.getDeclaredConstructor(String.class, String.class))); + tagv_filter_map.put(TagVRegexFilter.FILTER_NAME, + new Pair, Constructor>(TagVRegexFilter.class, + TagVRegexFilter.class.getDeclaredConstructor(String.class, String.class))); + tagv_filter_map.put(TagVWildcardFilter.FILTER_NAME, + new Pair, Constructor>(TagVWildcardFilter.class, + TagVWildcardFilter.class.getDeclaredConstructor(String.class, String.class))); + tagv_filter_map.put(TagVWildcardFilter.TagVIWildcardFilter.FILTER_NAME, + new Pair, Constructor>(TagVWildcardFilter.TagVIWildcardFilter.class, + TagVWildcardFilter.TagVIWildcardFilter.class.getDeclaredConstructor(String.class, String.class))); + /* TODO - this requires either a better HBase filter or more logic on our side + tagv_filter_map.put(TagVNotKeyFilter.FILTER_NAME, + new Pair, Constructor>(TagVNotKeyFilter.class, + TagVNotKeyFilter.class.getDeclaredConstructor(String.class, String.class))); + */ + } catch (SecurityException e) { + throw new RuntimeException("Failed to load a tag value filter", e); + } catch (NoSuchMethodException e) { + throw new RuntimeException("Failed to load a tag value filter", e); + } + } + + /** The tag key this filter is associated with */ + final protected String tagk; + + /** The raw, unparsed filter */ + final protected String filter; + + /** The tag key converted into a UID */ + protected byte[] tagk_bytes; + + /** An optional list of tag value UIDs if the filter matches on literals. */ + protected List tagv_uids; + + /** Whether or not to also group by this filter */ + @JsonProperty + protected boolean group_by; + + /** A flag to indicate whether or not we need to execute a post-scan lookup */ + protected boolean post_scan = true; + + /** + * Default Ctor needed for the service loader. Implementations must override + * and set the filterName(). + */ + public TagVFilter() { + this.tagk = null; + this.filter = null; + } + + /** + * The ctor that validates we have a good tag key to work with + * @param tagk The tag key to associate with this filter + * @param filter The unparsed filter + * @throws IllegalArgumentException if the tag was empty or null. + */ + public TagVFilter(final String tagk, final String filter) { + this.tagk = tagk; + this.filter = filter; + if (tagk == null || tagk.isEmpty()) { + throw new IllegalArgumentException("Filter must have a tagk"); + } + } + + /** + * Looks up the tag key in the given map and determines if the filter matches + * or not. If the tag key doesn't exist in the tag map, then the match fails. + * @param tags The tag map to use for looking up the value for the tagk + * @return True if the tag value matches, false if it doesn't. + */ + public abstract Deferred match(final Map tags); + + /** + * The name of this filter as used in queries. When used in URL queries the + * value will be in parentheses, e.g. filter(<exp>) + * The name will also be lowercased before storing it in the lookup map. + * @return The name of the filter. + */ + public abstract String getType(); + + /** + * A simple string of the filter settings for printing in toString() calls. + * @return A string with the format "{settings=<val>, ...}" + */ + @JsonIgnore + public abstract String debugInfo(); + + @Override + public String toString() { + final StringBuilder buf = new StringBuilder(); + buf.append("filter_name=") + .append(getType()) + .append(", tagk=").append(tagk) + .append(", group_by=").append(group_by) + .append(", tagk_bytes=").append(Bytes.pretty(tagk_bytes)) + .append(", config=") + .append(debugInfo()); + return buf.toString(); + } + + /** + * Parses the tag value and determines if it's a group by, a literal or a filter. + * @param tagk The tag key associated with this value + * @param filter The tag value, possibly a filter + * @return Null if the value was a group by or a literal, a valid filter object + * if it looked to be a filter. + * @throws IllegalArgumentException if the tag key or filter was null, empty + * or if the filter was malformed, e.g. a bad regular expression. + */ + public static TagVFilter getFilter(final String tagk, final String filter) { + if (tagk == null || tagk.isEmpty()) { + throw new IllegalArgumentException("Tagk cannot be null or empty"); + } + if (filter == null || filter.isEmpty()) { + throw new IllegalArgumentException("Filter cannot be null or empty"); + } + if (filter.length() == 1 && filter.charAt(0) == '*') { + return null; // group by filter + } + + final int paren = filter.indexOf('('); + if (paren > -1) { + final String prefix = filter.substring(0, paren).toLowerCase(); + return new Builder().setTagk(tagk) + .setFilter(stripParentheses(filter)) + .setType(prefix) + .build(); + } else if (filter.contains("*")) { + // a shortcut for wildcards since we don't allow asterisks to be stored + // in strings at this time. + return new TagVWildcardFilter(tagk, filter, true); + } else { + return null; // likely a literal or unknown + } + } + + /** + * Helper to strip parentheses from a filter name passed in over a URL + * or JSON. E.g. "regexp(foo.*)" returns "foo.*". + * @param filter The filter string to parse + * @return The filter value minus the surrounding name and parens. + */ + public static String stripParentheses(final String filter) { + if (filter == null || filter.isEmpty()) { + throw new IllegalArgumentException("Filter string cannot be null or empty"); + } + if (filter.charAt(filter.length() - 1) != ')') { + throw new IllegalArgumentException("Filter must end with a ')': " + filter); + } + final int start_pos = filter.indexOf('('); + if (start_pos < 0) { + throw new IllegalArgumentException("Filter must include a '(': " + filter); + } + return filter.substring(start_pos + 1, filter.length() - 1); + } + + /** + * Loads plugins from the plugin directory and loads them into the map. + * Built-in filters don't need to go through this process. + * @param tsdb A TSDB to use to initialize plugins + * @throws ClassNotFoundException If we found a class that we didn't... find? + * @throws NoSuchMethodException If the discovered plugin didn't have the + * proper (tagk, filter) ctor + * @throws InvocationTargetException if the static "initialize(tsdb)" method + * doesn't exist. + * @throws IllegalAccessException if something went really pear shaped + * @throws SecurityException if the JVM is really unhappy with the user + * @throws IllegalArgumentException really shouldn't happen but you know, + * checked exceptions... + */ + public static void initializeFilterMap(final TSDB tsdb) + throws ClassNotFoundException, NoSuchMethodException, NoSuchFieldException, + IllegalArgumentException, SecurityException, IllegalAccessException, + InvocationTargetException { + final List filter_plugins = + PluginLoader.loadPlugins(TagVFilter.class); + if (filter_plugins != null) { + for (final TagVFilter filter : filter_plugins) { + // validate required fields and methods + filter.getClass().getDeclaredMethod("description"); + filter.getClass().getDeclaredMethod("examples"); + filter.getClass().getDeclaredField("FILTER_NAME"); + + final Method initialize = filter.getClass() + .getDeclaredMethod("initialize", TSDB.class); + initialize.invoke(null, tsdb); + + final Constructor ctor = + filter.getClass().getDeclaredConstructor(String.class, String.class); + + final Pair, Constructor> existing = + tagv_filter_map.get(filter.getType()); + if (existing != null) { + LOG.warn("Overloading existing filter " + + existing.getClass().getCanonicalName() + + " with new filter " + filter.getClass().getCanonicalName()); + } + tagv_filter_map.put(filter.getType().toLowerCase(), + new Pair, Constructor>( + filter.getClass(), ctor)); + LOG.info("Successfully loaded TagVFilter plugin: " + + filter.getClass().getCanonicalName()); + } + LOG.info("Loaded " + tagv_filter_map.size() + " filters"); + } + } + + /** + * Converts the tag map to a filter list. If a filter already exists for a + * tag group by, then the duplicate is skipped. + * @param tags A set of tag keys and values. May be null or empty. + * @param filters A set of filters to add the converted filters to. This may + * not be null. + */ + public static void tagsToFilters(final Map tags, + final List filters) { + mapToFilters(tags, filters, true); + } + + /** + * Converts the map to a filter list. If a filter already exists for a + * tag group by and we're told to process group bys, then the duplicate + * is skipped. + * @param map A set of tag keys and values. May be null or empty. + * @param filters A set of filters to add the converted filters to. This may + * not be null. + * @param group_by Whether or not to set the group by flag and kick dupes + */ + public static void mapToFilters(final Map map, + final List filters, final boolean group_by) { + if (map == null || map.isEmpty()) { + return; + } + + for (final Map.Entry entry : map.entrySet()) { + TagVFilter filter = getFilter(entry.getKey(), entry.getValue()); + + if (filter == null && entry.getValue().equals("*")) { + filter = new TagVWildcardFilter(entry.getKey(), "*", true); + } else if (filter == null) { + filter = new TagVLiteralOrFilter(entry.getKey(), entry.getValue()); + } + + if (group_by) { + filter.setGroupBy(true); + boolean duplicate = false; + for (final TagVFilter existing : filters) { + if (filter.equals(existing)) { + LOG.debug("Skipping duplicate filter: " + existing); + existing.setGroupBy(true); + duplicate = true; + break; + } + } + + if (!duplicate) { + filters.add(filter); + } + } else { + filters.add(filter); + } + } + } + + /** + * Runs through the loaded plugin map and dumps the names, description and + * examples into a map to serialize via the API. + * @return A map of filter meta data. + */ + public static Map> loadedFilters() { + final Map> filters = + new HashMap>(tagv_filter_map.size()); + for (final Pair, Constructor> pair : + tagv_filter_map.values()) { + final Map filter_meta = new HashMap(1); + try { + Method method = pair.getKey().getDeclaredMethod("description"); + filter_meta.put("description", (String)method.invoke(null)); + + method = pair.getKey().getDeclaredMethod("examples"); + filter_meta.put("examples", (String)method.invoke(null)); + + final Field filter_name = pair.getKey().getDeclaredField("FILTER_NAME"); + filters.put((String)filter_name.get(null), filter_meta); + } catch (SecurityException e) { + throw new RuntimeException("Unexpected security exception", e); + } catch (NoSuchMethodException e) { + LOG.error("Filter plugin " + pair.getClass().getCanonicalName() + + " did not implement one of the \"description\" or \"examples\" methods"); + } catch (NoSuchFieldException e) { + LOG.error("Filter plugin " + pair.getClass().getCanonicalName() + + " did not have the \"FILTER_NAME\" field"); + } catch (IllegalArgumentException e) { + throw new RuntimeException("Unexpected exception", e); + } catch (IllegalAccessException e) { + throw new RuntimeException("Unexpected security exception", e); + } catch (InvocationTargetException e) { + throw new RuntimeException("Unexpected security exception", e); + } + + } + return filters; + } + + /** + * Asynchronously resolves the tagk name to it's UID. On a successful lookup + * the {@link tagk_bytes} will be set. + * @param tsdb The TSDB to use for the lookup + * @return A deferred to let the caller know that the lookup was completed. + * The value will be the tag UID (unless it's an exception of course) + */ + public Deferred resolveTagkName(final TSDB tsdb) { + class ResolvedCB implements Callback { + @Override + public byte[] call(final byte[] uid) throws Exception { + tagk_bytes = uid; + return uid; + } + } + + return tsdb.getUIDAsync(UniqueIdType.TAGK, tagk) + .addCallback(new ResolvedCB()); + } + + /** + * Resolves both the tagk to it's UID and a list of literal tag values to + * their UIDs. A filter may match a literal set (e.g. the pipe filter) in which + * case we can build the row key scanner with these values. + * Note that if "tsd.query.skip_unresolved_tagvs" is set in the config then + * any tag value UIDs that couldn't be found will be excluded. + * @param tsdb The TSDB to use for the lookup + * @param literals The list of unique strings to lookup + * @return A deferred to let the caller know that the lookup was completed. + * The value will be the tag UID (unless it's an exception of course) + */ + public Deferred resolveTags(final TSDB tsdb, + final Set literals) { + final Config config = tsdb.getConfig(); + + /** + * Allows the filter to avoid killing the entire query when we can't resolve + * a tag value to a UID. + */ + class TagVErrback implements Callback { + @Override + public byte[] call(final Exception e) throws Exception { + if (config.getBoolean("tsd.query.skip_unresolved_tagvs")) { + LOG.warn("Query tag value not found: " + e.getMessage()); + return null; + } else { + throw e; + } + } + } + + /** + * Stores the non-null UIDs in the local list and then sorts them in + * prep for use in the regex filter + */ + class ResolvedTagVCB implements Callback> { + @Override + public byte[] call(final ArrayList results) + throws Exception { + tagv_uids = new ArrayList(results.size() - 1); + for (final byte[] tagv : results) { + if (tagv != null) { + tagv_uids.add(tagv); + } + } + Collections.sort(tagv_uids, Bytes.MEMCMP); + return tagk_bytes; + } + } + + /** + * Super simple callback to set the local tagk and returns null so it won't + * be included in the tag value UID lookups. + */ + class ResolvedTagKCB implements Callback { + @Override + public byte[] call(final byte[] uid) throws Exception { + tagk_bytes = uid; + return null; + } + } + + final List> tagvs = + new ArrayList>(literals.size()); + for (final String tagv : literals) { + tagvs.add(tsdb.getUIDAsync(UniqueIdType.TAGV, tagv) + .addErrback(new TagVErrback())); + } + // ugly hack to resolve the tagk UID. The callback will return null and we'll + // remove it from the UID list. + tagvs.add(tsdb.getUIDAsync(UniqueIdType.TAGK, tagk) + .addCallback(new ResolvedTagKCB())); + return Deferred.group(tagvs).addCallback(new ResolvedTagVCB()); + } + + /** @return the tag key associated with this filter */ + public String getTagk() { + return tagk; + } + + /** @return the tag key UID associated with this filter. + * Call {@link #resolveTagkName(TSDB)} first */ + @JsonIgnore + public byte[] getTagkBytes() { + return tagk_bytes; + } + + /** @return a non-null list of tag value UIDs. May be empty. */ + @JsonIgnore + public List getTagVUids() { + return tagv_uids == null ? Collections.emptyList() : tagv_uids; + } + + /** @return A copy of this filter BEFORE tag resolution, as a new object. */ + @JsonIgnore + public TagVFilter getCopy() { + return Builder() + .setFilter(filter) + .setTagk(tagk) + .setType(getType()) + .setGroupBy(group_by) + .build(); + } + + /** @return whether or not to group by the results of this filter */ + @JsonIgnore + public boolean isGroupBy() { + return group_by; + } + + /** @param group_by Wether or not to group by the results of this filter */ + public void setGroupBy(final boolean group_by) { + this.group_by = group_by; + } + + public String getFilter() { + return filter; + } + + /** @return the simple class name of this filter */ + @JsonIgnore + public String getName() { + return this.getClass().getSimpleName(); + } + + /** @return Whether or not this filter should be executed against scan results */ + public boolean postScan() { + return post_scan; + } + + /** @param post_scan Whether or not this filter should be executed against + * scan results */ + public void setPostScan(final boolean post_scan) { + this.post_scan = post_scan; + } + + @Override + public int compareTo(final TagVFilter filter) { + return Bytes.memcmpMaybeNull(tagk_bytes, filter.tagk_bytes); + } + + /** @return a TagVFilter builder for constructing filters */ + public static Builder Builder() { + return new Builder(); + } + + /** + * Builder class used for deserializing filters from JSON queries via Jackson + * since we don't want the user to worry about the class name. The type, + * tagk and filter must be configured or the build will fail. + */ + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonPOJOBuilder(buildMethodName = "build", withPrefix = "set") + public static class Builder { + private String type; + private String tagk; + private String filter; + @JsonProperty + private boolean group_by; + + /** @param type The type of filter matching a valid filter name */ + public Builder setType(final String type) { + this.type = type; + return this; + } + + /** @param tagk The tag key to match on for this filter */ + public Builder setTagk(final String tagk) { + this.tagk = tagk; + return this; + } + + /** @param filter The filter expression to use for matching */ + public Builder setFilter(final String filter) { + this.filter = filter; + return this; + } + + /** @param group_by Whether or not the filter should group results */ + public Builder setGroupBy(final boolean group_by) { + this.group_by = group_by; + return this; + } + + /** + * Searches the filter map for the given type and returns an instantiated + * filter if found. The caller must set the type, tagk and filter values. + * @return A filter if instantiation was successful + * @throws IllegalArgumentException if one of the required parameters was + * not set or the filter couldn't be found. + * @throws RuntimeException if the filter couldn't be instantiated. Check + * the implementation if it's a plugin. + */ + public TagVFilter build() { + if (type == null || type.isEmpty()) { + throw new IllegalArgumentException( + "The filter type cannot be null or empty"); + } + if (tagk == null || tagk.isEmpty()) { + throw new IllegalArgumentException( + "The tagk cannot be null or empty"); + } + + final Pair, Constructor> filter_meta = + tagv_filter_map.get(type); + if (filter_meta == null) { + throw new IllegalArgumentException( + "Could not find a tag value filter of the type: " + type); + } + final Constructor ctor = filter_meta.getValue(); + final TagVFilter tagv_filter; + try { + tagv_filter = ctor.newInstance(tagk, filter); + } catch (IllegalArgumentException e) { + throw e; + } catch (InstantiationException e) { + throw new RuntimeException("Failed to instantiate filter: " + type, e); + } catch (IllegalAccessException e) { + throw new RuntimeException("Failed to instantiate filter: " + type, e); + } catch (InvocationTargetException e) { + if (e.getCause() != null) { + throw (RuntimeException)e.getCause(); + } + throw new RuntimeException("Failed to instantiate filter: " + type, e); + } + + tagv_filter.setGroupBy(group_by); + return tagv_filter; + } + } +} diff --git a/src/query/filter/TagVLiteralOrFilter.java b/src/query/filter/TagVLiteralOrFilter.java new file mode 100644 index 0000000000..f2ba5446ce --- /dev/null +++ b/src/query/filter/TagVLiteralOrFilter.java @@ -0,0 +1,211 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.filter; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import net.opentsdb.core.TSDB; +import net.opentsdb.utils.Config; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.base.Objects; +import com.stumbleupon.async.Deferred; + +/** + * A filter that lets the user list one or more explicit strings that should + * be included in a result set for aggregation. + * @since 2.2 + */ +public class TagVLiteralOrFilter extends TagVFilter { + + /** Name of this filter */ + final public static String FILTER_NAME = "literal_or"; + + /** A list of strings to match on */ + final protected Set literals; + + /** Whether or not the match should be case insensitive */ + final protected boolean case_insensitive; + + /** + * The default Ctor that disables case insensitivity + * @param tagk The tag key to associate with this filter + * @param filter The filter to match on + * @throws IllegalArgumentException if the tagk or filter were empty or null + */ + public TagVLiteralOrFilter(final String tagk, final String filter) { + this(tagk, filter, false); + } + + /** + * A ctor that allows enabling case insensitivity + * @param tagk The tag key to associate with this filter + * @param filter The filter to match on + * @param case_insensitive Whether or not to match on case + * @throws IllegalArgumentException if the tagk or filter were empty or null + */ + public TagVLiteralOrFilter(final String tagk, final String filter, + final boolean case_insensitive) { + super(tagk, filter); + this.case_insensitive = case_insensitive; + + // we have to have at least one character. + if (filter == null || filter.isEmpty()) { + throw new IllegalArgumentException("Filter cannot be null or empty"); + } + if (filter.length() == 1 && filter.charAt(0) == '|') { + throw new IllegalArgumentException("Filter must contain more than just a pipe"); + } + final String[] split = filter.split("\\|"); + if (case_insensitive) { + for (int i = 0; i < split.length; i++) { + split[i] = split[i].toLowerCase(); + } + } + literals = new HashSet(Arrays.asList(split)); + } + + @Override + public Deferred match(final Map tags) { + final String tagv = tags.get(tagk); + if (tagv == null) { + return Deferred.fromResult(false); + } + return Deferred.fromResult( + literals.contains(case_insensitive ? tagv.toLowerCase() : tagv)); + } + + @Override + public String debugInfo() { + return "{literals=" + literals + ", case=" + case_insensitive + "}"; + } + + /** + * Overridden here so that we can resolve the literal values if we don't have + * too many of them AND we're not searching with case insensitivity. + */ + @Override + public Deferred resolveTagkName(final TSDB tsdb) { + final Config config = tsdb.getConfig(); + + // resolve tag values if the filter is NOT case insensitive and there are + // fewer literals than the expansion limit + if (!case_insensitive && + literals.size() <= config.getInt("tsd.query.filter.expansion_limit")) { + return resolveTags(tsdb, literals); + } else { + return super.resolveTagkName(tsdb); + } + } + + /** @return Whether or not this filter has case insensitivity enabled */ + @JsonIgnore + public boolean isCaseInsensitive() { + return case_insensitive; + } + + @Override + public boolean equals(final Object obj) { + if (obj == null) { + return false; + } + if (!(obj instanceof TagVLiteralOrFilter)) { + return false; + } + if (obj == this) { + return true; + } + final TagVLiteralOrFilter filter = (TagVLiteralOrFilter)obj; + return Objects.equal(tagk, filter.tagk) + && Objects.equal(literals, filter.literals) + && Objects.equal(case_insensitive, filter.case_insensitive); + } + + @Override + public int hashCode() { + return Objects.hashCode(tagk, literals, case_insensitive); + } + + @Override + public String getType() { + return FILTER_NAME; + } + + /** @return a string describing the filter */ + public static String description() { + return "Accepts one or more exact values and matches if the series contains " + + "any of them. Multiple values can be included and must be separated " + + "by the | (pipe) character. The filter is case sensitive and will not " + + "allow characters that TSDB does not allow at write time."; + } + + /** @return a list of examples showing how to use the filter */ + public static String examples() { + return "host=literal_or(web01), host=literal_or(web01|web02|web03) " + + "{\"type\":\"literal_or\",\"tagk\":\"host\"," + + "\"filter\":\"web01|web02|web03\",\"groupBy\":false}"; + } + + /** + * Case insensitive version + */ + public static class TagVILiteralOrFilter extends TagVLiteralOrFilter { + + /** Name of this filter */ + final public static String FILTER_NAME = "iliteral_or"; + + public TagVILiteralOrFilter(final String tagk, final String filter) { + super(tagk, filter, true); + } + + @Override + public String getType() { + return FILTER_NAME; + } + + @Override + public boolean equals(final Object obj) { + if (obj == null) { + return false; + } + if (!(obj instanceof TagVILiteralOrFilter)) { + return false; + } + if (obj == this) { + return true; + } + final TagVILiteralOrFilter filter = (TagVILiteralOrFilter)obj; + return Objects.equal(tagk, filter.tagk) + && Objects.equal(literals, filter.literals) + && Objects.equal(case_insensitive, filter.case_insensitive); + } + + /** @return a string describing the filter */ + public static String description() { + return "Accepts one or more exact values and matches if the series contains " + + "any of them. Multiple values can be included and must be separated " + + "by the | (pipe) character. The filter is case insensitive and will not " + + "allow characters that TSDB does not allow at write time."; + } + + /** @return a list of examples showing how to use the filter */ + public static String examples() { + return "host=iliteral_or(web01), host=iliteral_or(web01|web02|web03) " + + "{\"type\":\"iliteral_or\",\"tagk\":\"host\"," + + "\"filter\":\"web01|web02|web03\",\"groupBy\":false}"; + } + } +} diff --git a/src/query/filter/TagVNotKeyFilter.java b/src/query/filter/TagVNotKeyFilter.java new file mode 100644 index 0000000000..fa2980082b --- /dev/null +++ b/src/query/filter/TagVNotKeyFilter.java @@ -0,0 +1,72 @@ +package net.opentsdb.query.filter; + +import java.util.Map; + +import com.google.common.base.Objects; +import com.stumbleupon.async.Deferred; + +public class TagVNotKeyFilter extends TagVFilter { + /** Name of this filter */ + final public static String FILTER_NAME = "not_key"; + + public TagVNotKeyFilter(final String tagk, final String filter) { + super(tagk, ""); + if (filter != null && filter.length() > 0) { + throw new IllegalArgumentException("The filter must be empty for the " + + FILTER_NAME + " filter"); + } + post_scan = true; + } + + @Override + public Deferred match(Map tags) { + if (tags.containsKey(tagk)) { + return Deferred.fromResult(false); + } + return Deferred.fromResult(true); + } + + @Override + public String getType() { + return FILTER_NAME; + } + + @Override + public String debugInfo() { + return "{}"; + } + + @Override + public boolean equals(final Object obj) { + if (obj == null) { + return false; + } + if (!(obj instanceof TagVRegexFilter)) { + return false; + } + if (obj == this) { + return true; + } + final TagVNotKeyFilter filter = (TagVNotKeyFilter)obj; + return Objects.equal(tagk, filter.tagk); + } + + @Override + public int hashCode() { + return Objects.hashCode(tagk); + } + + /** @return a string describing the filter */ + public static String description() { + return "Skips any time series with the given tag key, regardless of the " + + "value. This can be useful for situations where a metric has " + + "inconsistent tag sets. NOTE: The filter value must be null or an " + + "empty string."; + } + + /** @return a list of examples showing how to use the filter */ + public static String examples() { + return "host=not_key() {\"type\":\"not_key\",\"tagk\":\"host\"," + + "\"filter\":\"\",\"groupBy\":false}"; + } +} diff --git a/src/query/filter/TagVNotLiteralOrFilter.java b/src/query/filter/TagVNotLiteralOrFilter.java new file mode 100644 index 0000000000..1e6500c5e7 --- /dev/null +++ b/src/query/filter/TagVNotLiteralOrFilter.java @@ -0,0 +1,190 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.filter; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.base.Objects; +import com.stumbleupon.async.Deferred; + +/** + * A filter that lets the user list one or more explicit strings that should + * NOT be included in a result set for aggregation. + * @since 2.2 + */ +public class TagVNotLiteralOrFilter extends TagVFilter { + + /** Name of this filter */ + final public static String FILTER_NAME = "not_literal_or"; + + /** A list of strings to match on */ + final protected Set literals; + + /** Whether or not the match should be case insensitive */ + final protected boolean case_insensitive; + + /** + * The default Ctor that disables case insensitivity + * @param tagk The tag key to associate with this filter + * @param filter The filter to match on + * @throws IllegalArgumentException if the tagk or filter were empty or null + */ + public TagVNotLiteralOrFilter(final String tagk, final String filter) { + this(tagk, filter, false); + } + + /** + * A ctor that allows enabling case insensitivity + * @param tagk The tag key to associate with this filter + * @param filter The filter to match on + * @param case_insensitive Whether or not to match on case + * @throws IllegalArgumentException if the tagk or filter were empty or null + */ + public TagVNotLiteralOrFilter(final String tagk, final String filter, + final boolean case_insensitive) { + super(tagk, filter); + this.case_insensitive = case_insensitive; + + // we have to have at least one character. + if (filter == null || filter.isEmpty()) { + throw new IllegalArgumentException("Filter cannot be null or empty"); + } + if (filter.length() == 1 && filter.charAt(0) == '|') { + throw new IllegalArgumentException("Filter must contain more than just a pipe"); + } + final String[] split = filter.split("\\|"); + if (case_insensitive) { + for (int i = 0; i < split.length; i++) { + split[i] = split[i].toLowerCase(); + } + } + literals = new HashSet(Arrays.asList(split)); + } + + @Override + public Deferred match(final Map tags) { + final String tagv = tags.get(tagk); + if (tagv == null) { + return Deferred.fromResult(true); + } + return Deferred.fromResult( + !(literals.contains(case_insensitive ? tagv.toLowerCase() : tagv))); + } + + @Override + public String debugInfo() { + return "{literals=" + literals + ", case=" + case_insensitive + "}"; + } + + /** @return Whether or not this filter has case insensitivity enabled */ + @JsonIgnore + public boolean isCaseInsensitive() { + return case_insensitive; + } + + @Override + public String getType() { + return FILTER_NAME; + } + + @Override + public boolean equals(final Object obj) { + if (obj == null) { + return false; + } + if (!(obj instanceof TagVNotLiteralOrFilter)) { + return false; + } + if (obj == this) { + return true; + } + final TagVNotLiteralOrFilter filter = (TagVNotLiteralOrFilter)obj; + return Objects.equal(tagk, filter.tagk) + && Objects.equal(literals, filter.literals) + && Objects.equal(case_insensitive, filter.case_insensitive); + } + + @Override + public int hashCode() { + return Objects.hashCode(tagk, literals, case_insensitive); + } + + /** @return a string describing the filter */ + public static String description() { + return "Accepts one or more exact values and matches if the series does NOT " + + "contain any of them. Multiple values can be included and must be " + + "separated by the | (pipe) character. The filter is case sensitive " + + "and will not allow characters that TSDB does not allow at write time."; + } + + /** @return a list of examples showing how to use the filter */ + public static String examples() { + return "host=not_literal_or(web01), host=not_literal_or(web01|web02|web03) " + + "{\"type\":\"not_literal_or\",\"tagk\":\"host\"," + + "\"filter\":\"web01|web02|web03\",\"groupBy\":false}"; + } + + /** + * Case insensitive version + */ + public static class TagVNotILiteralOrFilter extends TagVNotLiteralOrFilter { + + /** Name of this filter */ + final public static String FILTER_NAME = "not_iliteral_or"; + + public TagVNotILiteralOrFilter(final String tagk, final String filter) { + super(tagk, filter, true); + } + + @Override + public String getType() { + return FILTER_NAME; + } + + @Override + public boolean equals(final Object obj) { + if (obj == null) { + return false; + } + if (!(obj instanceof TagVNotILiteralOrFilter)) { + return false; + } + if (obj == this) { + return true; + } + final TagVNotILiteralOrFilter filter = (TagVNotILiteralOrFilter)obj; + return Objects.equal(tagk, filter.tagk) + && Objects.equal(literals, filter.literals) + && Objects.equal(case_insensitive, filter.case_insensitive); + } + + /** @return a string describing the filter */ + public static String description() { + return "Accepts one or more exact values and matches if the series does NOT " + + "contain any of them. Multiple values can be included and must be " + + "separated by the | (pipe) character. The filter is case insensitive " + + "and will not allow characters that TSDB does not allow at write time."; + } + + /** @return a list of examples showing how to use the filter */ + public static String examples() { + return "host=not_iliteral_or(web01), host=not_iliteral_or(web01|web02|web03) " + + "{\"type\":\"not_iliteral_or\",\"tagk\":\"host\"," + + "\"filter\":\"web01|web02|web03\",\"groupBy\":false}"; + } + } +} diff --git a/src/query/filter/TagVRegexFilter.java b/src/query/filter/TagVRegexFilter.java new file mode 100644 index 0000000000..5be4ac2340 --- /dev/null +++ b/src/query/filter/TagVRegexFilter.java @@ -0,0 +1,108 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.filter; + +import java.util.Map; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +import com.google.common.base.Objects; +import com.stumbleupon.async.Deferred; + +/** + * A filter that allows for regular expression matching on tag values. + * @since 2.2 + */ +public class TagVRegexFilter extends TagVFilter { + /** Name of this filter */ + final public static String FILTER_NAME = "regexp"; + + /** The compiled pattern */ + final Pattern pattern; + + /** + * The default Ctor that disables case insensitivity + * @param tagk The tag key to associate with this filter + * @param filter The filter to match on + * @throws IllegalArgumentException if the tagk or filter were empty or null + * @throws PatternSyntaxException if the pattern was invalid + */ + public TagVRegexFilter(final String tagk, final String filter) { + super(tagk, filter); + // we have to have at least one character. + if (filter == null || filter.length() < 1) { + throw new IllegalArgumentException("Filter cannot be null or empty"); + } + pattern = Pattern.compile(filter); + } + + @Override + public Deferred match(final Map tags) { + final String tagv = tags.get(tagk); + if (tagv == null) { + return Deferred.fromResult(false); + } + return Deferred.fromResult(pattern.matcher(tagv).find()); + } + + @Override + public String debugInfo() { + return "{pattern=" + pattern.toString() + "}"; + } + + @Override + public boolean equals(final Object obj) { + if (obj == null) { + return false; + } + if (!(obj instanceof TagVRegexFilter)) { + return false; + } + if (obj == this) { + return true; + } + final TagVRegexFilter filter = (TagVRegexFilter)obj; + // NOTE: apparently different pattern objects with the SAME pattern will + // return a different hash. *sigh*. So cast the pattern to a string, THEN + // compare. + return Objects.equal(tagk, filter.tagk) + && Objects.equal(pattern.pattern(), filter.pattern.pattern()); + } + + @Override + public int hashCode() { + // NOTE: apparently different pattern objects with the SAME pattern will + // return a different hash. *sigh*. So cast the pattern to a string, THEN + // compare. + return Objects.hashCode(tagk, pattern.pattern()); + } + + @Override + public String getType() { + return FILTER_NAME; + } + + /** @return a string describing the filter */ + public static String description() { + return "Provides full, POSIX compliant regular expression using the " + + "built in Java Pattern class. Note that an expression containing " + + "curly braces {} will not parse properly in URLs. If the pattern " + + "is not a valid regular expression then an exception will be raised."; + } + + /** @return a list of examples showing how to use the filter */ + public static String examples() { + return "host=regexp(.*) {\"type\":\"regexp\",\"tagk\":\"host\"," + + "\"filter\":\".*\",\"groupBy\":false}"; + } +} diff --git a/src/query/filter/TagVWildcardFilter.java b/src/query/filter/TagVWildcardFilter.java new file mode 100644 index 0000000000..82d8da97ab --- /dev/null +++ b/src/query/filter/TagVWildcardFilter.java @@ -0,0 +1,229 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.filter; + +import java.util.Arrays; +import java.util.Map; + +import net.opentsdb.core.Tags; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.base.Objects; +import com.stumbleupon.async.Deferred; + +/** + * Performs basic wild card searching. It supports prefix, postfix, infix, + * multi-infix and case insensitive matching. The wildcard character is + * an asterisk. If case insensitivity is enabled, we simply drop everything + * to lower case. + * @since 2.2 + */ +public class TagVWildcardFilter extends TagVFilter { + + /** Name of this filter */ + final public static String FILTER_NAME = "wildcard"; + + /** Whether or not the filter had a postfix asterisk */ + protected final boolean has_postfix; + + /** Whether or not the filter had a prefix asterisk */ + protected final boolean has_prefix; + + /** The individual components to match on */ + protected final String[] components; + + /** Whether or not we'll match case */ + protected boolean case_insensitive; + + /** + * The default Ctor that disables case insensitivity + * @param tagk The tag key to associate with this filter + * @param filter The wildcard filter to match on + * @throws IllegalArgumentException if the tagk or filter were empty or null + */ + public TagVWildcardFilter(final String tagk, final String filter) { + this(tagk, filter, false); + } + + /** + * A ctor that allows enabling case insensitivity + * @param tagk The tag key to associate with this filter + * @param filter The wildcard filter to match on + * @param case_insensitive Whether or not to match on case + * @throws IllegalArgumentException if the tagk or filter were empty or null + */ + public TagVWildcardFilter(final String tagk, final String filter, + final boolean case_insensitive) { + super(tagk, filter); + this.case_insensitive = case_insensitive; + + if (filter == null || filter.length() < 1) { + throw new IllegalArgumentException("Filter cannot be null or empty"); + } + String actual = case_insensitive ? filter.toLowerCase() : filter; + if (!actual.contains("*")) { + throw new IllegalArgumentException("Filter must contain an asterisk"); + } + + if (actual.charAt(0) == '*') { + has_postfix = true; + while (actual.charAt(0) == '*') { + if (actual.length() < 2) { + break; + } + actual = actual.substring(1); + } + } else { + has_postfix = false; + } + if (actual.charAt(actual.length() - 1) == '*') { + has_prefix = true; + while(actual.charAt(actual.length() - 1) == '*') { + if (actual.length() < 2) { + break; + } + actual = actual.substring(0, actual.length() - 1); + } + } else { + has_prefix = false; + } + if (actual.indexOf('*') > 0) { + components = Tags.splitString(actual, '*'); + } else { + components = new String[1]; + components[0] = actual; + } + + // avoid resolving UIDs at scan time + if (components.length == 1 && components[0].equals("*")) { + post_scan = false; + } + } + + @Override + public Deferred match(final Map tags) { + String tagv = tags.get(tagk); + if (tagv == null) { + return Deferred.fromResult(false); + } else if (components.length == 1 && components[0].equals("*")) { + // match all + return Deferred.fromResult(true); + } else if (case_insensitive) { + tagv = tags.get(tagk).toLowerCase(); + } + if (has_postfix && !has_prefix && + !tagv.endsWith(components[components.length-1])) { + return Deferred.fromResult(false); + } + if (has_prefix && !has_postfix && !tagv.startsWith(components[0])) { + return Deferred.fromResult(false); + } + int idx = 0; + for (int i = 0; i < components.length; i++) { + if (tagv.indexOf(components[i], idx) < 0) { + return Deferred.fromResult(false); + } + idx += components[i].length(); + } + return Deferred.fromResult(true); + } + + @Override + public String debugInfo() { + return "{components=" + Arrays.toString(components) + ", case=" + + case_insensitive + "}"; + } + + /** @return Whether or not this filter has case insensitivity enabled */ + @JsonIgnore + public boolean isCaseInsensitive() { + return case_insensitive; + } + + @Override + public boolean equals(final Object obj) { + if (obj == null) { + return false; + } + if (!(obj instanceof TagVWildcardFilter)) { + return false; + } + if (obj == this) { + return true; + } + final TagVWildcardFilter filter = (TagVWildcardFilter)obj; + return Objects.equal(tagk, filter.tagk) + && Arrays.equals(components, filter.components) + && Objects.equal(case_insensitive, filter.case_insensitive); + } + + @Override + public int hashCode() { + return Objects.hashCode(tagk, Arrays.hashCode(components), case_insensitive); + } + + @Override + public String getType() { + return FILTER_NAME; + } + + /** @return a string describing the filter */ + public static String description() { + return "Performs pre, post and in-fix glob matching of values. The globs " + + "are case sensitive and multiple wildcards can be used. The wildcard " + + "character is the * (asterisk). At least one wildcard must be " + + "present in the filter value. A wildcard by itself can be used as " + + "well to match on any value for the tag key."; + } + + /** @return a list of examples showing how to use the filter */ + public static String examples() { + return "host=wildcard(web*), host=wildcard(web*.tsdb.net) " + + "{\"type\":\"wildcard\",\"tagk\":\"host\"," + + "\"filter\":\"web*.tsdb.net\",\"groupBy\":false}"; + } + + /** + * Case insensitive version + */ + public static class TagVIWildcardFilter extends TagVWildcardFilter { + /** Name of this filter */ + final public static String FILTER_NAME = "iwildcard"; + + public TagVIWildcardFilter(final String tagk, final String filter) { + super(tagk, filter, true); + } + + @Override + public String getType() { + return FILTER_NAME; + } + + /** @return a string describing the filter */ + public static String description() { + return "Performs pre, post and in-fix glob matching of values. The globs " + + "are case insensitive and multiple wildcards can be used. The wildcard " + + "character is the * (asterisk). Case insensitivity is achieved by " + + "dropping all values to lower case. At least one wildcard must be " + + "present in the filter value. A wildcard by itself can be used as " + + "well to match on any value for the tag key."; + } + + /** @return a list of examples showing how to use the filter */ + public static String examples() { + return "host=iwildcard(web*), host=iwildcard(web*.tsdb.net) " + + "{\"type\":\"iwildcard\",\"tagk\":\"host\"," + + "\"filter\":\"web*.tsdb.net\",\"groupBy\":false}"; + } + } +} diff --git a/src/query/pojo/Downsampler.java b/src/query/pojo/Downsampler.java new file mode 100644 index 0000000000..dfda751ffe --- /dev/null +++ b/src/query/pojo/Downsampler.java @@ -0,0 +1,146 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.pojo; + +import java.util.NoSuchElementException; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; +import com.google.common.base.Objects; + +import net.opentsdb.core.Aggregators; +import net.opentsdb.query.expression.NumericFillPolicy; +import net.opentsdb.utils.DateTime; + +/** + * Pojo builder class used for serdes of the downsampler component of a query + * @since 2.3 + */ +@JsonDeserialize(builder = Downsampler.Builder.class) +public class Downsampler extends Validatable { + /** The relative interval with value and unit, e.g. 60s */ + private String interval; + + /** The aggregator to use for downsampling */ + private String aggregator; + + /** A fill policy for downsampling and working with missing values */ + private NumericFillPolicy fill_policy; + + /** + * Default ctor + * @param builder The builder to pull values from + */ + public Downsampler(Builder builder) { + interval = builder.interval; + aggregator = builder.aggregator; + fill_policy = builder.fillPolicy; + } + + /** @return A new builder for the downsampler */ + public static Builder Builder() { + return new Builder(); + } + + /** Validates the downsampler + * @throws IllegalArgumentException if one or more parameters were invalid + */ + public void validate() { + if (interval == null || interval.isEmpty()) { + throw new IllegalArgumentException("Missing or empty interval"); + } + DateTime.parseDuration(interval); + + if (aggregator == null || aggregator.isEmpty()) { + throw new IllegalArgumentException("Missing or empty aggregator"); + } + try { + Aggregators.get(aggregator.toLowerCase()); + } catch (final NoSuchElementException e) { + throw new IllegalArgumentException("Invalid aggregator"); + } + + if (fill_policy != null) { + fill_policy.validate(); + } + } + + /** @return the interval for the downsampler */ + public String getInterval() { + return interval; + } + + /** @return the name of the aggregator to use */ + public String getAggregator() { + return aggregator; + } + + /** @return the fill policy to use */ + public NumericFillPolicy getFillPolicy() { + return fill_policy; + } + + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + final Downsampler downsampler = (Downsampler) o; + + return Objects.equal(interval, downsampler.interval) + && Objects.equal(aggregator, downsampler.aggregator) + && Objects.equal(fill_policy, downsampler.fill_policy); + } + + @Override + public int hashCode() { + return Objects.hashCode(interval, aggregator, fill_policy); + } + + /** + * A builder for the downsampler component of a query + */ + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonPOJOBuilder(buildMethodName = "build", withPrefix = "") + public static final class Builder { + @JsonProperty + private String interval; + @JsonProperty + private String aggregator; + @JsonProperty + private NumericFillPolicy fillPolicy; + + public Builder setInterval(String interval) { + this.interval = interval; + return this; + } + + public Builder setAggregator(String aggregator) { + this.aggregator = aggregator; + return this; + } + + public Builder setFillPolicy(NumericFillPolicy fill_policy) { + this.fillPolicy = fill_policy; + return this; + } + + public Downsampler build() { + return new Downsampler(this); + } + } +} diff --git a/src/query/pojo/Expression.java b/src/query/pojo/Expression.java new file mode 100644 index 0000000000..0b313d9835 --- /dev/null +++ b/src/query/pojo/Expression.java @@ -0,0 +1,198 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.pojo; + +import net.opentsdb.query.expression.ExpressionIterator; +import net.opentsdb.query.expression.NumericFillPolicy; +import net.opentsdb.query.expression.VariableIterator.SetOperator; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.commons.jexl2.Script; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; +import com.google.common.base.Objects; + +/** + * Pojo builder class used for serdes of the expression component of a query + * @since 2.3 + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonDeserialize(builder = Expression.Builder.class) +public class Expression extends Validatable { + /** An id for this expression for use in output selection or nested expressions */ + private String id; + + /** The raw expression as a string */ + private String expr; + + /** The joiner operator */ + private Join join; + + /** The fill policy to use for ? */ + private NumericFillPolicy fill_policy; + + /** Set of unique variables used by this expression. */ + private Set variables; + + /** The parsed expression via JEXL. */ + private Script parsed_expression; + + /** + * Default ctor + * @param builder The builder to pull values from + */ + protected Expression(Builder builder) { + id = builder.id; + expr = builder.expr; + join = builder.join; + fill_policy = builder.fillPolicy; + } + + /** @return the id for this expression for use in output selection or + * nested expressions */ + public String getId() { + return id; + } + + /** @return the raw expression as a string */ + public String getExpr() { + return expr; + } + + /** @return he joiner operator */ + public Join getJoin() { + return join; + } + + /** @return the fill policy to use for ? */ + public NumericFillPolicy getFillPolicy() { + return fill_policy; + } + + /** @return A new builder for the expression */ + public static Builder Builder() { + return new Builder(); + } + + /** Validates the expression + * @throws IllegalArgumentException if one or more parameters were invalid + */ + public void validate() { + if (id == null || id.isEmpty()) { + throw new IllegalArgumentException("missing or empty id"); + } + Query.validateId(id); + + if (expr == null || expr.isEmpty()) { + throw new IllegalArgumentException("missing or empty expr"); + } + + // parse it just to make sure we're happy and extract the variable names. + // Will throw JexlException + parsed_expression = ExpressionIterator.JEXL_ENGINE.createScript(expr); + variables = new HashSet(); + for (final List exp_list : + ExpressionIterator.JEXL_ENGINE.getVariables(parsed_expression)) { + for (final String variable : exp_list) { + variables.add(variable); + } + } + + // others are optional + if (join == null) { + join = Join.Builder().setOperator(SetOperator.UNION).build(); + } + } + + /** @return The parsed expression. May be null if {@link validate} has not + * been called yet. */ + @JsonIgnore + public Script getParsedExpression() { + return parsed_expression; + } + + /** @return A set of unique variables for the expression. May be null if + * {@link validate} has not been called yet. */ + @JsonIgnore + public Set getVariables() { + return variables; + } + + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + final Expression expression = (Expression) o; + + return Objects.equal(id, expression.id) + && Objects.equal(expr, expression.expr) + && Objects.equal(join, expression.join) + && Objects.equal(fill_policy, expression.fill_policy); + } + + @Override + public int hashCode() { + return Objects.hashCode(id, expr, join, fill_policy); + } + + /** + * A builder for the downsampler component of a query + */ + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonPOJOBuilder(buildMethodName = "build", withPrefix = "") + public static final class Builder { + @JsonProperty + private String id; + @JsonProperty + private String expr; + @JsonProperty + private Join join; + @JsonProperty + private NumericFillPolicy fillPolicy; + + public Builder setId(String id) { + Query.validateId(id); + this.id = id; + return this; + } + + public Builder setExpression(String expr) { + this.expr = expr; + return this; + } + + public Builder setJoin(Join join) { + this.join = join; + return this; + } + + public Builder setFillPolicy(NumericFillPolicy fill_policy) { + this.fillPolicy = fill_policy; + return this; + } + + public Expression build() { + return new Expression(this); + } + } +} diff --git a/src/query/pojo/Filter.java b/src/query/pojo/Filter.java new file mode 100644 index 0000000000..c1eeea2818 --- /dev/null +++ b/src/query/pojo/Filter.java @@ -0,0 +1,134 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.pojo; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; +import com.google.common.base.Objects; + +import net.opentsdb.query.filter.TagVFilter; + +import java.util.List; + +/** + * Pojo builder class used for serdes of a filter component of a query + * @since 2.3 + */ +@JsonDeserialize(builder = Filter.Builder.class) +public class Filter extends Validatable { + /** The id of the filter set to use in a metric query */ + private String id; + + /** The list of filters in the filter set */ + private List tags; + + /** Whether or not to only fetch series with exactly the same tag keys as + * in the filter list. */ + private boolean explicit_tags; + + /** + * Default ctor + * @param builder The builder to pull values from + */ + private Filter(Builder builder) { + this.id = builder.id; + this.tags = builder.tags; + this.explicit_tags = builder.explicitTags; + } + + /** @return the id of the filter set to use in a metric query */ + public String getId() { + return id; + } + + /** @return the list of filters in the filter set */ + public List getTags() { + return tags; + } + + /** @return Whether or not to only fetch series with exactly the same tag keys as + * in the filter list. */ + public boolean getExplicitTags() { + return explicit_tags; + } + + /** @return A new builder for the filter */ + public static Builder Builder() { + return new Builder(); + } + + /** Validates the filter set + * @throws IllegalArgumentException if one or more parameters were invalid + */ + public void validate() { + if (id == null || id.isEmpty()) { + throw new IllegalArgumentException("Missing or empty id"); + } + Query.validateId(id); + } + + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + final Filter filter = (Filter) o; + + return Objects.equal(id, filter.id) + && Objects.equal(tags, filter.tags) + && Objects.equal(explicit_tags, filter.explicit_tags); + } + + @Override + public int hashCode() { + return Objects.hashCode(id, tags, explicit_tags); + } + + /** + * A builder for the downsampler component of a query + */ + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonPOJOBuilder(buildMethodName = "build", withPrefix = "") + public static final class Builder { + @JsonProperty + private String id; + @JsonProperty + private List tags; + @JsonProperty + private boolean explicitTags; + + public Builder setId(String id) { + Query.validateId(id); + this.id = id; + return this; + } + + public Builder setTags(List tags) { + this.tags = tags; + return this; + } + + public Builder setExplicitTags(boolean explicit_tags) { + this.explicitTags = explicit_tags; + return this; + } + + public Filter build() { + return new Filter(this); + } + } +} diff --git a/src/query/pojo/Join.java b/src/query/pojo/Join.java new file mode 100644 index 0000000000..ea5c228fa3 --- /dev/null +++ b/src/query/pojo/Join.java @@ -0,0 +1,132 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.pojo; + +import net.opentsdb.query.expression.VariableIterator.SetOperator; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; +import com.google.common.base.Objects; + +/** + * Pojo builder class used for serdes of the join component of a query + * @since 2.3 + */ +@JsonDeserialize(builder = Join.Builder.class) +public class Join extends Validatable { + /** The set operator to use for joining sets */ + private SetOperator operator; + + /** Whether or not to use the original query tags instead of the resulting + * series tags when joining. */ + private boolean use_query_tags = false; + + /** Whether or not to use the aggregated tags in the results when joining. */ + private boolean include_agg_tags = true; + + /** + * Default ctor + * @param builder The builder to pull values from + */ + public Join(final Builder builder) { + operator = builder.operator; + use_query_tags = builder.useQueryTags; + include_agg_tags = builder.includeAggTags; + } + + /** @return the set operator to use for joining sets */ + public SetOperator getOperator() { + return operator; + } + + /** @return whether or not to use the original query tags instead of the + * resulting series tags when joining. */ + public boolean getUseQueryTags() { + return use_query_tags; + } + + /** @return Whether or not to use the aggregated tags in the results + * when joining. */ + public boolean getIncludeAggTags() { + return include_agg_tags; + } + + /** @return A new builder for the joiner */ + public static Builder Builder() { + return new Builder(); + } + + /** Validates the joiner + * @throws IllegalArgumentException if one or more parameters were invalid + */ + @Override + public void validate() { + if (operator == null) { + throw new IllegalArgumentException("Missing join operator"); + } + } + + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + final Join join = (Join) o; + + return Objects.equal(operator, join.operator) + && Objects.equal(use_query_tags, join.use_query_tags) + && Objects.equal(include_agg_tags, join.include_agg_tags); + } + + @Override + public int hashCode() { + return Objects.hashCode(operator, use_query_tags, include_agg_tags); + } + + /** + * A builder for the downsampler component of a query + */ + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonPOJOBuilder(buildMethodName = "build", withPrefix = "") + public static final class Builder { + @JsonProperty + private SetOperator operator; + @JsonProperty + private boolean useQueryTags = false; + @JsonProperty + private boolean includeAggTags = true; + + public Builder setOperator(final SetOperator operator) { + this.operator = operator; + return this; + } + + public Builder setUseQueryTags(final boolean use_query_tags) { + this.useQueryTags = use_query_tags; + return this; + } + + public Builder setIncludeAggTags(final boolean include_agg_tags) { + this.includeAggTags = include_agg_tags; + return this; + } + + public Join build() { + return new Join(this); + } + } +} diff --git a/src/query/pojo/Metric.java b/src/query/pojo/Metric.java new file mode 100644 index 0000000000..a5e85e0d5d --- /dev/null +++ b/src/query/pojo/Metric.java @@ -0,0 +1,206 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.pojo; + +import java.util.NoSuchElementException; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; +import com.google.common.base.Objects; + +import net.opentsdb.core.Aggregators; +import net.opentsdb.query.expression.NumericFillPolicy; +import net.opentsdb.utils.DateTime; + +/** + * Pojo builder class used for serdes of a metric component of a query + * @since 2.3 + */ +@JsonDeserialize(builder = Metric.Builder.class) +public class Metric extends Validatable { + /** The name of the metric */ + private String metric; + + /** An ID for the metric */ + private String id; + + /** The ID of a filter set */ + private String filter; + + /** An optional time offset for time over time expressions */ + private String time_offset; + + /** An optional aggregation override for the metric */ + private String aggregator; + + /** A fill policy for dealing with missing values in the metric */ + private NumericFillPolicy fill_policy; + + /** + * Default ctor + * @param builder The builder to pull values from + */ + public Metric(Builder builder) { + metric = builder.metric; + id = builder.id; + filter = builder.filter; + time_offset = builder.timeOffset; + aggregator = builder.aggregator; + fill_policy = builder.fillPolicy; + } + + /** @return the name of the metric */ + public String getMetric() { + return metric; + } + + /** @return an ID for the metric */ + public String getId() { + return id; + } + + /** @return the ID of a filter set */ + public String getFilter() { + return filter; + } + + /** @return an optional time offset for time over time expressions */ + public String getTimeOffset() { + return time_offset; + } + + /** @return an optional aggregation override for the metric */ + public String getAggregator() { + return aggregator; + } + + /** @return a fill policy for dealing with missing values in the metric */ + public NumericFillPolicy getFillPolicy() { + return fill_policy; + } + + /** @return A new builder for the metric */ + public static Builder Builder() { + return new Builder(); + } + + /** Validates the metric + * @throws IllegalArgumentException if one or more parameters were invalid + */ + public void validate() { + if (metric == null || metric.isEmpty()) { + throw new IllegalArgumentException("missing or empty metric"); + } + + if (id == null || id.isEmpty()) { + throw new IllegalArgumentException("missing or empty id"); + } + Query.validateId(id); + + if (time_offset != null) { + DateTime.parseDateTimeString(time_offset, null); + } + + if (aggregator != null && !aggregator.isEmpty()) { + try { + Aggregators.get(aggregator.toLowerCase()); + } catch (final NoSuchElementException e) { + throw new IllegalArgumentException("Invalid aggregator"); + } + } + + if (fill_policy != null) { + fill_policy.validate(); + } + } + + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + final Metric that = (Metric) o; + + return Objects.equal(that.filter, filter) + && Objects.equal(that.id, id) + && Objects.equal(that.metric, metric) + && Objects.equal(that.time_offset, time_offset) + && Objects.equal(that.aggregator, aggregator) + && Objects.equal(that.fill_policy, fill_policy); + } + + @Override + public int hashCode() { + return Objects.hashCode(metric, id, filter, time_offset, aggregator, + fill_policy); + } + + /** + * A builder for a metric component of a query + */ + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonPOJOBuilder(buildMethodName = "build", withPrefix = "") + public static final class Builder { + @JsonProperty + private String metric; + @JsonProperty + private String id; + @JsonProperty + private String filter; + @JsonProperty + private String timeOffset; + @JsonProperty + private String aggregator; + @JsonProperty + private NumericFillPolicy fillPolicy; + + public Builder setMetric(String metric) { + this.metric = metric; + return this; + } + + public Builder setId(String id) { + Query.validateId(id); + this.id = id; + return this; + } + + public Builder setFilter(String filter) { + this.filter = filter; + return this; + } + + public Builder setTimeOffset(String time_offset) { + this.timeOffset = time_offset; + return this; + } + + public Builder setAggregator(String aggregator) { + this.aggregator = aggregator; + return this; + } + + public Builder setFillPolicy(NumericFillPolicy fill_policy) { + this.fillPolicy = fill_policy; + return this; + } + + public Metric build() { + return new Metric(this); + } + } +} diff --git a/src/query/pojo/Output.java b/src/query/pojo/Output.java new file mode 100644 index 0000000000..8b996cf9ab --- /dev/null +++ b/src/query/pojo/Output.java @@ -0,0 +1,117 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.pojo; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; +import com.google.common.base.Objects; + +/** + * Pojo builder class used for serdes of the output component of a query + * @since 2.3 + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonDeserialize(builder = Output.Builder.class) +public class Output extends Validatable { + /** The ID of a metric or expression to emit */ + private String id; + + /** An alias to use as the metric name for the output */ + private String alias; + + /** + * Default ctor + * @param builder The builder to pull values from + */ + public Output(Builder builder) { + this.id = builder.id; + this.alias = builder.alias; + } + + /** @return the ID of a metric or expression to emit */ + public String getId() { + return id; + } + + /** @return an alias to use as the metric name for the output */ + public String getAlias() { + return alias; + } + + /** @return A new builder for the output */ + public static Builder Builder() { + return new Builder(); + } + + /** Validates the output + * @throws IllegalArgumentException if one or more parameters were invalid + */ + @Override public void validate() { + if (id == null || id.isEmpty()) { + throw new IllegalArgumentException("missing or empty id"); + } + Query.validateId(id); + } + + @Override + public String toString() { + return "var=" + id + ", alias=" + alias; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + Output output = (Output) o; + + return Objects.equal(output.alias, alias) + && Objects.equal(output.id, id); + } + + @Override + public int hashCode() { + return Objects.hashCode(id, alias); + } + + /** + * A builder for the downsampler component of a query + */ + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonPOJOBuilder(buildMethodName = "build", withPrefix = "") + public static final class Builder { + @JsonProperty + private String id; + @JsonProperty + private String alias; + + public Builder setId(String id) { + Query.validateId(id); + this.id = id; + return this; + } + + public Builder setAlias(String alias) { + this.alias = alias; + return this; + } + + public Output build() { + return new Output(this); + } + } +} diff --git a/src/query/pojo/Query.java b/src/query/pojo/Query.java new file mode 100644 index 0000000000..d00201bacc --- /dev/null +++ b/src/query/pojo/Query.java @@ -0,0 +1,300 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.pojo; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; +import com.google.common.base.Objects; + +import net.opentsdb.utils.JSON; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Pojo builder class used for serdes of the expression query + * @since 2.3 + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonDeserialize(builder = Query.Builder.class) +public class Query extends Validatable { + /** An optional name for the query */ + private String name; + + /** The timespan component of the query */ + private Timespan time; + + /** A list of filters */ + private List filters; + + /** A list of metrics */ + private List metrics; + + /** A list of expressions */ + private List expressions; + + /** A list of outputs */ + private List outputs; + + /** + * Default ctor + * @param builder The builder to pull values from + */ + public Query(Builder builder) { + this.name = builder.name; + this.time = builder.time; + this.filters = builder.filters; + this.metrics = builder.metrics; + this.expressions = builder.expressions; + this.outputs = builder.outputs; + } + + /** @return an optional name for the query */ + public String getName() { + return name; + } + + /** @return the timespan component of the query */ + public Timespan getTime() { + return time; + } + + /** @return a list of filters */ + public List getFilters() { + return filters; + } + + /** @return a list of metrics */ + public List getMetrics() { + return metrics; + } + + /** @return a list of expressions */ + public List getExpressions() { + return expressions; + } + + /** @return a list of outputs */ + public List getOutputs() { + return outputs; + } + + /** @return A new builder for the query */ + public static Builder Builder() { + return new Builder(); + } + + /** Validates the query + * @throws IllegalArgumentException if one or more parameters were invalid + */ + public void validate() { + if (time == null) { + throw new IllegalArgumentException("missing time"); + } + + validatePOJO(time, "time"); + + if (metrics == null || metrics.isEmpty()) { + throw new IllegalArgumentException("missing or empty metrics"); + } + + final Set variable_ids = new HashSet(); + for (Metric metric : metrics) { + if (variable_ids.contains(metric.getId())) { + throw new IllegalArgumentException("duplicated metric id: " + + metric.getId()); + } + variable_ids.add(metric.getId()); + } + + final Set filter_ids = new HashSet(); + + if (filters != null) { + for (Filter filter : filters) { + if (filter_ids.contains(filter.getId())) { + throw new IllegalArgumentException("duplicated filter id: " + + filter.getId()); + } + filter_ids.add(filter.getId()); + } + } + + if (expressions != null) { + for (Expression expression : expressions) { + if (variable_ids.contains(expression.getId())) { + throw new IllegalArgumentException("Duplicated variable or expression id: " + + expression.getId()); + } + variable_ids.add(expression.getId()); + } + } + + validateCollection(metrics, "metric"); + + if (filters != null) { + validateCollection(filters, "filter"); + } + + if (expressions != null) { + validateCollection(expressions, "expression"); + } + + validateFilters(); + + if (expressions != null) { + validateCollection(expressions, "expression"); + for (final Expression exp : expressions) { + if (exp.getVariables() == null) { + throw new IllegalArgumentException("No variables found for an " + + "expression?! " + JSON.serializeToString(exp)); + } + + for (final String var : exp.getVariables()) { + if (!variable_ids.contains(var)) { + throw new IllegalArgumentException("Expression [" + exp.getExpr() + + "] was missing input " + var); + } + } + } + } + } + + /** Validates the filters, making sure each metric has a filter + * @throws IllegalArgumentException if one or more parameters were invalid + */ + private void validateFilters() { + if (filters == null) { + return; + } + + Set ids = new HashSet(); + for (Filter filter : filters) { + ids.add(filter.getId()); + } + + for(Metric metric : metrics) { + if (metric.getFilter() != null && + !metric.getFilter().isEmpty() && + !ids.contains(metric.getFilter())) { + throw new IllegalArgumentException( + String.format("unrecognized filter id %s in metric %s", + metric.getFilter(), metric.getId())); + } + } + } + + /** + * Makes sure the ID has only letters and characters + * @param id The ID to parse + * @throws IllegalArgumentException if the ID is invalid + */ + public static void validateId(final String id) { + if (id == null || id.isEmpty()) { + throw new IllegalArgumentException("The ID cannot be null or empty"); + } + for (int i = 0; i < id.length(); i++) { + final char c = id.charAt(i); + if (!(Character.isLetterOrDigit(c))) { + throw new IllegalArgumentException("Invalid id (\"" + id + + "\"): illegal character: " + c); + } + } + if (id.length() == 1) { + if (Character.isDigit(id.charAt(0))) { + throw new IllegalArgumentException("The ID cannot be an integer"); + } + } + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + Query query = (Query) o; + + return Objects.equal(query.expressions, expressions) + && Objects.equal(query.filters, filters) + && Objects.equal(query.metrics, metrics) + && Objects.equal(query.name, name) + && Objects.equal(query.outputs, outputs) + && Objects.equal(query.time, time); + } + + @Override + public int hashCode() { + return Objects.hashCode(name, time, filters, metrics, expressions, outputs); + } + + /** + * A builder for the query component of a query + */ + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonPOJOBuilder(buildMethodName = "build", withPrefix = "") + public static final class Builder { + @JsonProperty + private String name; + @JsonProperty + private Timespan time; + @JsonProperty + private List filters; + @JsonProperty + private List metrics; + @JsonProperty + private List expressions; + @JsonProperty + private List outputs; + + public Builder() { } + + public Builder setName(final String name) { + this.name = name; + return this; + } + + public Builder setTime(final Timespan time) { + this.time = time; + return this; + } + + public Builder setFilters(final List filters) { + this.filters = filters; + return this; + } + + public Builder setMetrics(final List metrics) { + this.metrics = metrics; + return this; + } + + public Builder setExpressions(final List expressions) { + this.expressions = expressions; + return this; + } + + public Builder setOutputs(final List outputs) { + this.outputs = outputs; + return this; + } + + public Query build() { + return new Query(this); + } + } + +} diff --git a/src/query/pojo/Timespan.java b/src/query/pojo/Timespan.java new file mode 100644 index 0000000000..96e56ba255 --- /dev/null +++ b/src/query/pojo/Timespan.java @@ -0,0 +1,208 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.pojo; + +import java.util.NoSuchElementException; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; +import com.google.common.base.Objects; + +import net.opentsdb.core.Aggregators; +import net.opentsdb.utils.DateTime; + +/** + * Pojo builder class used for serdes of the timespan component of a query + * @since 2.3 + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonDeserialize(builder = Timespan.Builder.class) +public class Timespan extends Validatable { + /** User given start date/time, could be relative or absolute */ + private String start; + + /** User given end date/time, could be relative, absolute or empty */ + private String end; + + /** User's timezone used for converting absolute human readable dates */ + private String timezone; + + /** An optional downsampler for all queries */ + private Downsampler downsampler; + + /** The global aggregator to use */ + private String aggregator; + + /** Whether or not to compute a rate */ + private boolean rate; + + /** + * Default ctor + * @param builder The builder to pull values from + */ + public Timespan(Builder builder) { + start = builder.start; + end = builder.end; + timezone = builder.timezone; + downsampler = builder.downsampler; + aggregator = builder.aggregator; + rate = builder.rate; + } + + /** @return user given start date/time, could be relative or absolute */ + public String getStart() { + return start; + } + + /** @return user given end date/time, could be relative, absolute or empty */ + public String getEnd() { + return end; + } + + /** @return user's timezone used for converting absolute human readable dates */ + public String getTimezone() { + return timezone; + } + + /** @return an optional downsampler for all queries */ + public Downsampler getDownsampler() { + return downsampler; + } + + /** @return the global aggregator to use */ + public String getAggregator() { + return aggregator; + } + + /** @return whether or not to compute a rate */ + public boolean isRate() { + return rate; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + Timespan timespan = (Timespan) o; + + return Objects.equal(timespan.downsampler, downsampler) + && Objects.equal(timespan.end, end) + && Objects.equal(timespan.start, start) + && Objects.equal(timespan.timezone, timezone) + && Objects.equal(timespan.aggregator, aggregator) + && Objects.equal(timespan.rate, rate); + } + + @Override + public int hashCode() { + return Objects.hashCode(start, end, timezone, downsampler, aggregator, rate); + } + + /** @return A new builder for the downsampler */ + public static Builder Builder() { + return new Builder(); + } + + /** Validates the timespan + * @throws IllegalArgumentException if one or more parameters were invalid + */ + public void validate() { + if (start == null || start.isEmpty()) { + throw new IllegalArgumentException("missing or empty start"); + } + DateTime.parseDateTimeString(start, timezone); + + if (end != null && !end.isEmpty()) { + DateTime.parseDateTimeString(end, timezone); + } + + if (downsampler != null) { + downsampler.validate(); + } + + if (aggregator == null || aggregator.isEmpty()) { + throw new IllegalArgumentException("Missing or empty aggregator"); + } + + try { + Aggregators.get(aggregator.toLowerCase()); + } catch (final NoSuchElementException e) { + throw new IllegalArgumentException("Invalid aggregator"); + } + } + + /** + * A builder for the downsampler component of a query + */ + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonPOJOBuilder(buildMethodName = "build", withPrefix = "") + public static final class Builder { + @JsonProperty + private String start; + + @JsonProperty + private String end; + + @JsonProperty + private String timezone; + + @JsonProperty + private Downsampler downsampler; + + @JsonProperty + private String aggregator; + + @JsonProperty + private boolean rate; + + public Builder setStart(final String start) { + this.start = start; + return this; + } + + public Builder setEnd(final String end) { + this.end = end; + return this; + } + + public Builder setTimezone(final String timezone) { + this.timezone = timezone; + return this; + } + + public Builder setDownsampler(final Downsampler downsample) { + this.downsampler = downsample; + return this; + } + + public Builder setAggregator(final String aggregator) { + this.aggregator = aggregator; + return this; + } + + public Builder setRate(final boolean rate) { + this.rate = rate; + return this; + } + + public Timespan build() { + return new Timespan(this); + } + } + +} diff --git a/src/query/pojo/Validatable.java b/src/query/pojo/Validatable.java new file mode 100644 index 0000000000..77b53af650 --- /dev/null +++ b/src/query/pojo/Validatable.java @@ -0,0 +1,59 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query.pojo; + +import java.util.Collection; +import java.util.Iterator; + +/** + * An interface for the pojos to implement to make sure all the bits of the + * expression queries are there + * @since 2.3 + */ +public abstract class Validatable { + abstract public void validate(); + + /** + * Iterate through a field that is a collection of POJOs and validate each of + * them. Inherit member POJO's error message. + * @param collection the validatable POJO collection + * @param name name of the field + */ + void validateCollection(final Collection collection, + final String name) { + Iterator iterator = collection.iterator(); + int i = 0; + while (iterator.hasNext()) { + try { + iterator.next().validate(); + } catch (final IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid " + name + + " at index " + i, e); + } + i++; + } + } + + /** + * Validate a single POJO validate + * @param pojo The POJO object to validate + * @param name name of the field + */ + void validatePOJO(final T pojo, final String name) { + try { + pojo.validate(); + } catch (final IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid " + name, e); + } + } +} diff --git a/src/rollup/NoSuchRollupForIntervalException.java b/src/rollup/NoSuchRollupForIntervalException.java new file mode 100644 index 0000000000..d698ebec99 --- /dev/null +++ b/src/rollup/NoSuchRollupForIntervalException.java @@ -0,0 +1,34 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2010-2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.rollup; + +import java.util.NoSuchElementException; + +/** + * Exception thrown when a rollup couldn't be found in the interval to rollup + * interval map + * @since 2.4 + */ +public class NoSuchRollupForIntervalException extends NoSuchElementException { + + /** + * Ctor that builds the message based on a string interval lookup + * @param interval The interval, e.g. "1m" + */ + public NoSuchRollupForIntervalException(final String interval) { + super("No rollups configured for the interval: " + interval); + } + + private static final long serialVersionUID = -8225702079229243161L; + +} diff --git a/src/rollup/NoSuchRollupForTableException.java b/src/rollup/NoSuchRollupForTableException.java new file mode 100644 index 0000000000..ee2b6148a3 --- /dev/null +++ b/src/rollup/NoSuchRollupForTableException.java @@ -0,0 +1,34 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2010-2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.rollup; + +import java.util.NoSuchElementException; + +/** + * Exception thrown when a rollup couldn't be found in the table name to rollup + * interval map + * @since 2.4 + */ +public class NoSuchRollupForTableException extends NoSuchElementException { + + /** + * Ctor that builds the message based on a string table lookup + * @param table The table name + */ + public NoSuchRollupForTableException(final String table) { + super("No rollups configured for the table: " + table); + } + + private static final long serialVersionUID = 6620255176637863260L; + +} diff --git a/src/rollup/RollUpDataPoint.java b/src/rollup/RollUpDataPoint.java new file mode 100644 index 0000000000..39f816cdca --- /dev/null +++ b/src/rollup/RollUpDataPoint.java @@ -0,0 +1,133 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.rollup; + +import com.fasterxml.jackson.annotation.JsonInclude; + +import net.opentsdb.core.Aggregators; +import net.opentsdb.core.IncomingDataPoint; + +import java.util.List; +import java.util.Map; + +/** + * Represents a single rolled up data point. It overrides the + * {@link IncomingDataPoint} class. + * @since 2.4 + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public class RollUpDataPoint extends IncomingDataPoint { + + /** The interval in the format <#> such as 1m or 2h */ + private String interval; + + /** The name of the aggregator that created this data point */ + private String aggregator; + + /** Optional aggregation function if this was a pre-aggregated data point. */ + protected String groupby_aggregator; + + /** + * Default Ctor necessary for de/serialization + */ + public RollUpDataPoint() { + } + + @Override + public String toString() { + final StringBuilder buf = new StringBuilder(); + buf.append("metric=").append(metric) + .append(" ts=").append(this.timestamp) + .append(" value=").append(this.value).append(" "); + if (this.tags != null) { + for (Map.Entry entry : this.tags.entrySet()) { + buf.append(entry.getKey()).append("=").append(entry.getValue()); + } + } + buf.append(" groupByAggregator=") + .append(groupby_aggregator) + .append(" interval=").append(interval) + .append(" aggregator=").append(aggregator); + return buf.toString(); + } + + /** @return the interval as a string */ + public String getInterval() { + return interval; + } + + /** @param interval The interval for this data point such as "1m" */ + public void setInterval(final String interval) { + this.interval = interval; + } + + /** @return the name of the aggregator used to generate this data point */ + public String getAggregator() { + return aggregator; + } + + /** @param aggregator The name of the aggregator used to generate this dp */ + public void setAggregator(final String aggregator) { + this.aggregator = aggregator; + } + + /** @return If pre-aggregated, the function used. May be null. */ + public final String getGroupByAggregator() { + return groupby_aggregator; + } + + /** @param groupby_aggregator an optional aggregation function if the data + * point was pre-aggregated */ + public final void setGroupByAggregator(final String groupby_aggregator) { + this.groupby_aggregator = groupby_aggregator; + } + + @Override + public boolean validate(final List> details) { + if (!super.validate(details)) + return false; + + boolean is_groupby = false; + if (groupby_aggregator != null && !groupby_aggregator.isEmpty()) { + // Don't need to perform this check here as the addAggregatePoint() + // will handle that validation for us. + //Aggregators.get(groupby_aggregator.toLowerCase()); + is_groupby = true; + } + + // interval is only required if the the group by is NOT set + if (interval == null || interval.isEmpty()) { + if (!is_groupby) { + if (details != null) { + details.add(getHttpDetails("Missing interval")); + } + return false; + } + } + + if (aggregator == null || aggregator.isEmpty()) { + if (!is_groupby) { + // only error out if the groupby is false. + if (details != null) { + details.add(getHttpDetails("Missing aggregator")); + } + return false; + } + // Don't need to perform this check here as the addAggregatePoint() + // will handle that validation for us. + //Aggregators.get(aggregator); + } + + return true; + } +} diff --git a/src/rollup/RollupConfig.java b/src/rollup/RollupConfig.java new file mode 100644 index 0000000000..f71824e178 --- /dev/null +++ b/src/rollup/RollupConfig.java @@ -0,0 +1,349 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.rollup; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import net.opentsdb.core.Aggregators; +import net.opentsdb.core.TSDB; +import net.opentsdb.utils.JSON; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.stumbleupon.async.Deferred; +import com.stumbleupon.async.DeferredGroupException; +import java.util.TreeMap; + +/** + * A class that contains the runtime configuration for a TSD's raw and rollup + * tables. + * + * Each rollup requires two table names for writing, an interval and a span. + * temporal_table - The table name for raw, temporal only rollup data + * groupby_table - The table name for pre-aggregated and rolled up data + * interval - A interval that the rollup is aligned on, e.g. 10 minute intervals + * would be denoted as "10m" and hourly would be "1h". + * span - A time unit describing the width of the row, or how many data points + * could be in it. E.g 'h' would mean the row holds an hour of data while + * 'y' holds a full year. Possible values are: + * 'h' = hour + * 'd' = day + * 'n' = month + * 'y' = year + * + * @since 2.4 + */ +@JsonDeserialize(builder = RollupConfig.Builder.class) +public class RollupConfig { + private static final Logger LOG = LoggerFactory.getLogger(RollupConfig.class); + + /** The interval to interval map where keys are things like "10m" or "1d"*/ + protected final Map forward_intervals; + + /** The table name to interval map for queries */ + protected final Map reverse_intervals; + + /** The map of IDs to aggregators for use at query time. */ + protected final Map ids_to_aggregations; + + /** The map of aggregators to IDs for use at write time. */ + protected final Map aggregations_to_ids; + + /** + * Default ctor for the builder. + * @param builder A non-null builder to load from. + */ + protected RollupConfig(final Builder builder) { + forward_intervals = Maps.newHashMapWithExpectedSize(2); + reverse_intervals = Maps.newHashMapWithExpectedSize(2); + ids_to_aggregations = Maps.newHashMapWithExpectedSize(4); + aggregations_to_ids = Maps.newHashMapWithExpectedSize(4); + + if (builder.intervals == null || builder.intervals.isEmpty()) { + throw new IllegalArgumentException("Rollup config given but no intervals " + + "were found."); + } + if (builder.aggregationIds == null || builder.aggregationIds.isEmpty()) { + throw new IllegalArgumentException("Rollup config given but no aggegation " + + "ID mappings found."); + } + int defaults = 0; + for (final RollupInterval config_interval : builder.intervals) { + if (forward_intervals.containsKey(config_interval.getInterval())) { + throw new IllegalArgumentException( + "Only one interval of each type can be configured: " + + config_interval); + } + if (config_interval.isDefaultInterval() && defaults++ >= 1) { + throw new IllegalArgumentException("Multiple default intervals " + + "configured. Only one is allowed: " + config_interval); + } + + forward_intervals.put(config_interval.getInterval(), config_interval); + reverse_intervals.put(config_interval.getTable(), config_interval); + reverse_intervals.put(config_interval.getPreAggregationTable(), + config_interval); + LOG.info("Loaded rollup interval: " + config_interval); + } + + for (final Entry entry : builder.aggregationIds.entrySet()) { + if (entry.getValue() < 0 || entry.getValue() > 127) { + throw new IllegalArgumentException("ID for aggregator must be between " + + "0 and 127: " + entry); + } + final String agg = entry.getKey().toLowerCase(); + if (ids_to_aggregations.containsKey(entry.getValue())) { + throw new IllegalArgumentException("Multiple mappings for the " + + "ID '" + entry.getValue() + "' are not allowed."); + } + if (Aggregators.get(agg) == null) { + throw new IllegalArgumentException("No such aggregator found for " + agg); + } + aggregations_to_ids.put(agg, entry.getValue()); + ids_to_aggregations.put(entry.getValue(), agg); + LOG.info("Mapping aggregator '" + agg + "' to ID " + entry.getValue()); + } + + LOG.info("Configured [" + forward_intervals.size() + "] rollup intervals"); + } + + /** + * Fetches the RollupInterval corresponding to the forward interval string map + * @param interval The interval to lookup + * @return The RollupInterval object configured for the given interval + * @throws IllegalArgumentException if the interval is null or empty + * @throws NoSuchRollupForIntervalException if the interval was not configured + */ + public RollupInterval getRollupInterval(final String interval) { + if (interval == null || interval.isEmpty()) { + throw new IllegalArgumentException("Interval cannot be null or empty"); + } + final RollupInterval rollup = forward_intervals.get(interval); + if (rollup == null) { + throw new NoSuchRollupForIntervalException(interval); + } + return rollup; + } + + /** + * Fetches the RollupInterval corresponding to the integer interval in seconds. + * It returns a list of matching RollupInterval and best next matches in the + * order. It will help to search on the next best rollup tables. + * It is guaranteed that it return a non-empty list + * For example if the interval is 1 day + * then it may return RollupInterval objects in the order + * 1 day, 1 hour, 10 minutes, 1 minute + * @param interval The interval in seconds to lookup + * @param str_interval String representation of the interval, for logging + * @return The RollupInterval object configured for the given interval + * @throws IllegalArgumentException if the interval is null or empty + * @throws NoSuchRollupForIntervalException if the interval was not configured + */ + public List getRollupInterval(final long interval, + final String str_interval) { + + if (interval <= 0) { + throw new IllegalArgumentException("Interval cannot be null or empty"); + } + + final Map rollups = + new TreeMap(Collections.reverseOrder()); + boolean right_match = false; + + for (RollupInterval rollup: forward_intervals.values()) { + if (rollup.getIntervalSeconds() == interval) { + rollups.put((long) rollup.getIntervalSeconds(), rollup); + right_match = true; + } + else if (interval % rollup.getIntervalSeconds() == 0) { + rollups.put((long) rollup.getIntervalSeconds(), rollup); + } + } + + if (rollups.isEmpty()) { + throw new NoSuchRollupForIntervalException(Long.toString(interval)); + } + + List best_matches = + new ArrayList(rollups.values()); + + if (!right_match) { + LOG.warn("No such rollup interval found, " + str_interval + ". So falling " + + "back to the next best match " + best_matches.get(0). + getInterval()); + } + + return best_matches; + } + + /** + * Fetches the RollupInterval corresponding to the rollup or pre-agg table + * name. + * @param table The name of the table to fetch + * @return The RollupInterval object matching the table + * @throws IllegalArgumentException if the table is null or empty + * @throws NoSuchRollupForTableException if the interval was not configured + * for the given table + */ + public RollupInterval getRollupIntervalForTable(final String table) { + if (table == null || table.isEmpty()) { + throw new IllegalArgumentException("The table name cannot be null or empty"); + } + final RollupInterval rollup = reverse_intervals.get(table); + if (rollup == null) { + throw new NoSuchRollupForTableException(table); + } + return rollup; + } + + /** + * Makes sure each of the rollup tables exists + * @param tsdb The TSDB to use for fetching the HBase client + */ + public void ensureTablesExist(final TSDB tsdb) { + final List> deferreds = + new ArrayList>(forward_intervals.size() * 2); + + for (RollupInterval interval : forward_intervals.values()) { + deferreds.add(tsdb.getClient() + .ensureTableExists(interval.getTemporalTable())); + deferreds.add(tsdb.getClient() + .ensureTableExists(interval.getGroupbyTable())); + } + + try { + Deferred.group(deferreds).joinUninterruptibly(); + } catch (DeferredGroupException e) { + throw new RuntimeException(e.getCause()); + } catch (InterruptedException e) { + LOG.warn("Interrupted", e); + Thread.currentThread().interrupt(); + } catch (Exception e) { + throw new RuntimeException("Unexpected exception", e); + } + } + + /** @return an unmodifiable map of the rollups for printing and debugging */ + @JsonIgnore + public Map getRollups() { + return Collections.unmodifiableMap(forward_intervals); + } + + /** @return The immutable list of rollup intervals for serialization. */ + public List getIntervals() { + return Lists.newArrayList(forward_intervals.values()); + } + + /** @return The immutable map of aggregations to IDs for serialization. */ + public Map getAggregationIds() { + return Collections.unmodifiableMap(aggregations_to_ids); + } + + /** + * @param id The ID of an aggregator to search for. + * @return The aggregator if found, null if it was not mapped. + */ + public String getAggregatorForId(final int id) { + return ids_to_aggregations.get(id); + } + + /** + * @param aggregator The non-null and non-empty aggregator to search for. + * @return The ID of the aggregator if found. + * @throws IllegalArgumentException if the aggregator was not found or if the + * aggregator was null or empty. + */ + public int getIdForAggregator(final String aggregator) { + if (Strings.isNullOrEmpty(aggregator)) { + throw new IllegalArgumentException("Aggregator cannot be null or empty."); + } + Integer id = aggregations_to_ids.get(aggregator.toLowerCase()); + if (id == null) { + throw new IllegalArgumentException("No ID found mapping to aggregator: " + + aggregator); + } + return id; + } + + @Override + public String toString() { + return JSON.serializeToString(this); + } + + public static Builder builder() { + return new Builder(); + } + + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonPOJOBuilder(buildMethodName = "build", withPrefix = "") + public static class Builder { + @JsonProperty + private Map aggregationIds; + @JsonProperty + private List intervals; + + public Builder setAggregationIds(final Map aggregationIds) { + this.aggregationIds = aggregationIds; + return this; + } + + @JsonIgnore + public Builder addAggregationId(final String aggregation, final int id) { + if (aggregationIds == null) { + aggregationIds = Maps.newHashMapWithExpectedSize(1); + } + aggregationIds.put(aggregation, id); + return this; + } + + public Builder setIntervals(final List intervals) { + this.intervals = intervals; + return this; + } + + @JsonIgnore + public Builder addInterval(final RollupInterval interval) { + if (intervals == null) { + intervals = Lists.newArrayList(); + } + intervals.add(interval); + return this; + } + + @JsonIgnore + public Builder addInterval(final RollupInterval.Builder interval) { + if (intervals == null) { + intervals = Lists.newArrayList(); + } + intervals.add(interval.build()); + return this; + } + + public RollupConfig build() { + return new RollupConfig(this); + } + } +} diff --git a/src/rollup/RollupInterval.java b/src/rollup/RollupInterval.java new file mode 100644 index 0000000000..0ccedce6be --- /dev/null +++ b/src/rollup/RollupInterval.java @@ -0,0 +1,389 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.rollup; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; +import com.google.common.base.Objects; +import com.google.common.hash.HashCode; + +import net.opentsdb.core.Const; +import net.opentsdb.utils.DateTime; + +/** + * Holds information about a rollup interval. During construction the inputs + * are validated. + * @since 2.4 + */ +@JsonDeserialize(builder = RollupInterval.Builder.class) +public class RollupInterval { + /** Static intervals */ + private static final int MAX_SECONDS_IN_HOUR = 60 * 60; + private static final int MAX_SECONDS_IN_DAY = 60 * 60 * 24; + // account for leap years, etc + private static final int MAX_SECONDS_IN_MONTH = 60 * 60 * 24 * 32; + private static final int MAX_SECONDS_IN_YEAR = 60 * 60 * 24 * 366; + + /** Based on a 2 bytes with 4 bits reserved for data value size and type */ + public static int MAX_INTERVALS = 7774; + + /** The minimum # of intervals in a span */ + public static int MIN_INTERVALS = 12; + + /** User assigned name for the temporal only table */ + private final String temporal_table_name; + private byte[] temporal_table; + + /** User assigned name for the group by table for this interval */ + private final String groupby_table_name; + private byte[] groupby_table; + + /** User given interval as a string in the format <#> similar to a + * downsampling query + */ + private final String string_interval; + + /** How wide the row will be in values. */ + private final String row_span; + + /** Width of the row as a time unit, e.g. 'h' for hour, 'd' for day, 'm' for + * month and 'y' for year + */ + private final char units; + + /** How many of the units we want in our span */ + private final int unit_multiplier; + + /** Parsed interval unit from the user supplied string */ + private char interval_units; + + /** The interval in seconds */ + private int interval; + + /** The number of intervals in this span */ + private int intervals; + + /** Tells whether it is the default rollup interval + * Default interval is of 1m interval, and will be stored in normal + * tsdb table/s. So if true, which means the raw cell column qualifier format + * also it might be compacted. + */ + private final boolean is_default_interval; + + /** + * The delay SLA for this rollup interval. If a query is asking for data from a + * recent enough time interval that might not be available (or partially unavailable) + * in the table, the data points will be read from the raw table. + */ + private final String delay_sla; + + /** The delay SLA in seconds */ + private int max_delay_seconds; + + /** + * Protected ctor used by the builder. + * @param builder The non-null builder to load from. + */ + protected RollupInterval(final Builder builder) { + temporal_table_name = builder.table; + groupby_table_name = builder.preAggregationTable; + string_interval = builder.interval; + row_span = builder.rowSpan; + is_default_interval = builder.defaultInterval; + delay_sla = builder.delaySla != null ? builder.delaySla : ""; + + final String parsed_units = DateTime.getDurationUnits(row_span); + if (parsed_units.length() > 1) { + throw new IllegalArgumentException("Milliseconds are not supported"); + } + units = parsed_units.charAt(0); + this.unit_multiplier = DateTime.getDurationInterval(row_span); + + validateAndCompile(); + } + + @Override + public String toString() { + final StringBuilder buf = new StringBuilder(); + buf.append("table=").append(temporal_table_name) + .append(", preAggTable=").append(groupby_table_name) + .append(", rowSpan=").append(row_span) + .append(", isDefaultInterval=").append(is_default_interval) + .append(", interval=").append(string_interval) + .append(", units=").append(units) + .append(", unit_multipier=").append(unit_multiplier) + .append(", intervals=").append(intervals) + .append(", interval=").append(interval) + .append(", interval_units=").append(interval_units) + .append(", delay_sla=").append(delay_sla); + return buf.toString(); + } + + @Override + public int hashCode() { + return buildHashCode().asInt(); + } + + /** @return A HashCode object for deterministic, non-secure hashing */ + public HashCode buildHashCode() { + return Const.HASH_FUNCTION().newHasher() + .putString(temporal_table_name, Const.UTF8_CHARSET) + .putString(groupby_table_name, Const.UTF8_CHARSET) + .putString(string_interval, Const.UTF8_CHARSET) + .putString(row_span, Const.UTF8_CHARSET) + .putBoolean(is_default_interval) + .putString(delay_sla, Const.UTF8_CHARSET) + .hash(); + } + + @Override + public boolean equals(final Object obj) { + if (obj == null) { + return false; + } + if (!(obj instanceof RollupInterval)) { + return false; + } + if (obj == this) { + return true; + } + final RollupInterval interval = (RollupInterval)obj; + return Objects.equal(temporal_table_name, interval.temporal_table_name) + && Objects.equal(groupby_table_name, interval.groupby_table_name) + && Objects.equal(row_span, interval.row_span) + && Objects.equal(string_interval, interval.string_interval) + && Objects.equal(is_default_interval, interval.is_default_interval) + && Objects.equal(delay_sla, interval.delay_sla); + } + + /** + * Calculates the number of intervals in a given span for the rollup + * interval and makes sure we have table names. It also sets the table byte + * arrays. + * @return The number of intervals in the span + * @throws IllegalArgumentException if milliseconds were passed in the interval + * or the interval couldn't be parsed, the tables are missing, or if the + * duration is too large, too large for the span or the interval is too + * large or small for the span or if the span is invalid. + * @throws NullPointerException if the interval is empty or null + */ + void validateAndCompile() { + if (temporal_table_name == null || temporal_table_name.isEmpty()) { + throw new IllegalArgumentException("The rollup table cannot be null or empty"); + } + temporal_table = temporal_table_name.getBytes(Const.ASCII_CHARSET); + + if (groupby_table_name == null || groupby_table_name.isEmpty()) { + throw new IllegalArgumentException("The pre-aggregate rollup table cannot" + + " be null or empty"); + } + groupby_table = groupby_table_name.getBytes(Const.ASCII_CHARSET); + + if (units != 'h' && unit_multiplier > 1) { + throw new IllegalArgumentException("Multipliers are only usable with the 'h' unit"); + } else if (units == 'h' && unit_multiplier > 1 && unit_multiplier % 2 != 0) { + throw new IllegalArgumentException("The multiplier must be 1 or an even value"); + } + + interval = (int) (DateTime.parseDuration(string_interval) / 1000); + + if (interval >= Integer.MAX_VALUE) { + throw new IllegalArgumentException("Interval is too big: " + interval); + } + // The line above will validate for us + interval_units = string_interval.charAt(string_interval.length() - 1); + + if (delay_sla != null && !delay_sla.isEmpty()) { + max_delay_seconds = (int) (DateTime.parseDuration(delay_sla) / 1000); + if (max_delay_seconds < 1) { + throw new IllegalArgumentException("Milliseconds are not supported as the maximum delay"); + } + } + + int num_span = 0; + switch (units) { + case 'h': + num_span = MAX_SECONDS_IN_HOUR; + break; + case 'd': + num_span = MAX_SECONDS_IN_DAY; + break; + case 'n': + num_span = MAX_SECONDS_IN_MONTH; + break; + case 'y': + num_span = MAX_SECONDS_IN_YEAR; + break; + default: + throw new IllegalArgumentException("Unrecogznied span '" + units + "'"); + } + num_span *= unit_multiplier; + + if (interval >= num_span) { + throw new IllegalArgumentException("Interval [" + interval + + "] is too large for the span [" + units + "]"); + } + + intervals = num_span / (int)interval; + if (intervals > MAX_INTERVALS) { + throw new IllegalArgumentException("Too many intervals [" + intervals + + "] in the span. Must be smaller than [" + MAX_INTERVALS + + "] to fit in 14 bits"); + } + + if (intervals < MIN_INTERVALS) { + throw new IllegalArgumentException("Not enough intervals [" + intervals + + "] for the span. Must be at least [" + MIN_INTERVALS + "]"); + } + } + + /** @return the string name of the temporal rollup table */ + public String getTable() { + return temporal_table_name; + } + + /** @return the temporal rollup table name as a byte array */ + @JsonIgnore + public byte[] getTemporalTable() { + return temporal_table; + } + + /** @return the string name of the group by rollup table */ + public String getPreAggregationTable() { + return groupby_table_name; + } + + /** @return the group by table name as a byte array */ + @JsonIgnore + public byte[] getGroupbyTable() { + return groupby_table; + } + + /** @return the configured interval as a string */ + public String getInterval() { + return string_interval; + } + + /** @return the character describing the span of this interval */ + @JsonIgnore + public char getUnits() { + return units; + } + + /** @return the unit multiplier */ + @JsonIgnore + public int getUnitMultiplier() { + return unit_multiplier; + } + + /** @return the interval units character */ + @JsonIgnore + public char getIntervalUnits() { + return interval_units; + } + + /** @return the interval for this span in seconds */ + @JsonIgnore + public int getIntervalSeconds() { + return interval; + } + + /** @return the count of intervals in this span */ + @JsonIgnore + public int getIntervals() { + return intervals; + } + + /** + * Is it the default roll up interval that need to be written to default + * tsdb data table. So if true, which means the raw cell column qualifier + * is not encoded with the aggregate function and the cell might have been + * compacted + * @return true if it is default rollup interval + */ + public boolean isDefaultInterval() { + return is_default_interval; + } + + /** @return The width of each row as an interval string. */ + public String getRowSpan() { + return row_span; + } + + /** + * Rollup tables can have an SLA configured specifying by how much time the + * data in the table can be delayed. + * @return the maximum delay in seconds for a table as configured. + */ + public int getMaximumLag() { + return max_delay_seconds; + } + + public static Builder builder() { + return new Builder(); + } + + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonPOJOBuilder(buildMethodName = "build", withPrefix = "") + public static class Builder { + @JsonProperty + private String table; + @JsonProperty + private String preAggregationTable; + @JsonProperty + private String interval; + @JsonProperty + private String rowSpan; + @JsonProperty + private boolean defaultInterval; + @JsonProperty + private String delaySla; + + public Builder setTable(final String table) { + this.table = table; + return this; + } + + public Builder setPreAggregationTable(final String preAggregationTable) { + this.preAggregationTable = preAggregationTable; + return this; + } + + public Builder setInterval(final String interval) { + this.interval = interval; + return this; + } + + public Builder setRowSpan(final String rowSpan) { + this.rowSpan = rowSpan; + return this; + } + + public Builder setDefaultInterval(final boolean defaultInterval) { + this.defaultInterval = defaultInterval; + return this; + } + + public Builder setDelaySla(final String delaySla) { + this.delaySla = delaySla; + return this; + } + + public RollupInterval build() { + return new RollupInterval(this); + } + } +} diff --git a/src/rollup/RollupQuery.java b/src/rollup/RollupQuery.java new file mode 100644 index 0000000000..46d37cdce2 --- /dev/null +++ b/src/rollup/RollupQuery.java @@ -0,0 +1,211 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2010-2012 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.rollup; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.base.Objects; + +import net.opentsdb.core.Aggregator; +import net.opentsdb.core.Aggregators; +import net.opentsdb.utils.DateTime; + +/** + * Holds information about a rollup interval and rollup aggregator. + * Every RollupQuery object will have a valid RollupInterval and Aggregator + */ +public class RollupQuery { + + // TEMP + public static final byte[] SUM = new byte[] { 's', 'u', 'm' }; + public static final byte[] COUNT = new byte[] { 'c', 'o', 'u', 'n', 't' }; + + private final RollupInterval rollup_interval; + private final Aggregator rollup_agg; + private final Aggregator group_by; + + /** Rollup aggregate prefix along with the delimiter as byte array. It will be + * the same for the same rollup aggregator, but is defined here to + * reduce the number of calculations at scan time*/ + private final byte[] agg_prefix; + + /** Initial downsampling interval form the user, will be used to + * downsample the lower sampling rate, if data is not available for the + * requested sampling rate. It is in milliseconds*/ + private final long sample_interval_ms; + + /** + * Default constructor + * @param rollup_interval RollupInterval object + * @param rollup_agg Aggregator object + * @param sample_interval_ms Initial downsaple interval in milliseconds + * @param group_by Group by aggregation. + * @throws IllegalStateException if rollup interval, rollup aggregator or + * group by aggregator is null + */ + public RollupQuery(final RollupInterval rollup_interval, + final Aggregator rollup_agg, + long sample_interval_ms, + final Aggregator group_by) { + + if (rollup_interval == null) { + throw new IllegalStateException("Rollup interval is null"); + } + if (rollup_agg == null) { + throw new IllegalStateException("Rollup aggregator is null"); + } + if (group_by == null) { + throw new IllegalStateException("Group by aggregator is null"); + } + + this.rollup_interval = rollup_interval; + // we need to convert zimsum => sum, mimmax => max, mimmin => min so that + // we match properly on the column names + if (rollup_agg == Aggregators.ZIMSUM) { + this.rollup_agg = Aggregators.SUM; + } else if (rollup_agg == Aggregators.MIMMAX) { + this.rollup_agg = Aggregators.MAX; + } else if (rollup_agg == Aggregators.MIMMIN) { + this.rollup_agg = Aggregators.MIN; + } else { + this.rollup_agg = rollup_agg; + } + if (group_by == Aggregators.ZIMSUM) { + this.group_by = Aggregators.SUM; + } else if (group_by == Aggregators.MIMMAX) { + this.group_by = Aggregators.MAX; + } else if (group_by == Aggregators.MIMMIN) { + this.group_by = Aggregators.MIN; + } else { + this.group_by = group_by; + } + if (group_by == Aggregators.AVG) { + agg_prefix = RollupUtils.getRollupQualifierPrefix( + Aggregators.SUM.toString()); + } else { + agg_prefix = RollupUtils.getRollupQualifierPrefix( + this.group_by.toString()); + } + this.sample_interval_ms = sample_interval_ms; + } + + @Override + public int hashCode() { + return Objects.hashCode(rollup_interval, rollup_agg.toString()); + } + + @Override + public boolean equals(final Object obj) { + if (obj == null) { + return false; + } + if (!(obj instanceof RollupQuery)) { + return false; + } + if (obj == this) { + return true; + } + final RollupQuery query = (RollupQuery)obj; + return Objects.equal(rollup_agg, query.rollup_agg) + && rollup_interval.equals(query.rollup_interval); + } + + @Override + public String toString() { + final StringBuilder buf = new StringBuilder(); + buf.append("rollup interval=") + .append(rollup_interval.getInterval()) + .append(", rollup aggregator=") + .append(rollup_agg.toString()) + .append(", group_by=") + .append(group_by.toString()); + return buf.toString(); + } + + /** + * @return the count of intervals in this span + */ + public RollupInterval getRollupInterval() { + return rollup_interval; + } + + /** + * @return the rollup aggregator + */ + @JsonIgnore + public Aggregator getRollupAgg() { + return rollup_agg; + } + + @JsonIgnore + public Aggregator getGroupBy() { + return group_by; + } + + /** + * Does it contain a valid rollup interval, mainly says it is not the default + * rollup. Default rollup is of same resolution as raw data. So if true, + * which means the raw cell column qualifier is encoded with the aggregate + * function and the cell is not appended or compacted + * @param rollup_query related RollupQuery object, null if rollup is disabled + * @return true if it is rollup query + */ + public static boolean isValidQuery(final RollupQuery rollup_query) { + return (rollup_query != null && rollup_query.rollup_interval != null && + !rollup_query.rollup_interval.isDefaultInterval()); + } + + /** + * Rollup aggregate prefix + * @return aggregate prefix along with the delimiter as byte array + */ + public byte[] getRollupAggPrefix() { + return agg_prefix; + } + + /** @return The sample interval in milliseconds. */ + public long getSampleIntervalInMS() { + return sample_interval_ms; + } + + /** + * Tells whether the current rollup query sampling rate is lower than the + * initial request + * + * @return true if it is of lower sampling rate else false + */ + public boolean isLowerSamplingRate() { + return this.rollup_interval.getIntervalSeconds() * 1000 < sample_interval_ms; + } + + /** + * Looks at the SLA configured for the table to be queried and determines the + * timestamp of the latest data point that is guaranteed to be covered by the + * table. + * @return last timestamp in seconds of the period guaranteed to be covered + */ + public int getLastRollupTimestampSeconds() { + return (int) (DateTime.currentTimeMillis()/1000 - getRollupInterval().getMaximumLag()); + } + + /** + * Checks whether the passed timestamp is in the blackout period (between + * the latest guaranteed timestamp and now) + * @param timestampMillis The timestamp to check in milliseconds + * @return whether the timestamp is in the blackout period + */ + public boolean isInBlackoutPeriod(long timestampMillis) { + long latestRollupPointTimestamp = DateTime.currentTimeMillis() - getRollupInterval().getMaximumLag()*1000L; + + return timestampMillis > latestRollupPointTimestamp; + } +} diff --git a/src/rollup/RollupSeq.java b/src/rollup/RollupSeq.java new file mode 100644 index 0000000000..fb420ab4a3 --- /dev/null +++ b/src/rollup/RollupSeq.java @@ -0,0 +1,746 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.rollup; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; + +import net.opentsdb.core.Aggregators; +import net.opentsdb.core.Const; +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.IllegalDataException; +import net.opentsdb.core.Internal; +import net.opentsdb.core.RowKey; +import net.opentsdb.core.RowSeq; +import net.opentsdb.core.SeekableView; +import net.opentsdb.core.TSDB; +import net.opentsdb.core.Tags; +import net.opentsdb.core.iRowSeq; +import net.opentsdb.meta.Annotation; + +import org.hbase.async.Bytes; +import org.hbase.async.Bytes.ByteMap; +import org.hbase.async.KeyValue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.stumbleupon.async.Deferred; + +/** + * Represents a read-only sequence of continuous HBase rows. + *

    + * This class stores in memory the data of one or more continuous + * HBase rows for a given time series. To consolidate memory, the data points + * are stored in two byte arrays: one for the time offsets/flags and another + * for the values. Access is granted via pointers. + * @since 2.4 + */ +public final class RollupSeq implements iRowSeq { + private static final Logger LOG = LoggerFactory.getLogger(RollupSeq.class); + + /** The {@link TSDB} instance we belong to. */ + private final TSDB tsdb; + + /** The RollupQuery object holds information about a rollup interval and + * rollup aggregator */ + private final RollupQuery rollup_query; + + /** Whether or not we need counts with our data, e.g. to compute the average */ + private final boolean need_count; + private final int agg_id; + private final int count_id; + + /** First row key. */ + protected byte[] key; + + /** The qualifier and values for the request rollup type or SUM if the user + * asked for AVG or DEV. */ + protected byte[] qualifiers; + protected byte[] values; + protected long last_value_ts; + + /** If the user asked for AVG or DEV then we store the COUNT values here */ + protected byte[] count_qualifiers; + protected byte[] count_values; + protected long last_count_ts; + + /** An array of indices for the arrays above */ + protected int[] indices; // 0 = q, 1 = v, 2 = cq, 3 = cv + + /** Sentinels to make sure we don't sneak in any out-of-order values */ + protected int last_offset = -1; + protected int last_count_offset = -1; + + /** + * Default constructor. + * @param tsdb The TSDB to which we belong + * @param rollup_query holds information about a rollup interval and + * rollup aggregator + */ + public RollupSeq(final TSDB tsdb, final RollupQuery rollup_query) { + this.tsdb = tsdb; + this.rollup_query = rollup_query; + + // TODO - others + need_count = rollup_query.getRollupAgg() == Aggregators.AVG || + rollup_query.getRollupAgg() == Aggregators.DEV; + + // WARNING overallocation + qualifiers = new byte[rollup_query.getRollupInterval().getIntervals() * 2]; + // NEED to dynamically expand this sucker + values = new byte[rollup_query.getRollupInterval().getIntervals()]; + + if (need_count) { + count_qualifiers = new byte[rollup_query.getRollupInterval().getIntervals() * 2]; + count_values = new byte[rollup_query.getRollupInterval().getIntervals()]; + indices = new int[4]; + agg_id = tsdb.getRollupConfig().getIdForAggregator("sum"); + count_id = tsdb.getRollupConfig().getIdForAggregator("count"); + } else { + indices = new int[2]; + agg_id = tsdb.getRollupConfig() + .getIdForAggregator(rollup_query.getRollupAgg().toString()); + count_id = tsdb.getRollupConfig().getIdForAggregator("count"); + } + } + + /** + * Sets the row this instance holds in RAM using a row from a scanner. + * @param column The compacted HBase row to set. + * @throws IllegalStateException if this method was already called. + */ + public void setRow(final KeyValue column) { + //This api will be called only with the KeyValues from rollup table, as per + //the scan logic + if (key != null) { + throw new IllegalStateException("setRow was already called on " + this); + } + + key = column.key(); + + //Check whether the cell is generated by same rollup aggregator + if (need_count) { + if ((column.qualifier()[0] & RollupUtils.AGGREGATOR_MASK) == agg_id) { + append(column, false, false); + } else if ((column.qualifier()[0] & RollupUtils.AGGREGATOR_MASK) == count_id) { + append(column, true, false); + // OLD style for Yahoo! + } else if (Bytes.memcmp(RollupQuery.SUM, column.qualifier(), 0, RollupQuery.SUM.length) == 0) { + append(column, false, true); + } else if (Bytes.memcmp(RollupQuery.COUNT, column.qualifier(), 0, RollupQuery.COUNT.length) == 0) { + append(column, true, true); + } else { + throw new IllegalDataException("Attempt to add a different aggrregate cell =" + + column + ", expected aggregator either SUM or COUNT"); + } + } else { + if ((column.qualifier()[0] & RollupUtils.AGGREGATOR_MASK) == agg_id) { + append(column, false, false); + } else if (Bytes.memcmp(column.qualifier(), rollup_query.getRollupAggPrefix(), 0, + rollup_query.getRollupAggPrefix().length) == 0) { + append(column, false, true); + } else { + throw new IllegalDataException("Attempt to add a different aggrregate cell =" + + column + ", expected aggregator " + Bytes.pretty( + rollup_query.getRollupAggPrefix())); + } + } + } + + /**This method of parent/super class is not applicable to Rollup data point. + * @param column The compacted HBase row to merge into this instance. + * @throws IllegalStateException if {@link #setRow} wasn't called first. + * @throws IllegalArgumentException if the data points in the argument + * do not belong to the same row as this RowSeq + */ + public void addRow(final KeyValue column) { + if (key == null) { + throw new IllegalStateException("setRow was never called on " + this); + } + + if (Bytes.memcmp(column.key(), key, Const.SALT_WIDTH(), + key.length - Const.SALT_WIDTH()) != 0) { + throw new IllegalDataException("Attempt to add a different row=" + + column + ", this=" + this); + } + + //Check whether the cell is generated by same rollup aggregator + if (need_count) { + if ((column.qualifier()[0] & RollupUtils.AGGREGATOR_MASK) == agg_id) { + append(column, false, false); + } else if ((column.qualifier()[0] & RollupUtils.AGGREGATOR_MASK) == count_id) { + append(column, true, false); + // OLD style for Yahoo! + } else if (Bytes.memcmp(RollupQuery.SUM, column.qualifier(), 0, RollupQuery.SUM.length) == 0) { + append(column, false, true); + } else if (Bytes.memcmp(RollupQuery.COUNT, column.qualifier(), 0, RollupQuery.COUNT.length) == 0) { + append(column, true, true); + } else { + throw new IllegalDataException("Attempt to add a different aggrregate cell =" + + column + ", expected aggregator either SUM or COUNT"); + } + } else { + if ((column.qualifier()[0] & RollupUtils.AGGREGATOR_MASK) == agg_id) { + append(column, false, false); + } else if (Bytes.memcmp(column.qualifier(), rollup_query.getRollupAggPrefix(), 0, + rollup_query.getRollupAggPrefix().length) == 0) { + append(column, false, true); + } else { + throw new IllegalDataException("Attempt to add a different aggrregate cell =" + + column + ", expected aggregator " + Bytes.pretty( + rollup_query.getRollupAggPrefix())); + } + } + } + + /** + * Adds the column to the byte arrays. + * @param column The non-null key value to add. + * @param is_count Whether or not the column is for counts. + * @param strip_string Whether or not the column has the old style string + * header and needs cleaning. + */ + private void append(final KeyValue column, + final boolean is_count, + final boolean strip_string) { + // for now assume we properly allocated our qualifiers + if (is_count) { + int offset = strip_string ? + Internal.getOffsetFromQualifier(column.qualifier(), + RollupQuery.COUNT.length + 1) : + Internal.getOffsetFromQualifier(column.qualifier(), + 1); + if (last_count_offset > -1 && offset <= last_count_offset) { + // only accept equivalent offsets. If somehow we get an earlier one, HBase is broke + if (offset == last_count_offset && tsdb.getConfig().fix_duplicates()) { + if (column.timestamp() < last_count_ts) { + if (LOG.isDebugEnabled()) { + LOG.debug("Skipping older duplicate count value for " + column + + " of " + offset + " which is = the last offset " + last_count_offset + + " for " + this); + } + return; + } else { // if it's equal, just use the one we got first + // roll back the indices + indices[2] -= 2; + indices[3] -= Internal.getValueLengthFromQualifier( + count_qualifiers, indices[2]); + if (LOG.isDebugEnabled()) { + LOG.debug("Replacing older duplicate count with " + column + + " of " + offset + " which is = the last offset " + last_count_offset + + " for " + this); + } + } + } else { + throw new IllegalArgumentException("The count offset for " + column + + " of " + offset + " is <= the last offset " + last_count_offset + + " for " + this); + } + } + last_count_offset = offset; + last_count_ts = column.timestamp(); + if (strip_string) { + System.arraycopy(column.qualifier(), RollupQuery.COUNT.length + 1, + count_qualifiers, indices[2], 2); + } else { + System.arraycopy(column.qualifier(), 1, count_qualifiers, indices[2], 2); + } + indices[2] += 2; + + if (indices[3] + column.value().length > count_values.length) { + byte[] buf = new byte[count_values.length * 2]; + System.arraycopy(count_values, 0, buf, 0, count_values.length); + count_values = buf; + } + System.arraycopy(column.value(), 0, count_values, indices[3], + column.value().length); + indices[3] += column.value().length; + } else { + int offset = strip_string ? + Internal.getOffsetFromQualifier(column.qualifier(), + rollup_query.getRollupAggPrefix().length) + : Internal.getOffsetFromQualifier(column.qualifier(), 1); + if (last_offset > -1 && offset <= last_offset) { + // only accept equivalent offsets. If somehow we get an earlier one, HBase is broke + if (offset == last_offset && tsdb.getConfig().fix_duplicates()) { + if (column.timestamp() < last_value_ts) { + if (LOG.isDebugEnabled()) { + LOG.debug("Skipping older duplicate value for " + column + + " of " + offset + " which is = the last offset " + last_count_offset + + " for " + this); + } + return; + } else { // if it's equal, just use the one we got first + // roll back the indices + indices[0] -= 2; + indices[1] -= Internal.getValueLengthFromQualifier(qualifiers, indices[0]); + if (LOG.isDebugEnabled()) { + LOG.debug("Replacing older duplicate value with " + column + + " of " + offset + " is = the last offset " + last_count_offset + + " for " + this); + } + } + } else { + throw new IllegalDataException("The offset for " + column + + " of " + offset + " is <= the last offset " + last_offset + + " for " + this); + } + } + last_offset = offset; + last_value_ts = column.timestamp(); + if (strip_string) { + System.arraycopy(column.qualifier(), + rollup_query.getRollupAggPrefix().length, qualifiers, indices[0], 2); + } else { + System.arraycopy(column.qualifier(), 1, qualifiers, indices[0], 2); + } + indices[0] += 2; + + if (indices[1] + column.value().length > values.length) { + byte[] buf = new byte[values.length * 2]; + System.arraycopy(values, 0, buf, 0, values.length); + values = buf; + } + System.arraycopy(column.value(), 0, values, indices[1], + column.value().length); + indices[1] += column.value().length; + } + } + + @Override + public String toString() { + final StringBuilder buf = new StringBuilder(80 + + + (key == null ? 0 : key.length * 4) + + indices[0] * 8); + buf.append("RollupSeq(key=") + .append(key == null ? "" : Arrays.toString(key)) + .append(" base_time=") + .append(key == null ? "" : baseTime()) + .append(", basetime=") + .append(key == null ? "no data" : new Date(baseTime() * 1000)) + .append(", "); + buf.append("datapoints=").append(indices[0] / 2) + .append(", counts=").append(indices.length > 2 ? indices[2] / 2 : "0"); + buf.append(",\n (qualifier=[").append(Arrays.toString(qualifiers)); + buf.append("]),\n (values=[").append(Arrays.toString(values)); + buf.append("],\n (count_qualifier=[").append(Arrays.toString(count_qualifiers)); + buf.append("],\n (count_values=[").append(Arrays.toString(count_values)); + buf.append("])"); + return buf.toString(); + } + + public String metricName() { + try { + return metricNameAsync().join(); + } catch (InterruptedException iex) { + throw new RuntimeException("Interrupted the metric name call", iex); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Should never be here", e); + } + } + + public Deferred metricNameAsync() { + if (key == null) { + throw new IllegalStateException("the row key is null!"); + } + return RowKey.metricNameAsync(tsdb, key); + } + + public byte[] metricUID() { + return Arrays.copyOfRange(key, Const.SALT_WIDTH(), + Const.SALT_WIDTH() + TSDB.metrics_width()); + } + + public Map getTags() { + try { + return getTagsAsync().join(); + } catch (InterruptedException iex) { + throw new RuntimeException("Interrupted the tags call", iex); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Should never be here", e); + } + } + + public Deferred> getTagsAsync() { + return Tags.getTagsAsync(tsdb, key); + } + + @Override + public ByteMap getTagUids() { + return Tags.getTagUids(key); + } + + /** @return an empty list since aggregated tags cannot exist on a single row */ + public List getAggregatedTags() { + return Collections.emptyList(); + } + + public Deferred> getAggregatedTagsAsync() { + final List empty = Collections.emptyList(); + return Deferred.fromResult(empty); + } + + @Override + public List getAggregatedTagUids() { + return Collections.emptyList(); + } + + public List getTSUIDs() { + return Collections.emptyList(); + } + + /** @return null since annotations are stored at the SpanGroup level. They + * are filtered when a row is compacted */ + public List getAnnotations() { + return Collections.emptyList(); + } + + @Override + public int size() { + if (need_count) { + int count = 0; + final Iterator it = internalIterator(); + while (it.hasNext()) { + it.next(); + ++count; + } + return count; + } else { + return indices[0] / 2; + } + } + + @Override + public int aggregatedSize() { + return 0; + } + + @Override + public long timestamp(int i) { + if (i < 0) { + throw new IndexOutOfBoundsException("index " + i + + " must be positive for this=" + this); + } + if (need_count) { + final Iterator it = internalIterator(); + int count = 0; + while (it.hasNext()) { + final DataPoint dp = it.next(); + if (count == i) { + return dp.timestamp(); + } + ++count; + } + throw new IndexOutOfBoundsException("index " + i + " >= " + size() + + " for this=" + this); + } else { + if (i * 2 >= indices[0]) { + throw new IndexOutOfBoundsException("index " + i + " >= " + size() + + " for this=" + this); + } + return RollupUtils.getTimestampFromRollupQualifier(qualifiers, baseTime(), + rollup_query.getRollupInterval(), i * 2); + } + } + + @Override + public boolean isInteger(int i) { + throw new UnsupportedOperationException(); + } + + @Override + public long longValue(int i) { + throw new UnsupportedOperationException(); + } + + @Override + public double doubleValue(int i) { + throw new UnsupportedOperationException(); + } + + @Override + public int getQueryIndex() { + return 0; + } + + @Override + public byte[] key() { + return key; + } + + @Override + public long baseTime() { + return Internal.baseTime(key); + } + + @Override + public SeekableView iterator() { + return internalIterator(); + } + + @Override + public Iterator internalIterator() { + return new RollupIterator(); + } + + @Override + public boolean isPercentile() { + return false; + } + + @Override + public float getPercentile() { + throw new UnsupportedOperationException("getPercentile not supported"); + } + + /** Iterator for {@link RowSeq}s. */ + public final class RollupIterator implements iRowSeq.Iterator { + + /** Current qualifier. */ + private int qualifier; + + /** Next index in {@link #qualifiers}. */ + private int qual_index; + + /** Next index in {@link #values}. */ + private int value_index; + + /** Current qualifier for the counts */ + private int count_qualifier; + + /** Next index in {@link #count_qualifier}. */ + private int count_qual_index; + + /** Next index in {@link #count_values}. */ + private int count_value_index; + + /** Pre-extracted base time of this row sequence. */ + private final long base_time = baseTime(); + + RollupIterator() { + if (need_count) { + sync(); + } + } + + // ------------------ // + // Iterator interface // + // ------------------ // + + public boolean hasNext() { + if (need_count) { + sync(); + return qual_index < indices[0] && + count_qual_index < indices[2]; + } + return qual_index < indices[0]; + } + + void sync() { + if (qual_index >= indices[0] || count_qual_index >= indices[2]) { + return; + } + long q_ts = Internal.getOffsetFromQualifier(qualifiers, qual_index); + long c_ts = Internal.getOffsetFromQualifier(count_qualifiers, count_qual_index); + if (q_ts == c_ts) { + return; + } + LOG.warn("Different agg [" + q_ts + "] and count [" + c_ts + + "] offsets for " + this); + if (q_ts > c_ts) { + count_value_index += Internal.getValueLengthFromQualifier( + count_qualifiers, count_qual_index); + count_qual_index += 2; + if (count_qual_index >= count_qualifiers.length) { + LOG.warn("Ran out of counts: " + this); + return; + } + sync(); + } else { + value_index += Internal.getValueLengthFromQualifier(qualifiers, qual_index); + qual_index += 2; + if (qual_index >= qualifiers.length) { + LOG.warn("Ran out of qualifiers: " + this); + return; + } + sync(); + } + } + + public DataPoint next() { + if (!hasNext()) { + throw new NoSuchElementException("no more elements"); + } + value_index += Internal.getValueLengthFromQualifier(qualifiers, qual_index); + qualifier = Bytes.getUnsignedShort(qualifiers, qual_index); + qual_index += 2; + if (need_count) { + count_value_index += Internal.getValueLengthFromQualifier( + count_qualifiers, count_qual_index); + count_qualifier = Bytes.getUnsignedShort(count_qualifiers, count_qual_index); + count_qual_index += 2; + } + return this; + } + + public void remove() { + throw new UnsupportedOperationException(); + } + + // ---------------------- // + // SeekableView interface // + // ---------------------- // + + @Override + public void seek(final long timestamp) { + final long ts; + if ((timestamp & Const.SECOND_MASK) == 0) { + ts = timestamp * 1000; + } else { + ts = timestamp; + } + // reset all + qual_index = value_index = count_qual_index = count_value_index = 0; + if (!hasNext()) { + return; + } + + qualifier = Bytes.getUnsignedShort(qualifiers, qual_index); + if (need_count) { + count_qualifier = Bytes.getUnsignedShort(count_qualifiers, count_qual_index); + } + while (qual_index < indices[0] && + RollupUtils.getTimestampFromRollupQualifier(qualifiers, base_time, + rollup_query.getRollupInterval(), qual_index) < ts) { + value_index += Internal.getValueLengthFromQualifier(qualifiers, qual_index); + qualifier = Bytes.getUnsignedShort(qualifiers, qual_index); + qual_index += 2; + if (need_count) { + count_value_index += Internal.getValueLengthFromQualifier( + count_qualifiers, count_qual_index); + count_qualifier = Bytes.getUnsignedShort(count_qualifiers, count_qual_index); + count_qual_index += 2; + } + } + } + + // ------------------- // + // DataPoint interface // + // ------------------- // + + public long timestamp() { + return RollupUtils.getTimestampFromRollupQualifier(qualifier, base_time, + rollup_query.getRollupInterval()); + } + + public boolean isInteger() { + assert qual_index > 0: "not initialized: " + this; + return (qualifier & Const.FLAG_FLOAT) == 0x0; + } + + @Override + public long valueCount() { + if (count_values == null && rollup_query.getRollupAgg() != Aggregators.COUNT) { + // real values (sum, max, min) so just return 1. + return 1; + } + if (rollup_query.getRollupAgg() == Aggregators.COUNT) { + if (isInteger()) { + return longValue(); + } else { + return (long) doubleValue(); + } + } + final byte flags = (byte) count_qualifier; + final byte vlen = (byte) ((flags & Const.LENGTH_MASK) + 1); + if ((count_qualifier & Const.FLAG_FLOAT) == 0x0) { + return Internal.extractIntegerValue(count_values, count_value_index - vlen, flags); + } else { + return (long)Internal.extractFloatingPointValue(count_values, count_value_index - vlen, flags); + } + } + + public long longValue() { + if (!isInteger()) { + throw new ClassCastException("value @" + + qual_index + " is not a long in " + this); + } + + final byte flags = (byte) qualifier; + final byte vlen = (byte) ((flags & Const.LENGTH_MASK) + 1); + return Internal.extractIntegerValue(values, value_index - vlen, flags); + //return extractIntegerValue(values, value_index - vlen, flags); + } + + public double doubleValue() { + if (isInteger()) { + throw new ClassCastException("value @" + + qual_index + " is not a float in " + this); + } + final byte flags = (byte) qualifier; + final byte vlen = (byte) ((flags & Const.LENGTH_MASK) + 1); + return Internal.extractFloatingPointValue(values, value_index - vlen, flags); + //return extractFloatingPointValue(values, value_index - vlen, flags); + } + + public double toDouble() { + return isInteger() ? longValue() : doubleValue(); + } + + // ---------------- // + // Helpers for Span // + // ---------------- // + + /** Helper to take a snapshot of the state of this iterator. */ + long saveState() { + return ((long)qual_index << 32) | ((long)value_index & 0xFFFFFFFF); + } + + /** Helper to restore a snapshot of the state of this iterator. */ + void restoreState(long state) { + value_index = (int) state & 0xFFFFFFFF; + state >>>= 32; + qual_index = (int) state; + qualifier = 0; + } + + /** + * Look a head to see the next timestamp. + * @throws IndexOutOfBoundsException if we reached the end already. + */ + long peekNextTimestamp() { + return RollupUtils.getTimestampFromRollupQualifier(qualifiers, base_time, + rollup_query.getRollupInterval(), qual_index); + } + + /** Only returns internal state for the iterator itself. */ + String toStringSummary() { + return "RowSeq.Iterator(qual_index=" + qual_index + + ", value_index=" + value_index + ", cq_idx=" + count_qual_index + + ", cv_idx=" + count_value_index; + } + + public String toString() { + return toStringSummary() + ", seq=" + RollupSeq.this + ')'; + } + + } +} diff --git a/src/rollup/RollupSpan.java b/src/rollup/RollupSpan.java new file mode 100644 index 0000000000..2f9314c6fc --- /dev/null +++ b/src/rollup/RollupSpan.java @@ -0,0 +1,84 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.rollup; + +import org.hbase.async.Bytes; +import org.hbase.async.KeyValue; + +import net.opentsdb.core.RowSeq; +import net.opentsdb.core.Span; +import net.opentsdb.core.TSDB; +import net.opentsdb.core.iRowSeq; + +/** + * Represents a read-only sequence of continuous data points. + *

    + * This class stores a continuous sequence of {@link RowSeq}s in memory. + * @since 2.4 + */ +public final class RollupSpan extends Span { + + private RollupQuery rollup_query; + + private byte[] tsuid; + + /** + * Default constructor. + * @param tsdb The TSDB to which we belong + * @param rollup_query holds information about a rollup interval and + * rollup aggregator + * @throws IllegalStateException if it is default rollup interval + */ + public RollupSpan(final TSDB tsdb, RollupQuery rollup_query) { + super(tsdb); + + if (rollup_query.getRollupInterval().isDefaultInterval()) { + throw new IllegalStateException("Rolup Span is not applicable to default " + + "rollup interval. Default rollup interval is encoded in the same way" + + " as the raw data."); + } + + this.rollup_query = rollup_query; + } + + @Override + protected void addRow(final KeyValue row) { + if (rows.size() > 0) { + final byte[] key = row.key(); + final iRowSeq last = rows.get(rows.size() - 1); + String error = null; + if (key.length != last.key().length) { + error = "row key length mismatch"; + } + + if (Bytes.memcmp(last.key(), key) == 0) { + last.addRow(row); + return; + } + } + final iRowSeq rowseq = createRowSequence(tsdb); + rowseq.setRow(row); + rows.add(rowseq); + } + + /** + * RowSeq abstract factory API implementation + * @param tsdb The TSDB to which we belong + * @return RollupSeq object which stores read-only sequence + * of continuous HBase rows - rollup data + */ + @Override + protected iRowSeq createRowSequence(final TSDB tsdb) { + return new RollupSeq(tsdb, this.rollup_query); + } +} diff --git a/src/rollup/RollupUtils.java b/src/rollup/RollupUtils.java new file mode 100644 index 0000000000..8236171632 --- /dev/null +++ b/src/rollup/RollupUtils.java @@ -0,0 +1,264 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2010-2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.rollup; + +import java.util.Calendar; + +import org.hbase.async.Bytes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import net.opentsdb.core.Const; + +/** + * Static util class for dealing with parsing and storing rolled up data points + * @since 2.4 + */ +public final class RollupUtils { + private static final Logger LOG = LoggerFactory.getLogger(RollupUtils.class); + + public static final byte AGGREGATOR_MASK = (byte) 0x7F; + + public static final byte COMPACTED_MASK = (byte) 0x80; + + /** The rollup qualifier delimiter character */ + public static final String ROLLUP_QUAL_DELIM = ":"; + + private RollupUtils() { + // Do not instantiate me brah! + } + + /** + * Calculates the base time for a rollup interval, the time that can be + * stored in the row key. + * @param timestamp The data point timestamp to calculate from in seconds + * or milliseconds + * @param interval The configured interval object to use for calcaulting + * the base time with a valid span of 'h', 'd', 'm' or 'y' + * @return A base time as a unix epoch timestamp in seconds + * @throws IllegalArgumentException if the timestamp is negative or the interval + * has an unsupported span + */ + public static int getRollupBasetime(final long timestamp, + final RollupInterval interval) { + if (timestamp < 0) { + throw new IllegalArgumentException("Not supporting negative " + + "timestamps at this time: " + timestamp); + } + + // avoid instantiating a calendar at all costs! If we are based on an hourly + // span then use the old method of snapping to the hour + if (interval.getUnits() == 'h') { + int modulo = Const.MAX_TIMESPAN; + if (interval.getUnitMultiplier() > 1) { + modulo = interval.getUnitMultiplier() * 60 * 60; + } + if ((timestamp & Const.SECOND_MASK) != 0) { + // drop the ms timestamp to seconds to calculate the base timestamp + return (int) ((timestamp / 1000) - + ((timestamp / 1000) % modulo)); + } else { + return (int) (timestamp - (timestamp % modulo)); + } + } else { + final long time_milliseconds = (timestamp & Const.SECOND_MASK) != 0 ? + timestamp : timestamp * 1000; + + // gotta go the long way with the calendar to snap to an appropriate + // daily, monthly or weekly boundary + final Calendar calendar = Calendar.getInstance(Const.UTC_TZ); + calendar.setTimeInMillis(time_milliseconds); + + // zero out the hour, minutes, seconds + calendar.set(Calendar.HOUR_OF_DAY, 0); + calendar.set(Calendar.MINUTE, 0); + calendar.set(Calendar.SECOND, 0); + + switch (interval.getUnits()) { + case 'd': + // all set via the zeros above + break; + case 'n': + calendar.set(Calendar.DAY_OF_MONTH, 1); + break; + case 'y': + calendar.set(Calendar.DAY_OF_MONTH, 1); + calendar.set(Calendar.MONTH, 0); // 0 for January + break; + default: + throw new IllegalArgumentException("Unrecogznied span: " + interval); + } + + return (int) (calendar.getTimeInMillis() / 1000); + } + } + + /** + * Builds a rollup column qualifier, prepending the appender as a string + * then the offeset on 2 bytes for the interval after a colon with the last + * four bits reserved for the length and type flags. I.e. + * {@code : + * n : 2 bytes } + * @param timestamp The data point timestamp + * @param flags The length and type (float || int) flags for the value + * @param aggregator_id The numeric ID of the aggregator the value maps to. + * @param interval The RollupInterval object with data about the interval + * @return An n byte array to use as the qualifier + * @throws IllegalArgumentException if the aggregator is null or empty or the + * timestamp is too far from the base time to fit within the interval. + */ + public static byte[] buildRollupQualifier(final long timestamp, + final short flags, + final int aggregator_id, + final RollupInterval interval) { + return buildRollupQualifier(timestamp, + getRollupBasetime(timestamp, interval), flags, aggregator_id, interval); + } + + /** + * Builds a rollup column qualifier, prepending the appender as a string + * then the offeset on 2 bytes for the interval after a colon with the last + * four bits reserved for the length and type flags. I.e. + * {@code : + * n : 2 bytes } + * @param timestamp The data point timestamp + * @param basetime The base timestamp to calculate the offset from + * @param flags The length and type (float || int) flags for the value + * @param aggregator_id The numeric ID of the aggregator the value maps to. + * @param interval The RollupInterval object with data about the interval + * @return An n byte array to use as the qualifier + * @throws IllegalArgumentException if the aggregator is null or empty or the + * timestamp is too far from the base time to fit within the interval. + */ + public static byte[] buildRollupQualifier(final long timestamp, + final int basetime, + final short flags, + final int aggregator_id, + final RollupInterval interval) { + final byte[] qualifier = new byte[3]; + + final int time_seconds = (int) ((timestamp & Const.SECOND_MASK) != 0 ? + timestamp / 1000 : timestamp); + + // we shouldn't have a divide by 0 here as the rollup config validator makes + // sure the interval is positive + int offset = (time_seconds - basetime) / interval.getIntervalSeconds(); + if (offset >= interval.getIntervals()) { + throw new IllegalArgumentException("Offset of " + offset + " was greater " + + "than the configured intervals " + interval.getIntervals()); + } + + // shift the offset over 4 bits then apply the flag + offset = offset << Const.FLAG_BITS; + offset = offset | flags; + qualifier[0] = (byte) aggregator_id; + System.arraycopy(Bytes.fromShort((short) offset), 0, qualifier, 1, 2); + + return qualifier; + } + + /** + * Returns the absolute timestamp of a data point qualifier in milliseconds + * @param qualifier The qualifier to parse + * @param base_time The base time, in seconds, from the row key + * @param interval The RollupInterval object with data about the interval + * @param offset An offset within the byte array + * @return The absolute timestamp in milliseconds + */ + public static long getTimestampFromRollupQualifier(final byte[] qualifier, + final long base_time, + final RollupInterval interval, + final int offset) { + return (base_time * 1000) + + getOffsetFromRollupQualifier(qualifier, offset, interval); + } + + /** + * Returns the absolute timestamp of a data point qualifier in milliseconds + * @param qualifier The qualifier to parse + * @param base_time The base time, in seconds, from the row key + * @param interval The RollupInterval object with data about the interval + * @return The absolute timestamp in milliseconds + */ + public static long getTimestampFromRollupQualifier(final int qualifier, + final long base_time, + final RollupInterval interval) { + return (base_time * 1000) + getOffsetFromRollupQualifier(qualifier, interval); + } + + /** + * Returns the offset in milliseconds from the row base timestamp from a data + * point qualifier at the given offset (for compacted columns) + * @param qualifier The qualifier to parse + * @param byte_offset An offset within the byte array + * @param interval The RollupInterval object with data about the interval + * @return The offset in milliseconds from the base time + */ + public static long getOffsetFromRollupQualifier(final byte[] qualifier, + final int byte_offset, + final RollupInterval interval) { + + long offset = 0; + + if ((qualifier[byte_offset] & Const.MS_BYTE_FLAG) == Const.MS_BYTE_FLAG) { + offset = ((Bytes.getUnsignedInt(qualifier, byte_offset) & 0x0FFFFFC0) + >>> Const.MS_FLAG_BITS)/1000; + } else { + offset = (Bytes.getUnsignedShort(qualifier, byte_offset) & 0xFFFF) + >>> Const.FLAG_BITS; + } + + return offset * interval.getIntervalSeconds() * 1000; + } + + /** + * Returns the offset in milliseconds from the row base timestamp from a data + * point qualifier at the given offset (for compacted columns) + * @param qualifier The qualifier to parse + * @param interval The RollupInterval object with data about the interval + * @return The offset in milliseconds from the base time + */ + public static long getOffsetFromRollupQualifier(final int qualifier, + final RollupInterval interval) { + + long offset = 0; + if ((qualifier & Const.MS_FLAG) == Const.MS_FLAG) { + LOG.warn("Unexpected rollup qualifier in milliseconds: " + qualifier + + " for interval " + interval); + offset = (qualifier & 0x0FFFFFC0) >>> (Const.MS_FLAG_BITS) / 1000; + } else { + offset = (qualifier & 0xFFFF) >>> Const.FLAG_BITS; + } + return offset * interval.getIntervalSeconds() * 1000; + } + + /** + * Builds a rollup column qualifier prefix,prepending the appender as a string + * along with a colon as delimiter + * @param aggregator The aggregator used to generate the data + * @return An n byte array to use as the qualifier prefix + */ + public static byte[] getRollupQualifierPrefix(final String aggregator) { + return (aggregator.toLowerCase() + ROLLUP_QUAL_DELIM) + .getBytes(Const.ASCII_CHARSET); + } + + /** + * Determines whether or not the column has been compacted or appended. + * @param qualifier A non-null and non-empty byte array. + * @return True if the column was compacted, false if not. + */ + public static boolean isCompacted(final byte[] qualifier) { + return (qualifier[0] & COMPACTED_MASK) != 0; + } +} diff --git a/src/search/SearchPlugin.java b/src/search/SearchPlugin.java index d9bad4eb47..f1fdbed615 100644 --- a/src/search/SearchPlugin.java +++ b/src/search/SearchPlugin.java @@ -13,11 +13,16 @@ package net.opentsdb.search; import net.opentsdb.core.TSDB; +import net.opentsdb.core.TSQuery; import net.opentsdb.meta.Annotation; import net.opentsdb.meta.TSMeta; import net.opentsdb.meta.UIDMeta; import net.opentsdb.stats.StatsCollector; +import java.util.List; + +import org.hbase.async.Bytes.ByteMap; + import com.stumbleupon.async.Deferred; /** @@ -55,7 +60,7 @@ public abstract class SearchPlugin { * @param tsdb The parent TSDB object * @throws IllegalArgumentException if required configuration parameters are * missing - * @throws Exception if something else goes wrong + * @throws RuntimeException if something else goes wrong */ public abstract void initialize(final TSDB tsdb); @@ -144,6 +149,12 @@ public abstract class SearchPlugin { */ public abstract Deferred deleteAnnotation(final Annotation note); + public Deferred>> resolveTSQuery(final TSQuery query, + final int sub_query_index) { + throw new UnsupportedOperationException("Not implemented by this plugin: " + + this); + } + /** * Executes a very basic search query, returning the results in the SearchQuery * object passed in. diff --git a/src/search/TimeSeriesLookup.java b/src/search/TimeSeriesLookup.java index a95e3d791a..dfc829651c 100644 --- a/src/search/TimeSeriesLookup.java +++ b/src/search/TimeSeriesLookup.java @@ -14,19 +14,25 @@ import java.nio.charset.Charset; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.regex.Pattern; import net.opentsdb.core.Const; +import net.opentsdb.core.Internal; import net.opentsdb.core.RowKey; import net.opentsdb.core.TSDB; import net.opentsdb.core.Tags; +import net.opentsdb.meta.TSMeta; +import net.opentsdb.query.QueryUtil; import net.opentsdb.uid.NoSuchUniqueId; +import net.opentsdb.uid.NoSuchUniqueName; import net.opentsdb.uid.UniqueId; import net.opentsdb.uid.UniqueId.UniqueIdType; import net.opentsdb.utils.ByteArrayPair; +import net.opentsdb.utils.Exceptions; import net.opentsdb.utils.Pair; import org.hbase.async.Bytes; @@ -35,14 +41,18 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.stumbleupon.async.Callback; +import com.stumbleupon.async.Deferred; +import com.stumbleupon.async.DeferredGroupException; + /** * Lookup series related to a metric, tagk, tagv or any combination thereof. * This class doesn't handle wild-card searching yet. * * When dealing with tags, we can lookup on tagks, tagvs or pairs. Thus: - * tagk, null <- lookup all series with a tagk - * tagk, tagv <- lookup all series with a tag pair - * null, tagv <- lookup all series with a tag value somewhere + * tagk, null <- lookup all series with a tagk + * tagk, tagv <- lookup all series with a tag pair + * null, tagv <- lookup all series with a tag value somewhere * * The user can supply multiple tags in a query so the logic is a little goofy * but here it is: @@ -86,15 +96,30 @@ public class TimeSeriesLookup { /** The TSD to use for lookups */ private final TSDB tsdb; + /** The metric UID if given by the query, post resolution */ + private byte[] metric_uid; + + /** Tag UID pairs if given in the query. Key or value may be null. */ + private List pairs; + + /** The compiled row key regex for HBase filtering */ + private String rowkey_regex; + + /** Post scan filtering if we have a lot of values to look at */ + private String tagv_filter; + + /** The results to send to the caller */ + private final List tsuids; + /** * Default ctor * @param tsdb The TSD to which we belong - * @param metric A metric to match on, may be null - * @param tags One or more tags to match on, may be null + * @param query The search query to execute. */ public TimeSeriesLookup(final TSDB tsdb, final SearchQuery query) { this.tsdb = tsdb; this.query = query; + tsuids = Collections.synchronizedList(new ArrayList()); } /** @@ -109,25 +134,90 @@ public TimeSeriesLookup(final TSDB tsdb, final SearchQuery query) { * UID. */ public List lookup() { - LOG.info(query.toString()); - final StringBuilder tagv_filter = new StringBuilder(); - final Scanner scanner = getScanner(tagv_filter); - final List tsuids = new ArrayList(); - final Pattern tagv_regex = tagv_filter.length() > 1 ? - Pattern.compile(tagv_filter.toString()) : null; + try { + return lookupAsync().join(); + } catch (InterruptedException e) { + LOG.error("Interrupted performing lookup", e); + Thread.currentThread().interrupt(); + return null; + } catch (DeferredGroupException e) { + final Throwable ex = Exceptions.getCause(e); + if (ex instanceof NoSuchUniqueName) { + throw (NoSuchUniqueName)ex; + } + throw new RuntimeException("Unexpected exception", ex); + } catch (NoSuchUniqueName e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Unexpected exception", e); + } + } + + /** + * Lookup time series associated with the given metric, tagk, tagv or tag + * pairs. Either the meta table or the data table will be scanned. If no + * metric is given, a full table scan must be performed and this call may take + * a long time to complete. + * When dumping to stdout, if an ID can't be looked up, it will be logged and + * skipped. + * @return A list of TSUIDs matching the given lookup query. + * @throws NoSuchUniqueName if any of the given names fail to resolve to a + * UID. + * @since 2.2 + */ + public Deferred> lookupAsync() { + final Pattern tagv_regex = tagv_filter != null ? + Pattern.compile(tagv_filter) : null; + // we don't really know what size the UIDs will resolve to so just grab // a decent amount. final StringBuffer buf = to_stdout ? new StringBuffer(2048) : null; final long start = System.currentTimeMillis(); - - ArrayList> rows; - byte[] last_tsuid = null; // used to avoid dupes when scanning the data table - - try { - // synchronous to avoid stack overflows when scanning across the main data - // table. - while ((rows = scanner.nextRows().joinUninterruptibly()) != null) { + final int limit; + if (query.getLimit() > 0) { + if (query.useMeta() || Const.SALT_WIDTH() < 1) { + limit = query.getLimit(); + } else if (query.getLimit() < Const.SALT_BUCKETS()) { + limit = 1; + } else { + limit = query.getLimit() / Const.SALT_BUCKETS(); + } + } else { + limit = 0; + } + + class ScannerCB implements Callback>, + ArrayList>> { + private final Scanner scanner; + // used to avoid dupes when scanning the data table + private byte[] last_tsuid = null; + private int rows_read; + + ScannerCB(final Scanner scanner) { + this.scanner = scanner; + } + + Deferred> scan() { + return scanner.nextRows().addCallbackDeferring(this); + } + + @Override + public Deferred> call(final ArrayList> rows) + throws Exception { + if (rows == null) { + scanner.close(); + if (query.useMeta() || Const.SALT_WIDTH() < 1) { + LOG.debug("Lookup query matched " + tsuids.size() + " time series in " + + (System.currentTimeMillis() - start) + " ms"); + } + return Deferred.fromResult(tsuids); + } + for (final ArrayList row : rows) { + if (limit > 0 && rows_read >= limit) { + // little recursion to close the scanner and log above. + return call(null); + } final byte[] tsuid = query.useMeta() ? row.get(0).key() : UniqueId.getTSUIDFromKey(row.get(0).key(), TSDB.metrics_width(), Const.TIMESTAMP_BYTES); @@ -159,193 +249,325 @@ public List lookup() { buf.append(tag_pair.getKey()).append("=") .append(tag_pair.getValue()).append(" "); } - System.out.println(buf.toString()); } catch (NoSuchUniqueId nsui) { LOG.error("Unable to resolve UID in TSUID (" + UniqueId.uidToString(tsuid) + ") " + nsui.getMessage()); } - buf.setLength(0); // reset the buffer so we can re-use it + buf.setLength(0); // reset the buffer so we can re-use it } else { tsuids.add(tsuid); } + ++rows_read; } + + return scan(); + } + + @Override + public String toString() { + return "Scanner callback"; + } + } + + class CompleteCB implements Callback, ArrayList>> { + @Override + public List call(final ArrayList> unused) throws Exception { + LOG.debug("Lookup query matched " + tsuids.size() + " time series in " + + (System.currentTimeMillis() - start) + " ms"); + return tsuids; + } + @Override + public String toString() { + return "Final async lookup callback"; } - } catch (Exception e) { - throw new RuntimeException("Shouldn't be here", e); - } finally { - scanner.close(); } - LOG.debug("Lookup query matched " + tsuids.size() + " time series in " + - (System.currentTimeMillis() - start) + " ms"); - return tsuids; + class UIDCB implements Callback>, Object> { + @Override + public Deferred> call(Object arg0) throws Exception { + if (!query.useMeta() && Const.SALT_WIDTH() > 0 && metric_uid != null) { + final ArrayList>> deferreds = + new ArrayList>>(Const.SALT_BUCKETS()); + for (int i = 0; i < Const.SALT_BUCKETS(); i++) { + deferreds.add(new ScannerCB(getScanner(i)).scan()); + } + return Deferred.group(deferreds).addCallback(new CompleteCB()); + } else { + return new ScannerCB(getScanner(0)).scan(); + } + } + @Override + public String toString() { + return "UID resolution callback"; + } + } + + return resolveUIDs().addCallbackDeferring(new UIDCB()); } /** - * Configures the scanner for iterating over the meta or data tables. If the - * metric has been set, then we scan a small slice of the table where the - * metric lies, otherwise we have to scan the whole table. If tags are - * given then we setup a row key regex - * @return A configured scanner + * Resolves the metric and tag strings to their UIDs + * @return A deferred to wait on for resolution to complete. */ - private Scanner getScanner(final StringBuilder tagv_filter) { - final Scanner scanner = tsdb.getClient().newScanner( - query.useMeta() ? tsdb.metaTable() : tsdb.dataTable()); + private Deferred resolveUIDs() { - // if a metric is given, we need to resolve it's UID and set the start key - // to the UID and the stop key to the next row by incrementing the UID. - if (query.getMetric() != null && !query.getMetric().isEmpty() && - !query.getMetric().equals("*")) { - final byte[] metric_uid = tsdb.getUID(UniqueIdType.METRIC, - query.getMetric()); - LOG.debug("Found UID (" + UniqueId.uidToString(metric_uid) + - ") for metric (" + query.getMetric() + ")"); - scanner.setStartKey(metric_uid); - long uid = UniqueId.uidToLong(metric_uid, TSDB.metrics_width()); - uid++; // TODO - see what happens when this rolls over - scanner.setStopKey(UniqueId.longToUID(uid, TSDB.metrics_width())); - } else { - LOG.debug("Performing full table scan, no metric provided"); + class TagsCB implements Callback> { + @Override + public Object call(final ArrayList ignored) throws Exception { + rowkey_regex = getRowKeyRegex(); + return null; + } } - if (query.getTags() != null && !query.getTags().isEmpty()) { - final List pairs = - new ArrayList(query.getTags().size()); - for (Pair tag : query.getTags()) { - final byte[] tagk = tag.getKey() != null && !tag.getKey().equals("*")? - tsdb.getUID(UniqueIdType.TAGK, tag.getKey()) : null; - final byte[] tagv = tag.getValue() != null && !tag.getValue().equals("*")? - tsdb.getUID(UniqueIdType.TAGV, tag.getValue()) : null; - pairs.add(new ByteArrayPair(tagk, tagv)); - } - // remember, tagks are sorted in the row key so we need to supply a sorted - // regex or matching will fail. - Collections.sort(pairs); - - final short name_width = TSDB.tagk_width(); - final short value_width = TSDB.tagv_width(); - final short tagsize = (short) (name_width + value_width); - - int index = 0; - final StringBuilder buf = new StringBuilder( - 22 // "^.{N}" + "(?:.{M})*" + "$" + wiggle - + ((13 + tagsize) // "(?:.{M})*\\Q" + tagsize bytes + "\\E" - * (pairs.size()))); - buf.append("(?s)^.{").append(TSDB.metrics_width()) - .append("}"); - if (!query.useMeta()) { - buf.append("(?:.{").append(Const.TIMESTAMP_BYTES).append("})*"); - } - buf.append("(?:.{").append(tagsize).append("})*"); - - // at the top of the list will be the null=tagv pairs. We want to compile - // a separate regex for them. - for (; index < pairs.size(); index++) { - if (pairs.get(index).getKey() != null) { - break; + class PairResolution implements Callback> { + @Override + public Object call(final ArrayList tags) throws Exception { + if (tags.size() < 2) { + throw new IllegalArgumentException("Somehow we received an array " + + "that wasn't two bytes in size! " + tags); } - - if (index > 0) { - buf.append("|"); - } - buf.append("(?:.{").append(name_width).append("})"); - buf.append("\\Q"); - addId(buf, pairs.get(index).getValue()); - buf.append("\\E"); - } - buf.append("(?:.{").append(tagsize).append("})*") - .append("$"); - - if (index > 0 && index < pairs.size()) { - // we had one or more tagvs to lookup AND we have tagk or tag pairs to - // filter on, so we dump the previous regex into the tagv_filter and - // continue on with a row key - tagv_filter.append(buf.toString()); - LOG.debug("Setting tagv filter: " + buf.toString()); - } else if (index >= pairs.size()) { - // in this case we don't have any tagks to deal with so we can just - // pass the previously compiled regex to the rowkey filter of the - // scanner - scanner.setKeyRegexp(buf.toString(), CHARSET); - LOG.debug("Setting scanner row key filter with tagvs only: " + - buf.toString()); + pairs.add(new ByteArrayPair(tags.get(0), tags.get(1))); + return Deferred.fromResult(null); } - - // catch any left over tagk/tag pairs - if (index < pairs.size()){ - buf.setLength(0); - buf.append("(?s)^.{").append(TSDB.metrics_width()) - .append("}"); - if (!query.useMeta()) { - buf.append("(?:.{").append(Const.TIMESTAMP_BYTES).append("})*"); + } + + class TagResolution implements Callback, Object> { + @Override + public Deferred call(final Object unused) throws Exception { + if (query.getTags() == null || query.getTags().isEmpty()) { + return Deferred.fromResult(null); } - ByteArrayPair last_pair = null; - for (; index < pairs.size(); index++) { - if (last_pair != null && last_pair.getValue() == null && - Bytes.memcmp(last_pair.getKey(), pairs.get(index).getKey()) == 0) { - // tagk=null is a wildcard so we don't need to bother adding - // tagk=tagv pairs with the same tagk. - LOG.debug("Skipping pair due to wildcard: " + pairs.get(index)); - } else if (last_pair != null && - Bytes.memcmp(last_pair.getKey(), pairs.get(index).getKey()) == 0) { - // in this case we're ORing e.g. "host=web01|host=web02" - buf.append("|\\Q"); - addId(buf, pairs.get(index).getKey()); - addId(buf, pairs.get(index).getValue()); - buf.append("\\E"); + pairs = Collections.synchronizedList( + new ArrayList(query.getTags().size())); + final ArrayList> deferreds = + new ArrayList>(pairs.size()); + + for (final Pair tags : query.getTags()) { + final ArrayList> deferred_tags = + new ArrayList>(2); + if (tags.getKey() != null && !tags.getKey().equals("*")) { + deferred_tags.add(tsdb.getUIDAsync(UniqueIdType.TAGK, tags.getKey())); } else { - if (last_pair != null) { - buf.append(")"); - } - // moving on to the next tagk set - buf.append("(?:.{6})*"); // catch tag pairs in between - buf.append("(?:"); - if (pairs.get(index).getKey() != null && - pairs.get(index).getValue() != null) { - buf.append("\\Q"); - addId(buf, pairs.get(index).getKey()); - addId(buf, pairs.get(index).getValue()); - buf.append("\\E"); - } else { - buf.append("\\Q"); - addId(buf, pairs.get(index).getKey()); - buf.append("\\E"); - buf.append("(?:.{").append(value_width).append("})+"); - } + deferred_tags.add(Deferred.fromResult(null)); + } + if (tags.getValue() != null && !tags.getValue().equals("*")) { + deferred_tags.add(tsdb.getUIDAsync(UniqueIdType.TAGV, tags.getValue())); + } else { + deferred_tags.add(Deferred.fromResult(null)); } - last_pair = pairs.get(index); + deferreds.add(Deferred.groupInOrder(deferred_tags) + .addCallback(new PairResolution())); } - buf.append(")(?:.{").append(tagsize).append("})*").append("$"); - - scanner.setKeyRegexp(buf.toString(), CHARSET); - LOG.debug("Setting scanner row key filter: " + buf.toString()); + return Deferred.group(deferreds).addCallback(new TagsCB()); } } + + class MetricCB implements Callback, byte[]> { + @Override + public Deferred call(final byte[] uid) throws Exception { + metric_uid = uid; + LOG.debug("Found UID (" + UniqueId.uidToString(metric_uid) + + ") for metric (" + query.getMetric() + ")"); + return new TagResolution().call(null); + } + } + + if (query.getMetric() != null && !query.getMetric().isEmpty() && + !query.getMetric().equals("*")) { + return tsdb.getUIDAsync(UniqueIdType.METRIC, query.getMetric()) + .addCallbackDeferring(new MetricCB()); + } else { + try { + return new TagResolution().call(null); + } catch (Exception e) { + return Deferred.fromError(e); + } + } + } + + /** Compiles a scanner with the given salt ID if salting is enabled AND we're + * not scanning the meta table. + * @param salt An ID for the salt bucket + * @return A scanner to send to HBase. + */ + private Scanner getScanner(final int salt) { + final Scanner scanner = tsdb.getClient().newScanner( + query.useMeta() ? tsdb.metaTable() : tsdb.dataTable()); + scanner.setFamily(query.useMeta() ? TSMeta.FAMILY : TSDB.FAMILY()); + + if (metric_uid != null) { + byte[] key; + if (query.useMeta() || Const.SALT_WIDTH() < 1) { + key = metric_uid; + } else { + key = new byte[Const.SALT_WIDTH() + TSDB.metrics_width()]; + System.arraycopy(RowKey.getSaltBytes(salt), 0, key, 0, Const.SALT_WIDTH()); + System.arraycopy(metric_uid, 0, key, Const.SALT_WIDTH(), metric_uid.length); + } + scanner.setStartKey(key); + long uid = UniqueId.uidToLong(metric_uid, TSDB.metrics_width()); + uid++; + if (uid < Internal.getMaxUnsignedValueOnBytes(TSDB.metrics_width())) { + // if random metrics are enabled we could see a metric with the max UID + // value. If so, we need to leave the stop key as null + if (query.useMeta() || Const.SALT_WIDTH() < 1) { + key = UniqueId.longToUID(uid, TSDB.metrics_width()); + } else { + key = new byte[Const.SALT_WIDTH() + TSDB.metrics_width()]; + System.arraycopy(RowKey.getSaltBytes(salt), 0, key, 0, Const.SALT_WIDTH()); + System.arraycopy(UniqueId.longToUID(uid, TSDB.metrics_width()), 0, + key, Const.SALT_WIDTH(), metric_uid.length); + } + scanner.setStopKey(key); + } + } + + if (rowkey_regex != null) { + scanner.setKeyRegexp(rowkey_regex, CHARSET); + if (LOG.isDebugEnabled()) { + LOG.debug("Scanner regex: " + QueryUtil.byteRegexToString(rowkey_regex)); + } + } + return scanner; } /** - * Appends the given ID to the given buffer, escaping where appropriate - * @param buf The string buffer to append to - * @param id The ID to append + * Constructs a row key regular expression to pass to HBase if the user gave + * some tags in the query + * @return The regular expression to use. */ - private static void addId(final StringBuilder buf, final byte[] id) { - boolean backslash = false; - for (final byte b : id) { - buf.append((char) (b & 0xFF)); - if (b == 'E' && backslash) { // If we saw a `\' and now we have a `E'. - // So we just terminated the quoted section because we just added \E - // to `buf'. So let's put a litteral \E now and start quoting again. - buf.append("\\\\E\\Q"); - } else { - backslash = b == '\\'; + private String getRowKeyRegex() { + final StringBuilder tagv_buffer = new StringBuilder(); + // remember, tagks are sorted in the row key so we need to supply a sorted + // regex or matching will fail. + Collections.sort(pairs); + + final short name_width = TSDB.tagk_width(); + final short value_width = TSDB.tagv_width(); + final short tagsize = (short) (name_width + value_width); + + int index = 0; + final StringBuilder buf = new StringBuilder( + 22 // "^.{N}" + "(?:.{M})*" + "$" + wiggle + + ((13 + tagsize) // "(?:.{M})*\\Q" + tagsize bytes + "\\E" + * (pairs.size()))); + buf.append("(?s)^.{").append(query.useMeta() ? TSDB.metrics_width() : + TSDB.metrics_width() + Const.SALT_WIDTH()) + .append("}"); + if (!query.useMeta()) { + buf.append("(?:.{").append(Const.TIMESTAMP_BYTES).append("})*"); + } + buf.append("(?:.{").append(tagsize).append("})*"); + + // at the top of the list will be the null=tagv pairs. We want to compile + // a separate regex for them. + for (; index < pairs.size(); index++) { + if (pairs.get(index).getKey() != null) { + break; + } + + if (index > 0) { + buf.append("|"); + } + buf.append("(?:.{").append(name_width).append("})"); + buf.append("\\Q"); + QueryUtil.addId(buf, pairs.get(index).getValue(), true); + } + buf.append("(?:.{").append(tagsize).append("})*") + .append("$"); + + if (index > 0 && index < pairs.size()) { + // we had one or more tagvs to lookup AND we have tagk or tag pairs to + // filter on, so we dump the previous regex into the tagv_filter and + // continue on with a row key + tagv_buffer.append(buf.toString()); + LOG.debug("Setting tagv filter: " + QueryUtil.byteRegexToString(buf.toString())); + } else if (index >= pairs.size()) { + // in this case we don't have any tagks to deal with so we can just + // pass the previously compiled regex to the rowkey filter of the + // scanner + LOG.debug("Setting scanner row key filter with tagvs only: " + + QueryUtil.byteRegexToString(buf.toString())); + if (tagv_buffer.length() > 0) { + tagv_filter = tagv_buffer.toString(); + } + return buf.toString(); + } + + // catch any left over tagk/tag pairs + if (index < pairs.size()){ // This condition is true whenever the first tagk in the pairs has a null value. + buf.setLength(0); + buf.append("(?s)^.{").append(query.useMeta() ? TSDB.metrics_width() : + TSDB.metrics_width() + Const.SALT_WIDTH()) + .append("}"); + if (!query.useMeta()) { + buf.append("(?:.{").append(Const.TIMESTAMP_BYTES).append("})"); + } + + ByteArrayPair last_pair = null; + for (; index < pairs.size(); index++) { + if (last_pair != null && last_pair.getValue() == null && + Bytes.memcmp(last_pair.getKey(), pairs.get(index).getKey()) == 0) { + // tagk=null is a wildcard so we don't need to bother adding + // tagk=tagv pairs with the same tagk. + LOG.debug("Skipping pair due to wildcard: " + pairs.get(index)); + } else if (last_pair != null && + Bytes.memcmp(last_pair.getKey(), pairs.get(index).getKey()) == 0) { + // in this case we're ORing e.g. "host=web01|host=web02" + buf.append("|\\Q"); + QueryUtil.addId(buf, pairs.get(index).getKey(), false); + QueryUtil.addId(buf, pairs.get(index).getValue(), true); + } else { + if (last_pair != null) { + buf.append(")"); + } + // moving on to the next tagk set + buf.append("(?:.{").append(tagsize).append("})*"); // catch tag pairs in between + buf.append("(?:"); + if (pairs.get(index).getKey() != null && + pairs.get(index).getValue() != null) { + buf.append("\\Q"); + QueryUtil.addId(buf, pairs.get(index).getKey(), false); + QueryUtil.addId(buf, pairs.get(index).getValue(), true); + } else { + buf.append("\\Q"); + QueryUtil.addId(buf, pairs.get(index).getKey(), true); + buf.append("(?:.{").append(value_width).append("})"); + } + } + last_pair = pairs.get(index); } + buf.append(")(?:.{").append(tagsize).append("})*").append("$"); + } + if (tagv_buffer.length() > 0) { + tagv_filter = tagv_buffer.toString(); } + return buf.toString(); } /** @param to_stdout Whether or not to dump to standard out as we scan */ public void setToStdout(final boolean to_stdout) { this.to_stdout = to_stdout; } + + @Override + public String toString() { + final StringBuilder buf = new StringBuilder(); + buf.append("query={") + .append(query) + .append("}, to_stdout=") + .append(to_stdout) + .append(", metric_uid=") + .append(metric_uid == null ? "null" : Arrays.toString(metric_uid)) + .append(", pairs=") + .append(pairs) + .append(", rowkey_regex=") + .append(rowkey_regex) + .append(", tagv_filter=") + .append(tagv_filter); + return buf.toString(); + + } } diff --git a/src/stats/QueryStats.java b/src/stats/QueryStats.java new file mode 100644 index 0000000000..b0edee5f88 --- /dev/null +++ b/src/stats/QueryStats.java @@ -0,0 +1,897 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.stats; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; + +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Objects; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; + +import net.opentsdb.core.Const; +import net.opentsdb.core.QueryException; +import net.opentsdb.core.TSQuery; +import net.opentsdb.utils.DateTime; +import net.opentsdb.utils.JSON; +import net.opentsdb.utils.Pair; + +/** + * This class stores information about OpenTSDB queries executed through the + * HTTP API. It maintains a list of running queries as well as a cache of the + * last {@code COMPLETED_QUERY_CACHE_SIZE} number of queries executed. The + * stats can be observed via /api/query/stats. + * + * When a query is executed, it should instantiate an object of this class. + * Once the query is completed, make sure to call {@link #markSent()}. + * + * The cache will store each query based on the combination of the client, query + * and the result code. If the same query was executed multiple times then it + * will increment the "executed" counter for the query in the cache. + * + * NOTE: Record everything in nano seconds, then convert to floating millis for + * serialization. + * @since 2.2 + */ +public class QueryStats { + private static final Logger LOG = LoggerFactory.getLogger(QueryStats.class); + private static final Logger QUERY_LOG = LoggerFactory.getLogger("QueryLog"); + + /** Determines how many query stats to keep in the cache */ + private static int COMPLETED_QUERY_CACHE_SIZE = 256; + + /** Whether or not to allow duplicate queries from the same endpoint to + * run simultaneously. */ + private static boolean ENABLE_DUPLICATES = true; + + /** Stores queries currently executing. If a thread doesn't call into + * markComplete then it's possible for this map to fill up. + * Hash is the remote + query */ + private static ConcurrentHashMap running_queries = + new ConcurrentHashMap(); + + /** Size limited cache of queries from the past. + * Hash is the remote + query + response code */ + private static Cache completed_queries = + CacheBuilder.newBuilder().maximumSize(COMPLETED_QUERY_CACHE_SIZE).build(); + + /** Start time for the query in nano seconds. Can be set post construction + * if necessary */ + private final long query_start_ns; + + /** Start timestamp for the query in millis for printing */ + private final long query_start_ms; + + /** When the query was marked completed in nanoseconds */ + private long query_completed_ts; + + /** The remote address as :, may be ipv6 */ + private final String remote_address; + + /** The TSQuery object that contains the query specification */ + private final TSQuery query; + + /** HTTP response when the query was completed, either successfully or failed */ + private HttpResponseStatus response; + + /** Set if the query terminated with an exception */ + private Throwable exception; + + /** How many times this exact query was executed. Only updated on completion */ + private long executed; + + /** The users (if known) who executed this query (could be pulled from a header) */ + private String user; + + /** Stats for the entire query */ + private final Map overall_stats; + + /** Hold a list of stats for the sub queries */ + private final Map> query_stats; + + /** Holds a list of stats for each scanner */ + private final Map>> scanner_stats; + + /** Hold a list of the region servers encountered for each scanner */ + private final Map>> scanner_servers; + + /** Holds a lis tof the scanner IDs for each scanner */ + private final Map> scanner_ids; + + /** Holds a copy of the headers from the request */ + private final Map headers; + + /** Whether or not the data was successfully sent to the client */ + private boolean sent_to_client; + + /** + * A list of statistics surrounding individual queries + */ + public enum QueryStat { + // Query Setup stats + STRING_TO_UID_TIME ("stringToUidTime", true), + + // Storage stats + COLUMNS_FROM_STORAGE ("columnsFromStorage", false), + ROWS_FROM_STORAGE ("rowsFromStorage", false), + BYTES_FROM_STORAGE ("bytesFromStorage", false), + SUCCESSFUL_SCAN ("successfulScan", false), + + // Single Scanner stats + DPS_PRE_FILTER ("dpsPreFilter", false), + ROWS_PRE_FILTER ("rowsPreFilter", false), + DPS_POST_FILTER ("dpsPostFilter", false), + ROWS_POST_FILTER ("rowsPostFilter", false), + SCANNER_UID_TO_STRING_TIME ("scannerUidToStringTime", true), + COMPACTION_TIME ("compactionTime", true), + HBASE_TIME ("hbaseTime", true), + UID_PAIRS_RESOLVED ("uidPairsResolved", false), + SCANNER_TIME ("scannerTime", true), + + // Overall Salt Scanner stats + SCANNER_MERGE_TIME ("saltScannerMergeTime", true), + + // Post Scan stats + QUERY_SCAN_TIME ("queryScanTime", true), + GROUP_BY_TIME ("groupByTime", true), + + // Serialization time stats + UID_TO_STRING_TIME ("uidToStringTime", true), + AGGREGATED_SIZE ("emittedDPs", false), + NAN_DPS ("nanDPs", false), + AGGREGATION_TIME ("aggregationTime", true), + SERIALIZATION_TIME ("serializationTime", true), + + // Final stats + PROCESSING_PRE_WRITE_TIME ("processingPreWriteTime", true), + TOTAL_TIME ("totalTime", true), + + // MAX and Agg Times + MAX_HBASE_TIME ("maxHBaseTime", true), + AVG_HBASE_TIME ("avgHBaseTime", true), + MAX_SALT_SCANNER_TIME ("maxScannerTime", true), + AVG_SALT_SCANNER_TIME ("avgScannerTime", true), + MAX_UID_TO_STRING ("maxUidToStringTime", true), + AVG_UID_TO_STRING ("avgUidToStringTime", true), + MAX_COMPACTION_TIME ("maxCompactionTime", true), + AVG_COMPACTION_TIME ("avgCompactionTime", true), + MAX_SCANNER_UID_TO_STRING_TIME ("maxScannerUidToStringTime", true), + AVG_SCANNER_UID_TO_STRING_TIME ("avgScannerUidToStringTime", true), + MAX_SCANNER_MERGE_TIME ("maxSaltScannerMergeTime", true), + AVG_SCANNER_MERGE_TIME ("avgSaltScannerMergeTime", true), + MAX_SCAN_TIME ("maxQueryScanTime", true), + AVG_SCAN_TIME ("avgQueryScanTime", true), + MAX_AGGREGATION_TIME ("maxAggregationTime", true), + AVG_AGGREGATION_TIME ("avgAggregationTime", true), + MAX_SERIALIZATION_TIME ("maxSerializationTime", true), + AVG_SERIALIZATION_TIME ("avgSerializationTime", true) + ; + + /** The serializable name for this enum */ + private final String stat_name; + /** Whether or not the stat is time based */ + private final boolean is_time; + + private QueryStat(final String stat_name, final boolean is_time) { + this.stat_name = stat_name; + this.is_time = is_time; + } + + @Override + public String toString() { + return stat_name; + } + } + + // always AVG, MAX in the pair order + static final Map> AGG_MAP = + new HashMap>(); + static { + AGG_MAP.put(QueryStat.HBASE_TIME, new Pair( + QueryStat.AVG_HBASE_TIME, QueryStat.MAX_HBASE_TIME)); + AGG_MAP.put(QueryStat.SCANNER_TIME, new Pair( + QueryStat.AVG_SALT_SCANNER_TIME, QueryStat.MAX_HBASE_TIME)); + AGG_MAP.put(QueryStat.UID_TO_STRING_TIME, new Pair( + QueryStat.MAX_UID_TO_STRING, QueryStat.MAX_UID_TO_STRING)); + AGG_MAP.put(QueryStat.SCANNER_UID_TO_STRING_TIME, new Pair( + QueryStat.MAX_SCANNER_UID_TO_STRING_TIME, + QueryStat.AVG_SCANNER_UID_TO_STRING_TIME)); + AGG_MAP.put(QueryStat.QUERY_SCAN_TIME, new Pair( + QueryStat.MAX_SCAN_TIME, QueryStat.AVG_SCAN_TIME)); + AGG_MAP.put(QueryStat.AGGREGATION_TIME, new Pair( + QueryStat.MAX_AGGREGATION_TIME, QueryStat.AVG_AGGREGATION_TIME)); + AGG_MAP.put(QueryStat.SERIALIZATION_TIME, new Pair( + QueryStat.MAX_SERIALIZATION_TIME, + QueryStat.AVG_SERIALIZATION_TIME)); + } + + /** + * Default CTor + * @param remote_address Remote address of the client + * @param query Query being executed + * @param headers The HTTP headers passed with the query + * @throws QueryException if the exact query is already running, e.g if the + * client submitted the same query twice + */ + public QueryStats(final String remote_address, final TSQuery query, + final Map headers) { + if (remote_address == null || remote_address.isEmpty()) { + throw new IllegalArgumentException("Remote address was null or empty"); + } + if (query == null) { + throw new IllegalArgumentException("Query object was null"); + } + this.remote_address = remote_address; + this.query = query; + this.headers = headers; // can be null + executed = 1; + query_start_ns = DateTime.nanoTime(); + query_start_ms = DateTime.currentTimeMillis(); + overall_stats = new ConcurrentHashMap(); + query_stats = new ConcurrentHashMap>(1); + scanner_stats = new ConcurrentHashMap>>(1); + scanner_servers = new ConcurrentHashMap>>(1); + scanner_ids = new ConcurrentHashMap>(1); + if (LOG.isDebugEnabled()) { + LOG.debug("New query for remote " + remote_address + " with hash " + + hashCode() + " on thread " + Thread.currentThread().getId()); + } + if (running_queries.putIfAbsent(this.hashCode(), this) != null) { + if (ENABLE_DUPLICATES) { + LOG.warn("Query " + query + " is already executing for endpoint: " + + remote_address); + } else { + throw new QueryException("Query is already executing for endpoint: " + + remote_address); + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("Successfully put new query for remote " + remote_address + + " with hash " + hashCode() + " on thread " + + Thread.currentThread().getId() + " w q " + query.toString()); + } + LOG.info("Executing new query=" + JSON.serializeToString(this)); + } + + /** + * Returns the hash based on the remote address and the query + */ + @Override + public int hashCode() { + return remote_address.hashCode() ^ query.hashCode(); + } + + /** + * Equals is based solely on the endpoint and the original query + */ + @Override + public boolean equals(final Object obj) { + if (obj == null) { + return false; + } + if (!(obj instanceof QueryStats)) { + return false; + } + if (obj == this) { + return true; + } + final QueryStats stats = (QueryStats)obj; + return Objects.equal(remote_address, stats.remote_address) + && Objects.equal(query, stats.query); + } + + @Override + public String toString() { + // have to hack it to get the details. By default we dump just the highest + // level of stats. + final Map details = new HashMap(); + details.put("queryStartTimestamp", getQueryStartTimestamp()); + details.put("queryCompletedTimestamp", getQueryCompletedTimestamp()); + details.put("exception", getException()); + details.put("httpResponse", getHttpResponse()); + details.put("numRunningQueries", getNumRunningQueries()); + details.put("query", getQuery()); + details.put("user", getUser()); + details.put("requestHeaders", getRequestHeaders()); + details.put("executed", getExecuted()); + details.put("stats", getStats(true, true)); + return JSON.serializeToString(details); + } + + /** + * Marks a query as completed successfully with the 200 HTTP response code + * without an exception. + * Moves it from the running map to the cache, updating the cache if it already + * existed. + */ + public void markSerializationSuccessful() { + markSerialized(HttpResponseStatus.OK, null); + } + + /** + * Marks a query as completed with the given HTTP code with exception and + * moves it from the running map to the cache, updating the cache if it + * already existed. + * @param response The HTTP response to log + * @param exception The exception thrown + */ + public void markSerialized(final HttpResponseStatus response, + final Throwable exception) { + this.exception = exception; + this.response = response; + + query_completed_ts = DateTime.currentTimeMillis(); + overall_stats.put(QueryStat.PROCESSING_PRE_WRITE_TIME, DateTime.nanoTime() - query_start_ns); + synchronized (running_queries) { + if (!running_queries.containsKey(this.hashCode())) { + if (!ENABLE_DUPLICATES) { + LOG.warn("Query was already marked as complete: " + this); + } + } + running_queries.remove(hashCode()); + if (LOG.isDebugEnabled()) { + LOG.debug("Removed completed query " + remote_address + " with hash " + + hashCode() + " on thread " + Thread.currentThread().getId()); + } + } + + aggQueryStats(); + + final int cache_hash = this.hashCode() ^ response.toString().hashCode(); + synchronized (completed_queries) { + final QueryStats old_query = completed_queries.getIfPresent(cache_hash); + if (old_query == null) { + completed_queries.put(cache_hash, this); + } else { + old_query.executed++; + } + } + } + + /** + * Marks the query as complete and logs it to the proper logs. This is called + * after the data has been sent to the client. + */ + public void markSent() { + sent_to_client = true; + overall_stats.put(QueryStat.TOTAL_TIME, DateTime.nanoTime() - query_start_ns); + LOG.info("Completing query=" + JSON.serializeToString(this)); + QUERY_LOG.info(this.toString()); + } + + /** Leaves the sent_to_client field as false when we were unable to write to + * the client end point. */ + public void markSendFailed() { + overall_stats.put(QueryStat.TOTAL_TIME, DateTime.nanoTime() - query_start_ns); + LOG.info("Completing query=" + JSON.serializeToString(this)); + QUERY_LOG.info(this.toString()); + } + + /** + * Builds a serializable map from the running and cached query maps to be + * returned to a caller. + * @return A map for serialization + */ + public static Map getRunningAndCompleteStats() { + Map root = new TreeMap(); + + if (running_queries.isEmpty()) { + root.put("running", Collections.emptyList()); + } else { + final List running = new ArrayList(running_queries.size()); + root.put("running", running); + + // don't need to lock the map beyond what the iterator will do implicitly + for (final QueryStats stats : running_queries.values()) { + final Map obj = new HashMap(10); + obj.put("query", stats.query); + obj.put("remote", stats.remote_address); + obj.put("user", stats.user); + obj.put("headers", stats.headers); + obj.put("queryStart", stats.query_start_ms); + obj.put("elapsed", DateTime.msFromNanoDiff(DateTime.nanoTime(), + stats.query_start_ns)); + running.add(obj); + } + } + + final Map completed = completed_queries.asMap(); + if (completed.isEmpty()) { + root.put("completed", Collections.emptyList()); + } else { + root.put("completed", completed.values()); + } + + return root; + } + + /** + * Fetches data about the running queries and status of queries in the cache + * @param collector The collector to write to + */ + public static void collectStats(final StatsCollector collector) { + collector.record("query.count", running_queries.size(), "type=running"); + } + + /** + * Add an overall statistic for the query (i.e. not associated with a sub + * query or scanner) + * @param name The name of the stat + * @param value The value to store + */ + public void addStat(final QueryStat name, final long value) { + overall_stats.put(name, value); + } + + /** + * Adds a stat for a sub query, replacing it if it exists. Times must be + * in nanoseconds. + * @param query_index The index of the sub query to update + * @param name The name of the stat to update + * @param value The value to set + */ + public void addStat(final int query_index, final QueryStat name, + final long value) { + Map qs = query_stats.get(query_index); + if (qs == null) { + qs = new HashMap(); + query_stats.put(query_index, qs); + } + qs.put(name, value); + } + + /** + * Aggregates the various stats from the lower to upper levels. This includes + * calculating max and average time values for stats marked as time based. + */ + public void aggQueryStats() { + // These are overall aggregations + final Map> overall_cumulations = + new HashMap>(); + + // scanner aggs + for (final Entry>> entry : + scanner_stats.entrySet()) { + final int query_index = entry.getKey(); + + final Map> cumulations = + new HashMap>(); + + for (final Entry> scanner : + entry.getValue().entrySet()) { + + for (final Entry stat : scanner.getValue().entrySet()) { + if (stat.getKey().is_time) { + if (!AGG_MAP.containsKey(stat.getKey())) { + // we're not aggregating this value + continue; + } + + // per query aggs + Pair pair = cumulations.get(stat.getKey()); + if (pair == null) { + pair = new Pair(0L, Long.MIN_VALUE); + cumulations.put(stat.getKey(), pair); + } + pair.setKey(pair.getKey() + stat.getValue()); + if (stat.getValue() > pair.getValue()) { + pair.setValue(stat.getValue()); + } + + // overall aggs required here for proper time averaging + pair = overall_cumulations.get(stat.getKey()); + if (pair == null) { + pair = new Pair(0L, Long.MIN_VALUE); + overall_cumulations.put(stat.getKey(), pair); + } + pair.setKey(pair.getKey() + stat.getValue()); + if (stat.getValue() > pair.getValue()) { + pair.setValue(stat.getValue()); + } + } else { + // only add counters for the per query maps as they'll be rolled + // up below into the overall + updateStat(query_index, stat.getKey(), stat.getValue()); + } + } + } + + // per query aggs + for (final Entry> cumulation : + cumulations.entrySet()) { + // names can't be null as we validate above that it exists + final Pair names = AGG_MAP.get(cumulation.getKey()); + addStat(query_index, names.getKey(), + (cumulation.getValue().getKey() / entry.getValue().size())); + addStat(query_index, names.getValue(), cumulation.getValue().getValue()); + } + } + + // handle the per scanner aggs + for (final Entry> cumulation : + overall_cumulations.entrySet()) { + // names can't be null as we validate above that it exists + final Pair names = AGG_MAP.get(cumulation.getKey()); + addStat(names.getKey(), + (cumulation.getValue().getKey() / + (scanner_stats.size() * + Const.SALT_WIDTH() > 0 ? Const.SALT_BUCKETS() : 1))); + addStat(names.getValue(), cumulation.getValue().getValue()); + } + overall_cumulations.clear(); + + // aggregate counters from the sub queries + for (final Map sub_query : query_stats.values()) { + for (final Entry stat : sub_query.entrySet()) { + if (stat.getKey().is_time) { + if (!AGG_MAP.containsKey(stat.getKey())) { + // we're not aggregating this value + continue; + } + Pair pair = overall_cumulations.get(stat.getKey()); + if (pair == null) { + pair = new Pair(0L, Long.MIN_VALUE); + overall_cumulations.put(stat.getKey(), pair); + } + pair.setKey(pair.getKey() + stat.getValue()); + if (stat.getValue() > pair.getValue()) { + pair.setValue(stat.getValue()); + } + } else if (overall_stats.containsKey(stat.getKey())) { + overall_stats.put(stat.getKey(), + overall_stats.get(stat.getKey()) + stat.getValue()); + } else { + overall_stats.put(stat.getKey(), stat.getValue()); + } + } + } + + for (final Entry> cumulation : + overall_cumulations.entrySet()) { + // names can't be null as we validate above that it exists + final Pair names = AGG_MAP.get(cumulation.getKey()); + overall_stats.put(names.getKey(), + (cumulation.getValue().getKey() / query_stats.size())); + overall_stats.put(names.getValue(), cumulation.getValue().getValue()); + } + } + + /** + * Increments the cumulative value for a cumulative stat. If it's a time then + * it must be in nanoseconds + * @param query_index The index of the sub query + * @param name The name of the stat + * @param value The value to add to the existing value + */ + public void updateStat(final int query_index, final QueryStat name, + final long value) { + Map qs = query_stats.get(query_index); + long cum_time = value; + if (qs == null) { + qs = new HashMap(); + query_stats.put(query_index, qs); + } + + if (qs.containsKey(name)) { + cum_time += qs.get(name); + } + qs.put(name, cum_time); + } + + /** + * Adds a value for a specific scanner for a specific sub query. If it's a time + * then it must be in nanoseconds. + * @param query_index The index of the sub query + * @param id The numeric ID of the scanner + * @param name The name of the stat + * @param value The value to add to the map + */ + public void addScannerStat(final int query_index, final int id, + final QueryStat name, final long value) { + Map> qs = scanner_stats.get(query_index); + if (qs == null) { + qs = new ConcurrentHashMap>( + Const.SALT_WIDTH() > 0 ? Const.SALT_BUCKETS() : 1); + scanner_stats.put(query_index, qs); + } + Map scanner_stat_map = qs.get(id); + if (scanner_stat_map == null) { + scanner_stat_map = new HashMap(); + qs.put(id, scanner_stat_map); + } + scanner_stat_map.put(name, value); + } + + /** + * Adds or overwrites the list of servers scanned by a scanner + * @param query_index The index of the sub query + * @param id The numeric ID of the scanner + * @param servers The list of servers encountered + */ + public void addScannerServers(final int query_index, final int id, + final Set servers) { + Map> query_servers = scanner_servers.get(query_index); + if (query_servers == null) { + query_servers = new ConcurrentHashMap>( + Const.SALT_WIDTH() > 0 ? Const.SALT_BUCKETS() : 1); + scanner_servers.put(query_index, query_servers); + } + query_servers.put(id, servers); + } + + /** + * Updates or adds a stat for a specific scanner. IF it's a time it must + * be in nanoseconds + * @param query_index The index of the sub query + * @param id The numeric ID of the scanner + * @param name The name of the stat + * @param value The value to update to the map + */ + public void updateScannerStat(final int query_index, final int id, + final QueryStat name, final long value) { + Map> qs = scanner_stats.get(query_index); + long cum_time = value; + if (qs == null) { + qs = new ConcurrentHashMap>(); + scanner_stats.put(query_index, qs); + } + Map scanner_stat_map = qs.get(id); + if (scanner_stat_map == null) { + scanner_stat_map = new HashMap(); + qs.put(id, scanner_stat_map); + } + + if (scanner_stat_map.containsKey(name)) { + cum_time += scanner_stat_map.get(name); + } + + scanner_stat_map.put(name, cum_time); + } + + /** + * Adds a scanner for a sub query to the stats along with the description of + * the scanner. + * @param query_index The index of the sub query + * @param id The numeric ID of the scanner + * @param string_id The description of the scanner + */ + public void addScannerId(final int query_index, final int id, + final String string_id) { + Map scanners = scanner_ids.get(query_index); + if (scanners == null) { + scanners = new ConcurrentHashMap(); + scanner_ids.put(query_index, scanners); + } + scanners.put(id, string_id); + } + + /** @return the start time of the query in nano seconds */ + public long queryStart() { + return query_start_ns; + } + + /** @param user The user who executed the query */ + public void setUser(final String user) { + this.user = user; + } + + /** @return The user who executed the query if known */ + public String getUser() { + return user; + } + + /** @return The multi-mapped set of request headers associated with the query */ + public Map getRequestHeaders() { + return headers; + } + + /** @return The number of currently running queries */ + public int getNumRunningQueries() { + return running_queries.size(); + } + + /** @return An exception associated with the query or null if the query + * returned successfully. */ + public String getException() { + if (exception == null) { + return "null"; + } + return exception.getMessage() + + (exception.getStackTrace() != null && exception.getStackTrace().length > 0 + ? "\n" + exception.getStackTrace()[0].toString() : ""); + } + + /** @return The HTTP status response for the query */ + public HttpResponseStatus getHttpResponse() { + return response; + } + + /** @return The number of times this query has been executed from the same + * endpoint. */ + public long getExecuted() { + return executed; + } + + /** @return The full query */ + public TSQuery getQuery() { + return query; + } + + /** @return When the query was received and started executing, in ms */ + public long getQueryStartTimestamp() { + return query_start_ms; + } + + /** @return When the query was marked as completed, in ms */ + public long getQueryCompletedTimestamp() { + return query_completed_ts; + } + + /** @return Whether or not the data was successfully sent to the client */ + public boolean getSentToClient() { + return sent_to_client; + } + + /** @return A map with the subset of query measurements, not including scanners + * or sub queries */ + public Map getStats() { + return getStats(false, false); + } + + /** + * Returns measurements of the given query + * @param with_scanners Whether or not to dump individual scanner stats + * @return A map with stats for the query + */ + public Map getStats(final boolean with_sub_queries, + final boolean with_scanners) { + final Map map = new TreeMap(); + + for (final Entry entry : overall_stats.entrySet()) { + if (entry.getKey().is_time) { + map.put(entry.getKey().toString(), DateTime.msFromNano(entry.getValue())); + } else { + map.put(entry.getKey().toString(), entry.getValue()); + } + } + + if (with_sub_queries) { + final Iterator>> it = + query_stats.entrySet().iterator(); + while (it.hasNext()) { + final Entry> entry = it.next(); + final Map qs = new HashMap(1); + qs.put(String.format("queryIdx_%02d", entry.getKey()), + getQueryStats(entry.getKey(), with_scanners)); + map.putAll(qs); + } + } + return map; + } + + /** + * Returns a map of stats for a single sub query + * @param index The sub query to fetch + * @param with_scanners Whether or not to print detailed stats for each scanner + * @return A map with stats to print or null if the sub query didn't have any + * data. + */ + public Map getQueryStats(final int index, + final boolean with_scanners) { + + final Map qs = query_stats.get(index); + if (qs == null) { + return null; + } + + final Map query_map = new TreeMap(); + query_map.put("queryIndex", index); + + final Iterator> stats_it = + qs.entrySet().iterator(); + while (stats_it.hasNext()) { + final Entry stat = stats_it.next(); + if (stat.getKey().is_time) { + query_map.put(stat.getKey().toString(), + DateTime.msFromNano(stat.getValue())); + } else { + query_map.put(stat.getKey().toString(), stat.getValue()); + } + } + + if (with_scanners) { + final Map> scanner_stats_map = + scanner_stats.get(index); + + final Map scanner_maps = new TreeMap(); + + query_map.put("scannerStats", scanner_maps); + if (scanner_stats_map != null) { + final Map scanners = scanner_ids.get(index); + final Iterator>> scanner_it = + scanner_stats_map.entrySet().iterator(); + while (scanner_it.hasNext()) { + final Entry> scanner = scanner_it.next(); + final Map scanner_map = new TreeMap(); + scanner_maps.put(String.format("scannerIdx_%02d", scanner.getKey()), + scanner_map); + final String id; + if (scanners != null) { + id = scanners.get(scanner.getKey()); + } else { + id = null; + } + scanner_map.put("scannerId", id); + /* Uncomment when AsyncHBase supports this + final Map> servers = scanner_servers.get(index); + if (servers != null) { + scanner_map.put("regionServers", servers.get(scanner.getKey())); + } else { + scanner_map.put("regionServers", null); + } + */ + final Iterator> scanner_stats_it = + scanner.getValue().entrySet().iterator(); + while (scanner_stats_it.hasNext()) { + final Entry scanner_stats = scanner_stats_it.next(); + if (!scanner_stats.getKey().is_time) { + scanner_map.put(scanner_stats.getKey().toString(), + scanner_stats.getValue()); + } else { + scanner_map.put(scanner_stats.getKey().toString(), + DateTime.msFromNano(scanner_stats.getValue())); + } + } + } + } + } + return query_map; + } + + /** @return A stat for the overall query or -1 if the stat didn't exist. */ + public long getStat(final QueryStat stat) { + if (!overall_stats.containsKey(stat)) { + return -1; + } + return overall_stats.get(stat); + } + + /** @return a timed stat for the overall query or NaN if the stat didn't exist */ + public double getTimeStat(final QueryStat stat) { + if (!stat.is_time) { + throw new IllegalArgumentException("The stat is not a time stat"); + } + if (!overall_stats.containsKey(stat)) { + return Double.NaN; + } + return DateTime.msFromNano(overall_stats.get(stat)); + } + + /** @param enable_dupes whether or not to allow duplicate queries to run */ + public static void setEnableDuplicates(final boolean enable_dupes) { + ENABLE_DUPLICATES = enable_dupes; + } +} diff --git a/src/stats/StatsCollector.java b/src/stats/StatsCollector.java index 6d002e1568..6170533fff 100644 --- a/src/stats/StatsCollector.java +++ b/src/stats/StatsCollector.java @@ -15,10 +15,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import net.opentsdb.utils.Config; + import java.net.InetAddress; import java.net.UnknownHostException; import java.util.HashMap; import java.util.Map; +import java.util.Map.Entry; /** * Receives various stats/metrics from the current process. @@ -34,6 +37,9 @@ public abstract class StatsCollector { private static final Logger LOG = LoggerFactory.getLogger(StatsCollector.class); + /** Tags to add to every stat emitted by the collector */ + private static Map global_tags; + /** Prefix to add to every metric name, for example `tsd'. */ protected final String prefix; @@ -42,7 +48,7 @@ public abstract class StatsCollector { /** Buffer used to build lines emitted. */ private final StringBuilder buf = new StringBuilder(); - + /** * Constructor. * @param prefix A prefix to add to every metric name, for example @@ -50,8 +56,13 @@ public abstract class StatsCollector { */ public StatsCollector(final String prefix) { this.prefix = prefix; + if (global_tags != null && !global_tags.isEmpty()) { + for (final Entry entry : global_tags.entrySet()) { + addExtraTag(entry.getKey(), entry.getValue()); + } + } } - + /** * Method to override to actually emit a data point. * @param datapoint A data point in a format suitable for a text @@ -240,4 +251,21 @@ public final void clearExtraTag(final String name) { extratags.remove(name); } + /** + * Parses the configuration to determine if any extra tags should be included + * with every stat emitted. + * @param config The config object to parse + * @throws IllegalArgumentException if the config is null. Other exceptions + * may be thrown if the config values are unparseable. + */ + public static final void setGlobalTags(final Config config) { + if (config == null) { + throw new IllegalArgumentException("Configuration cannot be null."); + } + + if (config.getBoolean("tsd.core.stats_with_port")) { + global_tags = new HashMap(1); + global_tags.put("port", config.getString("tsd.network.port")); + } + } } diff --git a/src/tools/ArgValueValidator.java b/src/tools/ArgValueValidator.java new file mode 100644 index 0000000000..5a5c7478fa --- /dev/null +++ b/src/tools/ArgValueValidator.java @@ -0,0 +1,30 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2010-2016 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.tools; + +import net.opentsdb.tools.ConfigArgP.ConfigurationItem; + +/** + *

    Title: ArgValueValidator

    + *

    Description: Defines a class that can validate a value instance of a declared ConfigMetaType

    + */ +public interface ArgValueValidator { + /** + * Validates the passed configuration item + * @param citem The item to validate + */ + public void validate(ConfigurationItem citem); + + /** A static exception that needs no stack trace or whatever */ + public static final Exception EX = new Exception(); +} diff --git a/src/tools/CliOptions.java b/src/tools/CliOptions.java index aeccb1bb36..60ddefc179 100644 --- a/src/tools/CliOptions.java +++ b/src/tools/CliOptions.java @@ -67,7 +67,7 @@ static void addAutoMetricFlag(final ArgP argp) { /** * Parse the command line arguments with the given options. - * @param options Options to parse in the given args. + * @param opt,ions Options to parse in the given args. * @param args Command line arguments to parse. * @return The remainder of the command line or * {@code null} if {@code args} were invalid and couldn't be parsed. @@ -77,7 +77,7 @@ static String[] parse(final ArgP argp, String[] args) { args = argp.parse(args); } catch (IllegalArgumentException e) { System.err.println("Invalid usage. " + e.getMessage()); - return null; + System.exit(2); } honorVerboseFlag(argp); return args; @@ -107,7 +107,7 @@ static final Config getConfig(final ArgP argp) throws IOException { config.setAutoMetric(config.getBoolean("tsd.core.auto_create_metrics")); return config; } - + /** * Copies the parsed command line options to the {@link Config} class * @param config Configuration instance to override @@ -120,6 +120,10 @@ static void overloadConfig(final ArgP argp, final Config config) { // map the overrides if (entry.getKey().toLowerCase().equals("--auto-metric")) { config.overrideConfig("tsd.core.auto_create_metrics", "true"); + } else if (entry.getKey().toLowerCase().equals("--disable-ui")) { + config.overrideConfig("tsd.core.enable_ui", "false"); + } else if (entry.getKey().toLowerCase().equals("--disable-api")) { + config.overrideConfig("tsd.core.enable_api", "false"); } else if (entry.getKey().toLowerCase().equals("--table")) { config.overrideConfig("tsd.storage.hbase.data_table", entry.getValue()); } else if (entry.getKey().toLowerCase().equals("--uidtable")) { @@ -140,16 +144,22 @@ static void overloadConfig(final ArgP argp, final Config config) { config.overrideConfig("tsd.core.flushinterval", entry.getValue()); } else if (entry.getKey().toLowerCase().equals("--backlog")) { config.overrideConfig("tsd.network.backlog", entry.getValue()); + } else if (entry.getKey().toLowerCase().equals("--read-only")) { + config.overrideConfig("tsd.mode", "ro"); } else if (entry.getKey().toLowerCase().equals("--bind")) { config.overrideConfig("tsd.network.bind", entry.getValue()); } else if (entry.getKey().toLowerCase().equals("--async-io")) { config.overrideConfig("tsd.network.async_io", entry.getValue()); } else if (entry.getKey().toLowerCase().equals("--worker-threads")) { config.overrideConfig("tsd.network.worker_threads", entry.getValue()); + } else if(entry.getKey().toLowerCase().equals("--use-otsdb-ts")) { + config.overrideConfig("tsd.storage.use_otsdb_timestamp", "true"); + } else if (entry.getKey().toLowerCase().equals("--dtc-ts")) { + config.overrideConfig("tsd.storage.get_date_tiered_compaction_start", entry.getValue()); } } } - + /** Changes the log level to 'WARN' unless --verbose is passed. */ private static void honorVerboseFlag(final ArgP argp) { if (argp.optionExists("--verbose") && !argp.has("--verbose") diff --git a/src/tools/CliUtils.java b/src/tools/CliUtils.java index 3fc78b3cc5..f67c966dd4 100644 --- a/src/tools/CliUtils.java +++ b/src/tools/CliUtils.java @@ -17,12 +17,17 @@ import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; +import net.opentsdb.core.Const; +import net.opentsdb.core.Internal; +import net.opentsdb.core.RowKey; import net.opentsdb.core.TSDB; import net.opentsdb.uid.UniqueId; import org.hbase.async.Bytes; import org.hbase.async.GetRequest; +import org.hbase.async.HBaseClient; import org.hbase.async.HBaseException; import org.hbase.async.KeyValue; import org.hbase.async.Scanner; @@ -140,6 +145,96 @@ static final Scanner getDataTableScanner(final TSDB tsdb, final long start_id, return scanner; } + /** + * Generates a list of Scanners to use for iterating over the full TSDB + * data table. If salting is enabled then {@link Const.SaltBukets()} scanners + * will be returned. If salting is disabled then {@link num_scanners} + * scanners will be returned. + * @param tsdb The TSDB to generate scanners from + * @param num_scanners The max number of scanners if salting is disabled + * @return A list of scanners to use for scanning the table. + */ + static final List getDataTableScanners(final TSDB tsdb, + final int num_scanners) { + if (num_scanners < 1) { + throw new IllegalArgumentException( + "Number of scanners must be 1 or more: " + num_scanners); + } + // TODO - It would be neater to get a list of regions then create scanners + // on those boundaries. We'll have to modify AsyncHBase for that to avoid + // creating lots of custom HBase logic in here. + final short metric_width = TSDB.metrics_width(); + final List scanners = new ArrayList(); + + if (Const.SALT_WIDTH() > 0) { + // salting is enabled so we'll create one scanner per salt for now + byte[] start_key = HBaseClient.EMPTY_ARRAY; + byte[] stop_key = HBaseClient.EMPTY_ARRAY; + + for (int i = 1; i < Const.SALT_BUCKETS() + 1; i++) { + // move stop key to start key + if (i > 1) { + start_key = Arrays.copyOf(stop_key, stop_key.length); + } + + if (i >= Const.SALT_BUCKETS()) { + stop_key = HBaseClient.EMPTY_ARRAY; + } else { + stop_key = RowKey.getSaltBytes(i); + } + final Scanner scanner = tsdb.getClient().newScanner(tsdb.dataTable()); + scanner.setStartKey(Arrays.copyOf(start_key, start_key.length)); + scanner.setStopKey(Arrays.copyOf(stop_key, stop_key.length)); + scanner.setFamily(TSDB.FAMILY()); + scanners.add(scanner); + } + + } else { + // No salt, just go by the max metric ID + long max_id = CliUtils.getMaxMetricID(tsdb); + if (max_id < 1) { + max_id = Internal.getMaxUnsignedValueOnBytes(metric_width); + } + final long quotient = max_id % num_scanners == 0 ? max_id / num_scanners : + (max_id / num_scanners) + 1; + + byte[] start_key = HBaseClient.EMPTY_ARRAY; + byte[] stop_key = new byte[metric_width]; + + for (int i = 0; i < num_scanners; i++) { + // move stop key to start key + if (i > 0) { + start_key = Arrays.copyOf(stop_key, stop_key.length); + } + + // setup the next stop key + final byte[] stop_id; + if ((i +1) * quotient > max_id) { + stop_id = null; + } else { + stop_id = Bytes.fromLong((i + 1) * quotient); + } + if ((i +1) * quotient >= max_id) { + stop_key = HBaseClient.EMPTY_ARRAY; + } else { + System.arraycopy(stop_id, stop_id.length - metric_width, stop_key, + 0, metric_width); + } + + final Scanner scanner = tsdb.getClient().newScanner(tsdb.dataTable()); + scanner.setStartKey(Arrays.copyOf(start_key, start_key.length)); + if (stop_key != null) { + scanner.setStopKey(Arrays.copyOf(stop_key, stop_key.length)); + } + scanner.setFamily(TSDB.FAMILY()); + scanners.add(scanner); + } + + } + + return scanners; + } + /** * Invokes the reflected {@code UniqueId.toBytes()} method with the given * string using the UniqueId character set. diff --git a/src/tools/ConfigArgP.java b/src/tools/ConfigArgP.java new file mode 100644 index 0000000000..92779b3348 --- /dev/null +++ b/src/tools/ConfigArgP.java @@ -0,0 +1,841 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2010-2016 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.tools; + +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.lang.management.ManagementFactory; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLConnection; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.script.Bindings; +import javax.script.ScriptContext; +import javax.script.ScriptEngine; +import javax.script.ScriptEngineManager; + +import net.opentsdb.utils.Config; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + *

    Title: ConfigArgP

    + *

    Description: Wraps {@link Config} and {@link ArgP} instances for a consolidated configuration and command line handler

    + */ + +public class ConfigArgP { + /** Static class logger */ + protected static final Logger LOG = LoggerFactory.getLogger(ConfigArgP.class); + /** The command line argument holder for all (default and extended) options */ + protected final ArgP argp = new ArgP(); + /** The command line argument holder for default options */ + protected final ArgP dargp = new ArgP(); + /** The non config option arguments */ + protected String[] nonOptionArgs = {}; + /** The base configuration */ + protected Config config; + /** The default configuration items keyed by the item key and cl-option */ + protected final Map defaultConfItems; + /** The command line args */ + protected final Set commandLineArgs; + + /** The raw configuration items loaded from the json file */ + protected final TreeSet configItemsByKey = new TreeSet(); + /** The raw configuration items loaded from the json file */ + protected final TreeSet configItemsByCl = new TreeSet(); + + /** The regex pattern to perform a substitution for + * ${<sysprop>:<default>} + * patterns in strings */ + public static final Pattern SYS_PROP_PATTERN = Pattern.compile("\\$\\{(.*?)(?::(.*?))??\\}"); + /** The regex pattern to perform a substitution for + * $[<javascript snippet>] + * patterns in strings */ + public static final Pattern JS_PATTERN = Pattern.compile("\\$\\[(.*?)\\]", Pattern.MULTILINE); + + /** The config key for the TSD RPC addin classes */ + public static final String TSD_RPC_ADDIN_KEY = "tsd.addins.rpcs"; + /** The config key for the TSD RPC addin classpath */ + public static final String TSD_RPC_ADDIN_CP_KEY = "tsd.addins.rpcs.classpath"; + + /** Indicates if we're on Windows, in which case the SysProp handling needs a few tweaks */ + public static final boolean IS_WINDOWS = System.getProperty("os.name").toLowerCase().contains("windows"); + /** The JavaScript Engine to interpret $[<javascript snippet>] values */ + protected static final ScriptEngine scriptEngine = new ScriptEngineManager().getEngineByExtension("js"); + /** The JavaScript bindings */ + protected static final Bindings bindings = scriptEngine.getBindings(ScriptContext.ENGINE_SCOPE); + + + + static { + // Initialize js bindings + Map map = new HashMap(); + for(String key: System.getProperties().stringPropertyNames()) { + map.put(key, System.getProperty(key)); + } + map.put("cores", ManagementFactory.getOperatingSystemMXBean().getAvailableProcessors()); + map.put("maxheap", Runtime.getRuntime().maxMemory()); + bindings.putAll(map); + bindings.putAll(System.getenv()); + bindings.put("bindings", bindings); + } + + /** + * Creates a new ConfigArgP + * @param args The command line arguments + */ + public ConfigArgP(String...args) { + InputStream is = null; + commandLineArgs = Collections.unmodifiableSet(new LinkedHashSet(Arrays.asList(args))); + try { + config = new NoLoadConfig(); + is = ConfigArgP.class.getClassLoader().getResourceAsStream("opentsdb.conf.json"); + ObjectMapper jsonMapper = new ObjectMapper(); + JsonNode root = jsonMapper.reader().readTree(is); + JsonNode configRoot = root.get("config-items"); + scriptEngine.eval("var config = " + configRoot.toString() + ";"); + processBindings(jsonMapper, root); + // Contains all the defaults + final ConfigurationItem[] loadedItems = jsonMapper.reader(ConfigurationItem[].class).readValue(configRoot); + // Contains all the defaults + final TreeSet items = new TreeSet(Arrays.asList(loadedItems)); + + Map tmpItems = new HashMap(items.size()); + for(Iterator iter = items.iterator(); iter.hasNext();) { + ConfigurationItem item = iter.next(); + if(tmpItems.containsKey(item.getKey())) throw new RuntimeException("opentsdb.conf.json contains duplicate key: [" + item.getKey() + "]"); + if(tmpItems.containsKey(item.getClOption())) throw new RuntimeException("opentsdb.conf.json contains duplicate clOption: [" + item.getClOption() + "]"); + tmpItems.put(item.getKey(), item); + tmpItems.put(item.getClOption(), item); +// if("BOOL".equals(item.getMeta())) iter.remove(); + } + defaultConfItems = Collections.unmodifiableMap(tmpItems); + LOG.debug("Loaded [{}] Configuration Items from opentsdb.conf.json", items.size()); + if(LOG.isDebugEnabled()) { + StringBuilder b = new StringBuilder("Configs:"); + for(ConfigurationItem ci: items) { + b.append("\n\t").append(ci.toString()); + } + b.append("\n"); + LOG.debug(b.toString()); + } + for(ConfigurationItem ci: items) { + LOG.debug("Processing CI [{}]", ci.getKey()); + if(ci.meta!=null) { + argp.addOption(ci.clOption, ci.meta, ci.description); + LOG.debug("Registered Meta ArgP cl:[{}], meta:[{}]", ci.clOption, ci.meta); + if("default".equals(ci.help)) dargp.addOption(ci.clOption, ci.meta, ci.description); + } else { + argp.addOption(ci.clOption, ci.description); + LOG.debug("Registered No Meta ArgP cl:[{}]", ci.clOption); + if("default".equals(ci.help)) dargp.addOption(ci.clOption, ci.description); + } + if(!configItemsByKey.add(ci)) { + throw new RuntimeException("Duplicate configuration key [" + ci.key + "] in opentsdb.conf.json. Programmer Error."); + } + if(!configItemsByCl.add(ci)) { + throw new RuntimeException("Duplicate configuration command line option [" + ci.clOption + "] in opentsdb.conf.json. Programmer Error."); + } + if(ci.getDefaultValue()!=null && !ci.getDefaultValue().trim().isEmpty()) { + ci.setValue(processConfigValue(ci.getDefaultValue())); + config.overrideConfig(ci.key, processConfigValue(ci.getValue())); + } + } + nonOptionArgs = applyArgs(args); + loadExternalConfigs(items); + config = new Config(config); + + } catch (Exception ex) { + if(ex instanceof IllegalArgumentException) { + throw (IllegalArgumentException)ex; + } + throw new RuntimeException("Failed to read opentsdb.conf.json", ex); + } finally { + if(is!=null) try { is.close(); } catch (Exception x) { /* No Op */ } + } + } + + /** + * Loads an externally defined configuration and/or a configuration include + * @param source The existing configuration items + */ + protected void loadExternalConfigs(final Set source) { + ConfigurationItem include = getConfigurationItemByKey(source, "tsd.core.config"); + if(include!=null && include.getValue()!=null) { + loadConfigSource(this, include.getValue()); + } + include = getConfigurationItemByKey(source, "tsd.core.config.include"); + if(include!=null && include.getValue()!=null) { + loadConfigSource(this, include.getValue()); + } + } + + /** + * Applies the properties from the named source to the main configuration + * @param config the main configuration to apply to + * @param source the name of the source to apply properties from + */ + protected static void loadConfigSource(ConfigArgP config, String source) { + Properties p = loadConfig(source); + Config c = config.getConfig(); + for(String key: p.stringPropertyNames()) { + String value = p.getProperty(key); + ConfigurationItem ci = config.getConfigurationItem(key); + if(ci!=null) { // if we recognize the key, validate it + ci.setValue(processConfigValue(value)); + } + c.overrideConfig(key, processConfigValue(value)); + } + } + + /** + * Loads properties from a file or url with the passed name + * @param name The name of the file or URL + * @return the loaded properties + */ + protected static Properties loadConfig(String name) { + try { + URL url = new URL(name); + return loadConfig(url); + } catch (Exception ex) { + return loadConfig(new File(name)); + } + } + + /** + * Loads properties from the passed input stream + * @param source The name of the source the properties are being loaded from + * @param is The input stream to load from + * @return the loaded properties + */ + protected static Properties loadConfig(String source, InputStream is) { + try { + Properties p = new Properties(); + p.load(is); + // trim the value as it may have trailing white-space + Set keys = p.stringPropertyNames(); + for(String key: keys) { + p.setProperty(key, p.getProperty(key).trim()); + } + return p; + } catch (IllegalArgumentException iae) { + throw iae; + } catch (Exception ex) { + throw new IllegalArgumentException("Failed to load configuration from [" + source + "]"); + } + } + + /** + * Loads properties from the passed file + * @param file The file to load from + * @return the loaded properties + */ + protected static Properties loadConfig(File file) { + InputStream is = null; + try { + is = new FileInputStream(file); + return loadConfig(file.getAbsolutePath(), is); + } catch (Exception ex) { + throw new IllegalArgumentException("Failed to load configuration from [" + file.getAbsolutePath() + "]"); + }finally { + if(is!=null) try { is.close(); } catch (Exception ex) { /* No Op */ } + } + } + + /** + * Loads properties from the passed URL + * @param url The url to load from + * @return the loaded properties + */ + protected static Properties loadConfig(URL url) { + InputStream is = null; + try { + URLConnection connection = url.openConnection(); + if(connection instanceof HttpURLConnection) { + HttpURLConnection conn = (HttpURLConnection)connection; + conn.setConnectTimeout(5000); + conn.setReadTimeout(5000); + } + is = connection.getInputStream(); + return loadConfig(url.toString(), is); + } catch (Exception ex) { + ex.printStackTrace(System.err); + throw new IllegalArgumentException("Failed to load configuration from [" + url + "]", ex); + }finally { + if(is!=null) try { is.close(); } catch (Exception ex) { /* No Op */ } + } + } + + + /** + *

    Title: NoLoadConfig

    + *

    Description: A {@link Config} override that does not trigger {@link Config#loadStaticVariables()} when {@link Config#overrideConfig(String, String)} is called.

    + */ + private static class NoLoadConfig extends Config { + /** + * {@inheritDoc} + * @see net.opentsdb.utils.Config#overrideConfig(java.lang.String, java.lang.String) + */ + @Override + public void overrideConfig(final String property, final String value) { + this.properties.put(property, value); + } + } + + + /** + * Parses the command line arguments, and where the options are recognized config items, the value is validated, then applied to the config + * @param clargs The command line arguments + * @return The un-applied command line arguments + */ + public String[] applyArgs(String[] clargs) { + LOG.debug("Applying Command Line Args {}", Arrays.toString(clargs)); + final List nonFlagArgs = new ArrayList(Arrays.asList(clargs)); + extractAndApplyFlags(nonFlagArgs); + + String[] args = nonFlagArgs.toArray(new String[0]); + String[] nonArgs = argp.parse(args); + LOG.debug("Applying Command Line ArgP {}", argp); + LOG.debug("configItemsByCl Keys: [{}]", configItemsByCl.toString()); + for(Map.Entry entry: argp.getParsed().entrySet()) { + String key = entry.getKey(), value = entry.getValue(); + ConfigurationItem citem = getConfigurationItemByClOpt(key); + LOG.debug("Loaded CI for command line option [{}]: Found:{}", key, citem!=null); + if("BOOL".equals(citem.getMeta())) { + citem.setValue(value!=null ? value : "true"); + } else { + if(value!=null) { + citem.setValue(processConfigValue(value)); + } + } +// log("CL Override [%s] --> [%s]", citem.getKey(), citem.getValue()); + config.overrideConfig(citem.getKey(), citem.getValue()); + } + return nonArgs; + } + + private void extractAndApplyFlags(final List args) { + for(Iterator iter = args.iterator(); iter.hasNext();) { + String arg = iter.next(); + if(!arg.startsWith("--")) continue; + ConfigurationItem ci = getConfigurationItemByClOpt(arg); + if(ci==null) continue; + if(ci.isBool()) { + LOG.debug("Enabling flag [{}]", ci.getClOption()); + config.overrideConfig(ci.getKey(), "true"); + iter.remove(); + } + } + } + + private ConfigurationItem getConfigurationItemByKey(final Set source, final String key) { + for(final ConfigurationItem ci: source) { + if(ci.getKey().equals(key)) return ci; + } + return null; + } + + /** + * Returns the {@link ConfigurationItem} with the passed key + * @param key The key of the item to fetch + * @return The matching ConfigurationItem or null if one was not found + */ + public ConfigurationItem getConfigurationItem(final String key) { + if(key==null || key.trim().isEmpty()) throw new IllegalArgumentException("The passed key was null or empty"); + return getConfigurationItemByKey(configItemsByKey, key); + } + + /** + * Returns the {@link ConfigurationItem} with the passed cl-option + * @param clopt The cl-option of the item to fetch + * @return The matching ConfigurationItem or null if one was not found + */ + public ConfigurationItem getConfigurationItemByClOpt(final String clopt) { + if(clopt==null || clopt.trim().isEmpty()) throw new IllegalArgumentException("The passed cl-opt was null or empty"); + for(final ConfigurationItem ci: configItemsByCl) { + if(ci.getClOption().equals(clopt)) return ci; + } + return null; + + } + + + /** + * {@inheritDoc} + */ + public String toString() { + StringBuilder b = new StringBuilder(); + for(ConfigurationItem ci: configItemsByKey) { + b.append(ci.toString()).append("\n"); + } + return b.toString(); + } + + /** + * Returns a default usage banner with optional prefixed messages, one per line. + * @param msgs The optional message + * @return the formatted usage banner + */ + public String getDefaultUsage(String...msgs) { + StringBuilder b = new StringBuilder("\n"); + for(String msg: msgs) { + b.append(msg).append("\n"); + } + b.append(dargp.usage()); + return b.toString(); + } + + /** + * Returns an extended usage banner with optional prefixed messages, one per line. + * @param msgs The optional message + * @return the formatted usage banner + */ + public String getExtendedUsage(String...msgs) { + StringBuilder b = new StringBuilder("\n"); + for(String msg: msgs) { + b.append(msg).append("\n"); + } + b.append(argp.usage()); + return b.toString(); + } + + + /** + * System out logger + * @param format The message format + * @param args The message arg tokens + */ + public static void log(String format, Object...args) { + System.out.println(String.format(format, args)); + } + + /** + * Performs sys-prop and js evals on the passed value + * @param text The value to process + * @return the processed value + */ + public static String processConfigValue(CharSequence text) { + final String pv = evaluate(tokenReplaceSysProps(text)); + return (pv==null || pv.trim().isEmpty()) ? null : pv; + } + + /** + * Replaces all matched tokens with the matching system property value or a configured default + * @param text The text to process + * @return The substituted string + */ + public static String tokenReplaceSysProps(CharSequence text) { + if(text==null) return null; + Matcher m = SYS_PROP_PATTERN.matcher(text); + StringBuffer ret = new StringBuffer(); + while(m.find()) { + String replacement = decodeToken(m.group(1), m.group(2)==null ? "" : m.group(2)); + if(replacement==null) { + throw new IllegalArgumentException("Failed to fill in SystemProperties for expression with no default [" + text + "]"); + } + if(IS_WINDOWS) { + replacement = replacement.replace(File.separatorChar , '/'); + } + m.appendReplacement(ret, replacement); + } + m.appendTail(ret); + return ret.toString(); + } + + /** + * Evaluates JS expressions defines as configuration values + * @param text The value of a configuration item to evaluate for JS expressions + * @return The passed value with any embedded JS expressions evaluated and replaced + */ + public static String evaluate(CharSequence text) { + if(text==null) return null; + Matcher m = JS_PATTERN.matcher(text); + StringBuffer ret = new StringBuffer(); + final boolean isNas = scriptEngine.getFactory().getEngineName().toLowerCase().contains("nashorn"); + while(m.find()) { + String source = (isNas ? + "load(\"nashorn:mozilla_compat.js\");\nimportPackage(java.lang); " + : + "\nimportPackage(java.lang); " + ) + + m.group(1); + try { + Object obj = scriptEngine.eval(source, bindings); + if(obj!=null) { + //log("Evaled [%s] --> [%s]", source, obj); + m.appendReplacement(ret, obj.toString()); + } else { + m.appendReplacement(ret, ""); + } + } catch (Exception ex) { + ex.printStackTrace(System.err); + throw new IllegalArgumentException("Failed to evaluate expression [" + text + "]"); + } + } + m.appendTail(ret); + return ret.toString(); + } + + /** + * Attempts to decode the passed dot delimited as a system property, and if not found, attempts a decode as an + * environmental variable, replacing the dots with underscores. e.g. for the key: buffer.size.max, + * a system property named buffer.size.max will be looked up, and then an environmental variable + * named buffer.size.max will be looked up. + * @param key The dot delimited key to decode + * @param defaultValue The default value returned if neither source can decode the key + * @return the decoded value or the default value if neither source can decode the key + */ + public static String decodeToken(String key, String defaultValue) { + String value = System.getProperty(key, System.getenv(key.replace('.', '_'))); + return value!=null ? value : defaultValue; + } + + + /** + *

    Title: ConfigurationItem

    + *

    Description: A container class for deserialized configuration items from opentsdb.conf.json.

    + */ + public static class ConfigurationItem implements Comparable { + /** The internal configuration key */ + @JsonProperty("key") + protected String key; + /** The command line option key that maps to this item */ + @JsonProperty("cl-option") + protected String clOption; + /** The original value, loaded from opentsdb.conf.json, and never overwritten */ + @JsonProperty("defaultValue") + protected String defaultValue; + /** A description of the configuration item */ + @JsonProperty("description") + protected String description; + /** The command line help level at which this item will be displayed ('default' or 'extended') */ + @JsonProperty("help") + protected String help; + /** The meta symbol representing the type of value expected for a parameterized command line arg */ + @JsonProperty("meta") + protected String meta; + + /** The decoded or overriden value */ + protected String value; + + /** + * Creates a new ConfigurationItem + */ + public ConfigurationItem() {} + + /** + * Creates a new ConfigurationItem + * @param key The internal configuration key + * @param clOption The command line option key that maps to this item + * @param defaultValue The original value, loaded from opentsdb.conf.json, and never overwritten + * @param description A description of the configuration item + * @param help The command line help level at which this item will be displayed ('default' or 'extended') + * @param meta The meta symbol representing the type of value expected for a parameterized command line arg + */ + public ConfigurationItem(String key, String clOption, + String defaultValue, String description, String help, + String meta) { + this.key = key; + this.clOption = clOption; + this.defaultValue = defaultValue; + this.description = description; + this.help = help; + this.meta = meta; + } + + /** + * Validates the value + */ + public void validate() { + if(meta!=null && value!=null) { + ConfigMetaType.byName(meta, key).validate(this); + } + } + + + + /** + * Returns a descriptive name with the cl option and key + * @return a descriptive name + */ + public String getName() { + return String.format("cl: %s, key: %s", clOption, key); + } + + + + /** + * Returns the item key name + * @return the itemName + */ + public String getKey() { + return key; + } + + /** + * Returns the command line option mapping to this item + * @return the clOption + */ + public String getClOption() { + return clOption; + } + + /** + * Returns the item current value + * @return the value + */ + public String getValue() { + return value!=null ? value : defaultValue; + } + + /** + * Indicates if this ConfigurationItem is a BOOL meta type + * @return true if this ConfigurationItem is a BOOL meta type, false otherwise + */ + public boolean isBool() { + return "BOOL".equals(this.meta); + } + + + /** + * Sets a new value for this item + * @param newValue The new value + */ + public void setValue(final String newValue) { + final String currValue = newValue; + value = newValue.trim(); + try { + validate(); + } catch (IllegalArgumentException ex) { + value = currValue; + throw ex; + } + } + + /** + * Returns the original raw value loaded from opentsdb.conf.json + * @return the original raw value + */ + public String getDefaultValue() { + return defaultValue; + } + + /** + * Returns the item description + * @return the description + */ + public String getDescription() { + return description; + } + + /** + * Returns the help level for this option + * @return the help + */ + public String getHelp() { + return help; + } + + /** + * Returns the meta symbol + * @return the meta + */ + public String getMeta() { + return meta; + } + + /** + * {@inheritDoc} + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return String + .format("ConfigurationItem [key=%s, clOption=%s, value=%s, description=%s, help=%s, meta=%s, defaultValue=%s]", + key, clOption, value, description, help, + meta, defaultValue); + } + + /** + * {@inheritDoc} + * @see java.lang.Object#hashCode() + */ + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((key == null) ? 0 : key.hashCode()); + return result; + } + + /** + * {@inheritDoc} + * @see java.lang.Object#equals(java.lang.Object) + */ + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + ConfigurationItem other = (ConfigurationItem) obj; + if (key == null) { + if (other.key != null) + return false; + } else if (!key.equals(other.key)) + return false; + return true; + } + + /** + *

    Sorts {@link ConfigurationItem}s by the underlying {@link ConfigMetaType}

    + * {@inheritDoc} + * @see java.lang.Comparable#compareTo(java.lang.Object) + */ + @Override + public int compareTo(final ConfigurationItem other) { + if(this==other) return 0; + if((other.meta==null || other.meta.isEmpty()) && (meta==null || meta.isEmpty())) { + return this.key.compareTo(other.key); + } + if(other.meta==null || other.meta.isEmpty()) { + return 1; + } + if(meta==null || meta.isEmpty()) { + return -1; + } + + final ConfigMetaType otherType = ConfigMetaType.byName(other.meta, other.key); + final ConfigMetaType thisType = ConfigMetaType.byName(meta, key); + int c = thisType.compareTo(otherType); + if(c==0) { + c = this.key.compareTo(other.key); + } + return c; + } + } + + + /** + * Returns the command line argument processor + * @return the argp + */ + public ArgP getArgp() { + return argp; + } + + /** + * Returns the command line argument holder for default options + * @return the command line argument holder + */ + public ArgP getDargp() { + return dargp; + } + + /** + * Returns the TSDB config instance + * @return the config + */ + public Config getConfig() { + return config; + } + + /** + * Returns the non config option arguments + * @return the non config option arguments + */ + public String[] getNonOptionArgs() { + return nonOptionArgs; + } + + /** + * Returns the default {@link ConfigurationItem} for the passed name + * which can be the item's key or cl-option + * @param name The item's key or cl-option + * @return The named ConfigurationItem or null if one was not found + */ + public ConfigurationItem getDefaultItem(final String name) { + if(name==null) throw new IllegalArgumentException("The passed name was null"); + return defaultConfItems.get(name); + } + + + /** + * Determines if the passed key is contained in the non option args + * @param nonOptionKey The non option key to check for + * @return true if the passed key is present, false otherwise + */ + public boolean hasNonOption(String nonOptionKey) { + if(nonOptionArgs==null || nonOptionArgs.length==0 || nonOptionKey==null || nonOptionKey.trim().isEmpty()) return false; + return Arrays.binarySearch(nonOptionArgs, nonOptionKey) >= 0; + } + + /** + * Indicates if the passed arg was a command line option + * @param arg The arg to test for + * @return true if the passed arg was a command line option, false otherwise + */ + public boolean isClArg(final String arg) { + return commandLineArgs.contains(arg); + } + + /** + * Checks the opentsdb.conf.json document to see if it has a bindings segment + * which contains JS statements to evaluate which will prime variables used by the configuration. + * @param jsonMapper The JSON mapper + * @param root The root opentsdb.conf.json document + */ + protected void processBindings(ObjectMapper jsonMapper, JsonNode root) { + String script = null; + try { + if(root.has("bindings")) { + JsonNode bindingsNode = root.get("bindings"); + if(bindingsNode.isArray()) { + String[] jsLines = jsonMapper.reader(String[].class).readValue(bindingsNode); + StringBuilder b = new StringBuilder(); + for(String s: jsLines) { + b.append(s).append("\n"); + } + script = b.toString(); + scriptEngine.eval(script); + LOG.debug("Successfully evaluated [{}] lines of JS in bindings", jsLines.length); + } + } + } catch (Exception ex) { + throw new IllegalArgumentException("Failed to evaluate opentsdb.conf.json javascript binding [" + script + "]", ex); + } + } + + +} \ No newline at end of file diff --git a/src/tools/ConfigMetaType.java b/src/tools/ConfigMetaType.java new file mode 100644 index 0000000000..9065a9df98 --- /dev/null +++ b/src/tools/ConfigMetaType.java @@ -0,0 +1,564 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2010-2012 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.tools; + +import java.io.File; +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.net.URL; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.TimeZone; +import java.util.regex.Pattern; + +import javax.management.MBeanServer; +import javax.management.ObjectName; +import javax.management.loading.MLet; + +import net.opentsdb.tools.ConfigArgP.ConfigurationItem; + +/** + *

    Title: ConfigMetaType

    + *

    Description: Defines the recognized meta types for command line argument values

    + */ + +public enum ConfigMetaType implements ArgValueValidator { + /** An HBase table name */ + TABLE("An HBase table name", new StringValidator("TABLE")), + /** A zookeeper quorum spec */ + SPEC("A zookeeper quorum spec", new StringValidator("SPEC")), + /** A class name */ + CLASS("A class name", new StringValidator("CLASS")), + /** A class name of a class loadable at boot time */ + BCLASS("A class name of a class loadable at boot time", new BootLoadableClass("BCLASS")), + /** A comma separated list of values */ + LIST("A comma separated list of values", new StringValidator("List of comma sep values")), + /** An existing file */ + EFILE("An existing file", new FileSystemValidator(false, true)), + /** A list of existing files or URLs */ + FILELIST("A list of existing files or accessible URLs", new FileListValidator()), + /** A list of existing files or URLs that comprise a classpath, and when this option is loaded, a ClassLoader MBean will be registered */ + CLASSPATH("A list of comma separated existing files or accessible URLs", new ClassPathValidator()), + /** A file, optionally existing */ + FILE("A file, optionally existing", new FileSystemValidator(false, false)), + /** An existing directory */ + EDIR("An existing directory", new FileSystemValidator(true, true)), + /** A fully qualified file name where the parent directory must exist, but the file is optional */ + EDIRFILE("An existing directory", new DirFileSystemValidator()), + /** A valid URL or readable file */ + URLORFILE("A valid URL or readable file", new URLOrFileValidator()), + /** A directory, optionally existing */ + DIR("A directory, optionally existing", new FileSystemValidator(true, false)), + /** A host name or address */ + ADDR("A host name or address", new StringValidator("ADDR")), // we could use InetAddress.getByName('') but it might be really slow. + /** An integer value */ + INT("An integer value", new IntegerValidator(Integer.MAX_VALUE)), + /** A positive integer value */ + POSINT("A positive integer value", new IntegerValidator(0)), + /** A non zero positive integer value */ + GTZEROINT("A non zero positive integer value", new IntegerValidator(1)), + /** A boolean value (true|false) */ + BOOL("A boolean value (true|false)", new BooleanValidator()), + /** A znode path name */ + ZPATH("A znode path name", new StringValidator("ZPATH")), + /** The read write mode */ + RWMODE("Read/Write mode specification", new ReadWriteModeValidator()), + /** A time zone. e.g. "America/Los_Angeles"*/ + TIMEZONE("A time zone name", new TimeZoneValidator()), + /** A list of comma separated class names that should be loadable with (or without) the assistance of a {@link #CLASSPATH} configured classloader */ + BCLASSLIST("A comma separated list of loadable classes", new ClasspathConfiguredClassList()); + + /** The platform MBeanServer */ + public static final MBeanServer server = ManagementFactory.getPlatformMBeanServer(); + /** Comma splitter */ + public static final Pattern COMMA_SPLITTER = Pattern.compile(","); + + + /** + * Decodes the passed name with trim and upper + * @param name The name to decode + * @param key The key of the item (for error reporting) + * @return the decoded value + */ + public static ConfigMetaType byName(CharSequence name, String key) { + if(name==null) throw new IllegalArgumentException("Null ConfigMetaType"); + String cname = name.toString().toUpperCase().trim(); + try { + return ConfigMetaType.valueOf(cname); + } catch (Exception ex) { + throw new IllegalArgumentException("Invalid ConfigMetaType [" + cname + "]. Key: [" + key + "]"); + } + } + + private ConfigMetaType(String description, ArgValueValidator validator) { + this.description = description; + this.validator = validator; + } + + /** A short description of the meta type */ + public final String description; + /** A validator instance for the meta type */ + public final ArgValueValidator validator; + + + /** + * {@inheritDoc} + * @see net.opentsdb.tools.ArgValueValidator#validate(net.opentsdb.tools.ConfigArgP.ConfigurationItem) + */ + @Override + public void validate(final ConfigurationItem citem) { + validator.validate(citem); + } + + /** + *

    Title: ReadWriteModeValidator

    + *

    Description: Validator for ReadWrite Modes

    + */ + public static class ReadWriteModeValidator implements ArgValueValidator { + /** The supported mode codes */ + private static final Set MODES = Collections.unmodifiableSet(new HashSet(Arrays.asList( + "rw", // READ AND WRITE + "wo", // WRITE ONLY + "ro" // READ ONLY + ))); + /** + * Creates a new ReadWriteModeValidator + */ + public ReadWriteModeValidator() { + + } + + /** + * {@inheritDoc} + * @see net.opentsdb.tools.ArgValueValidator#validate(net.opentsdb.tools.ConfigArgP.ConfigurationItem) + */ + @Override + public void validate(final ConfigurationItem citem) { + try { + final String _mode = citem.getValue(); + if(!MODES.contains(_mode)) throw new Exception(); + } catch (Exception ex) { + throw new IllegalArgumentException("Invalid ReadWrite Mode [" + citem.getValue() + "] for " + citem.getName()); + } + } + } + + /** + *

    Title: ClassListValidator

    + *

    Description: Validator for loadable class lists

    + */ + public static class ClassListValidator implements ArgValueValidator { + /** + * Creates a new ClassListValidator + */ + public ClassListValidator() { + + } + + /** + * {@inheritDoc} + * @see net.opentsdb.tools.ArgValueValidator#validate(net.opentsdb.tools.ConfigArgP.ConfigurationItem) + */ + @Override + public void validate(final ConfigurationItem citem) { + final String val = citem.getValue(); + if(val.trim().isEmpty()) return; + final String[] classNames = COMMA_SPLITTER.split(val.trim()); + for(String className: classNames) { + className = className.trim(); + if(className.isEmpty()) continue; + try { + Class.forName(className); + } catch (Exception ex) { + throw new IllegalArgumentException("Failed to load class [" + className + "] defined in configuration item [" + citem.getName() + "]", ex); + } + } + } + } + + + /** + *

    Title: IntegerValidator

    + *

    Description: Validator for integers

    + */ + public static class IntegerValidator implements ArgValueValidator { + private final int min; + /** + * Creates a new IntegerValidator + * @param min The minumum value of the integer + */ + public IntegerValidator(final int min) { + this.min = min; + } + /** + * {@inheritDoc} + * @see net.opentsdb.tools.ArgValueValidator#validate(net.opentsdb.tools.ConfigArgP.ConfigurationItem) + */ + @Override + public void validate(final ConfigurationItem citem) { + try { + int i = Integer.parseInt(citem.getValue()); + if(i < min) throw EX; + } catch (Exception ex) { + throw new IllegalArgumentException("Invalid Integer value [" + citem.getValue() + "] for " + citem.getName()); + } + } + } + + /** + *

    Title: BooleanValidator

    + *

    Description: Validator for booleans (true|false)

    + */ + public static class BooleanValidator implements ArgValueValidator { + /** + * {@inheritDoc} + * @see net.opentsdb.tools.ArgValueValidator#validate(net.opentsdb.tools.ConfigArgP.ConfigurationItem) + */ + @Override + public void validate(final ConfigurationItem citem) { + if(!"true".equalsIgnoreCase(citem.getValue()) && !"false".equalsIgnoreCase(citem.getValue())) { + throw new IllegalArgumentException("Invalid Boolean value [" + citem.getValue() + "] for " + citem.getName()); + } + } + } + + /** + *

    Title: URLOrFileValidator

    + *

    Description: Validator for URLs or Files

    + */ + public static class URLOrFileValidator implements ArgValueValidator { + /** + * {@inheritDoc} + * @see net.opentsdb.tools.ArgValueValidator#validate(net.opentsdb.tools.ConfigArgP.ConfigurationItem) + */ + @Override + public void validate(final ConfigurationItem citem) { + try { + new URL(citem.getValue()); + } catch (Exception ex) { + File f = new File(citem.getValue()); + if(!f.exists() || !f.isFile() || !f.canRead()) { + throw new IllegalArgumentException("Invalid URL or File [" + citem.getValue() + "]", ex); + } + } + } + } + + + /** + *

    Title: StringValidator

    + *

    Description: Empty validator for generic string values, used when we don't have a decent or practical validation routine

    + */ + public static class StringValidator implements ArgValueValidator { + /** Informational name */ + protected final String name; + + /** + * Creates a new StringValidator + * @param name The name of the type + */ + public StringValidator(String name) { + this.name = name; + } + /** + * {@inheritDoc} + * @see net.opentsdb.tools.ArgValueValidator#validate(net.opentsdb.tools.ConfigArgP.ConfigurationItem) + */ + @Override + public void validate(final ConfigurationItem citem) { + if(citem.getValue()==null || citem.getValue().trim().isEmpty()) { + throw new IllegalArgumentException("Null or empty " + name + " value for " + citem.getName()); + } + } + } + + /** + *

    Title: BootLoadableClassValidator

    + *

    Description: A validator for boot time loadable class names

    + */ + public static class BootLoadableClass extends StringValidator { + /** + * Creates a new BootLoadableClass + * @param name The name of the type + */ + public BootLoadableClass(String name) { + super(name); + } + /** + * {@inheritDoc} + * @see net.opentsdb.tools.ConfigMetaType.StringValidator#validate(net.opentsdb.tools.ConfigArgP.ConfigurationItem) + */ + @Override + public void validate(final ConfigurationItem citem) { + super.validate(citem); + try { + Class.forName(citem.getValue().trim(), true, ConfigMetaType.class.getClassLoader()); + } catch (Exception ex) { + throw new IllegalArgumentException("Failed to load boot time class [" + citem.getValue() + "] for " + citem.getName()); + } + } + } + + /** + *

    Title: BootLoadableClassValidator

    + *

    Description: A validator for a comma separated list of classnames for which a supplementary + * classpath has been specified in a {@link #FILELIST} configuration item named the same as this one + * but with .classpath appended.

    + */ + public static class ClasspathConfiguredClassList implements ArgValueValidator { + /** The template for the classloader MBean's ObjectName */ + public static final String CLASSLOADER_OBJECTNAME = "net.opentsdb.classpath:type=ClassLoader,name=%s.classpath"; + + /** + * Creates a new ClasspathConfiguredClassList + */ + public ClasspathConfiguredClassList() { + + } + /** + * {@inheritDoc} + * @see net.opentsdb.tools.ConfigMetaType.StringValidator#validate(net.opentsdb.tools.ConfigArgP.ConfigurationItem) + */ + @Override + public void validate(final ConfigurationItem citem) { + Set classNames = new LinkedHashSet(Arrays.asList(COMMA_SPLITTER.split(citem.getValue()))); + if(classNames.isEmpty()) return; + for(final Iterator iter = classNames.iterator(); iter.hasNext();) { + String fileName = iter.next().trim(); + if(fileName.isEmpty()) iter.remove(); + } + if(classNames.isEmpty()) return; + String className = null; + try { + final ClassLoader CL; + final ObjectName on = new ObjectName(String.format(CLASSLOADER_OBJECTNAME, citem.getKey())); + if(server.isRegistered(on)) { + CL = server.getClassLoader(on); + } else { + CL = Thread.currentThread().getContextClassLoader(); + } + for(String cl: classNames) { + className = cl.trim(); + Class.forName(className, true, CL); + } + Class.forName(citem.getValue().trim(), true, ConfigMetaType.class.getClassLoader()); + } catch (Exception ex) { + throw new IllegalArgumentException("Failed to load boot time class [" + className + "] for " + citem.getName()); + } + } + } + + /** + *

    Title: DirFileSystemValidator

    + *

    Description: Validator for fully qualified file names where the parent directory + * must exist but the file is optional

    + */ + public static class DirFileSystemValidator implements ArgValueValidator { + + @Override + public void validate(final ConfigurationItem citem) { + final String name = citem.getValue(); + if(name==null || name.trim().isEmpty()) return; + final File f = new File(citem.getValue()).getAbsoluteFile(); + final File dir = f.getParentFile(); + if(dir.exists() && dir.isDirectory()) return; + if(!dir.exists()) throw new IllegalArgumentException("No directory named [" + dir + "] exists. Invalid value for config item:" + citem.getKey()); + if(!dir.isDirectory()) throw new IllegalArgumentException("Specified parent directory [" + dir + "] is not a directory. Invalid value for config item:" + citem.getKey()); + } + + } + + + /** + *

    Title: FileSystemValidator

    + *

    Description: Validator for files and directories

    + */ + public static class FileSystemValidator implements ArgValueValidator { + private final boolean dir; + private final boolean mustExist; + + /** + * Creates a new FileSystemValidator + * @param dir true for validating directories, false for files + * @param mustExist true if the target must exist, false if it can be created if it does not exist + */ + public FileSystemValidator(boolean dir, boolean mustExist) { + this.dir = dir; + this.mustExist = mustExist; + } + + /** + * {@inheritDoc} + * @see net.opentsdb.tools.ArgValueValidator#validate(net.opentsdb.tools.ConfigArgP.ConfigurationItem) + */ + @Override + public void validate(final ConfigurationItem citem) { + String value = citem.getValue(); + File target = new File(value); + if(mustExist) { + if(!target.exists()) throw new IllegalArgumentException((dir ? "Directory [" : " File [") + value + "] does not exist for " + citem.getName()); + if(dir) { + if(!target.isDirectory()) throw new IllegalArgumentException(("[") + value + "] is a file not a directory for " + citem.getName()); + } else { + if(!target.isFile()) throw new IllegalArgumentException(("[") + value + "] is a directory not a file for " + citem.getName()); + } + } else { + if(dir) { + if(!target.exists()) { + if(!target.mkdirs()) throw new IllegalArgumentException(("Could not create directory [") + value + "] for " + citem.getName()); + } else { + if(!target.isDirectory()) throw new IllegalArgumentException(("[") + value + "] is a file not a directory for " + citem.getName()); + } + } else { + if(!target.getParentFile().exists()) { + if(!target.getParentFile().mkdirs()) throw new IllegalArgumentException(("Could not create parent directory for file [") + value + "] for " + citem.getName()); + } + + if(!target.exists()) { + try { + if(!target.createNewFile()) throw new IllegalArgumentException(("Could not create file [") + value + "] for " + citem.getName()); + } catch (IOException e) { + throw new IllegalArgumentException(("Could not create file [") + value + "] for " + citem.getName()); + } finally { + target.delete(); + } + } + } + } + } + } + + /** + *

    Title: TimeZoneValidator

    + *

    Description: Validates time zone typed configuration items.

    + *

    net.opentsdb.tools.TimeZoneValidator

    + */ + public static class TimeZoneValidator implements ArgValueValidator { + private static final TimeZone GMT = TimeZone.getTimeZone("GMT"); + + @Override + public void validate(final ConfigurationItem citem) { + final String tz = citem.getValue(); + if("DEFAULT".equalsIgnoreCase(tz)) { + citem.setValue(TimeZone.getDefault().getDisplayName()); + return; + } + TimeZone timeZone = TimeZone.getTimeZone(tz); + if("GMT".equals(timeZone.getID())) { + if(!tz.toUpperCase().contains("GMT")) { + throw new IllegalArgumentException("Unrecognized TimeZone [" + tz + "]"); + } + } + } + + } + + /** + *

    Title: FileListValidator

    + *

    Description: A validator for a list of existing files or accessible URLs

    + */ + public static class FileListValidator implements ArgValueValidator { + /** + * {@inheritDoc} + * @see net.opentsdb.tools.ConfigMetaType.StringValidator#validate(net.opentsdb.tools.ConfigArgP.ConfigurationItem) + */ + @Override + public void validate(final ConfigurationItem citem) { + buildURLSet(citem); + } + + /** + * Builds a set of URLs from the configured file list and validates each one + * @param citem The configuration item + * @return A [possibly empty] set of URLs + */ + protected Set buildURLSet(final ConfigurationItem citem) { + final Set urls = new LinkedHashSet(); + String[] files = COMMA_SPLITTER.split(citem.getValue()); + Set failed = new LinkedHashSet(); + for(String file: files) { + String name = file.trim(); + if(name.isEmpty()) continue; + boolean isURL = false; + URL url = null; + try { + url = new URL(name); + isURL = true; + urls.add(url); + } catch (Exception ex) { /* No Op */ } + if(isURL) continue; + File f = new File(name); + if(f.exists() && f.canRead()) { +// if(f.isDirectory()) { +// failed.add(name + ":Not a file"); +// } + try { + urls.add(f.toURI().toURL()); + } catch (Exception ex) { + failed.add(name + ":Could not convert to URL"); + } + } else { + failed.add(name + ":Not found"); + continue; + } + } + if(!failed.isEmpty()) { + StringBuilder b = new StringBuilder("FileList Validation Failures:"); + for(String s: failed) { + b.append("\n\t").append(s); + } + throw new IllegalArgumentException("Invalid Files or URLs in File List for " + citem.getName() + "\n" + b.toString()); + } + + return urls; + } + + } + + /** + *

    Title: ClassPathValidator

    + *

    Description: A class path validator and ClassLoader factory

    + *

    Company: Helios Development Group LLC

    + */ + public static class ClassPathValidator extends FileListValidator { + /** The template for the classloader MBean's ObjectName */ + public static final String CLASSLOADER_OBJECTNAME = "net.opentsdb.classpath:type=ClassLoader,name=%s"; + + /** + * {@inheritDoc} + * @see net.opentsdb.tools.ConfigMetaType.FileListValidator#validate(net.opentsdb.tools.ConfigArgP.ConfigurationItem) + */ + @Override + public void validate(final ConfigurationItem citem) { + super.validate(citem); + Set urls = buildURLSet(citem); + final MLet classLoader = new MLet(urls.toArray(new URL[urls.size()])); + try { + final ObjectName on = new ObjectName(String.format(CLASSLOADER_OBJECTNAME, citem.getKey())); + if(server.isRegistered(on)) { + server.unregisterMBean(on); + } + server.registerMBean(classLoader, on); + } catch (Exception ex) { + throw new IllegalArgumentException("Failed to create class loader for [" + citem.getName() + "]", ex); + } + + } + } + + +} diff --git a/src/tools/DumpSeries.java b/src/tools/DumpSeries.java index a090f1e851..ac710878fd 100644 --- a/src/tools/DumpSeries.java +++ b/src/tools/DumpSeries.java @@ -15,9 +15,12 @@ import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Date; +import java.util.List; import java.util.Map; +import net.opentsdb.core.AppendDataPoints; import org.hbase.async.DeleteRequest; import org.hbase.async.HBaseClient; import org.hbase.async.KeyValue; @@ -96,49 +99,51 @@ private static void doDump(final TSDB tsdb, final StringBuilder buf = new StringBuilder(); for (final Query query : queries) { - final Scanner scanner = Internal.getScanner(query); - ArrayList> rows; - while ((rows = scanner.nextRows().joinUninterruptibly()) != null) { - for (final ArrayList row : rows) { - buf.setLength(0); - final byte[] key = row.get(0).key(); - final long base_time = Internal.baseTime(tsdb, key); - final String metric = Internal.metricName(tsdb, key); - // Print the row key. - if (!importformat) { - buf.append(Arrays.toString(key)) - .append(' ') - .append(metric) - .append(' ') - .append(base_time) - .append(" (").append(date(base_time)).append(") "); - try { - buf.append(Internal.getTags(tsdb, key)); - } catch (RuntimeException e) { - buf.append(e.getClass().getName() + ": " + e.getMessage()); - } - buf.append('\n'); - System.out.print(buf); - } - - // Print individual cells. - buf.setLength(0); - if (!importformat) { - buf.append(" "); - } - for (final KeyValue kv : row) { - // Discard everything or keep initial spaces. - buf.setLength(importformat ? 0 : 2); - formatKeyValue(buf, tsdb, importformat, kv, base_time, metric); - if (buf.length() > 0) { + final List scanners = Internal.getScanners(query); + for (Scanner scanner : scanners) { + ArrayList> rows; + while ((rows = scanner.nextRows().joinUninterruptibly()) != null) { + for (final ArrayList row : rows) { + buf.setLength(0); + final byte[] key = row.get(0).key(); + final long base_time = Internal.baseTime(tsdb, key); + final String metric = Internal.metricName(tsdb, key); + // Print the row key. + if (!importformat) { + buf.append(Arrays.toString(key)) + .append(' ') + .append(metric) + .append(' ') + .append(base_time) + .append(" (").append(date(base_time)).append(") "); + try { + buf.append(Internal.getTags(tsdb, key)); + } catch (RuntimeException e) { + buf.append(e.getClass().getName() + ": " + e.getMessage()); + } buf.append('\n'); System.out.print(buf); } - } - if (delete) { - final DeleteRequest del = new DeleteRequest(table, key); - client.delete(del); + // Print individual cells. + buf.setLength(0); + if (!importformat) { + buf.append(" "); + } + for (final KeyValue kv : row) { + // Discard everything or keep initial spaces. + buf.setLength(importformat ? 0 : 2); + formatKeyValue(buf, tsdb, importformat, kv, base_time, metric); + if (buf.length() > 0) { + buf.append('\n'); + System.out.print(buf); + } + } + + if (delete) { + final DeleteRequest del = new DeleteRequest(table, key); + client.delete(del); + } } } } @@ -177,7 +182,7 @@ private static void formatKeyValue(final StringBuilder buf, final byte[] value = kv.value(); final int q_len = qualifier.length; - if (q_len % 2 != 0) { + if (!AppendDataPoints.isAppendDataPoints(qualifier) && q_len % 2 != 0) { if (!importformat) { // custom data object, not a data point if (kv.qualifier()[0] == Annotation.PREFIX()) { @@ -200,8 +205,16 @@ private static void formatKeyValue(final StringBuilder buf, appendImportCell(buf, cell, base_time, tags); } } else { - // compacted column - final ArrayList cells = Internal.extractDataPoints(kv); + final Collection cells; + if (q_len == 3) { + // append data points + final AppendDataPoints adps = new AppendDataPoints(); + cells = adps.parseKeyValue(tsdb, kv); + } else { + // compacted column + cells = Internal.extractDataPoints(kv); + } + if (!importformat) { buf.append(Arrays.toString(kv.qualifier())) .append('\t') diff --git a/src/tools/Fsck.java b/src/tools/Fsck.java index 02e39a2be2..1881290f4a 100644 --- a/src/tools/Fsck.java +++ b/src/tools/Fsck.java @@ -1,5 +1,5 @@ // This file is part of OpenTSDB. -// Copyright (C) 2014 The OpenTSDB Authors. +// Copyright (C) 2014-2021 The OpenTSDB Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU Lesser General Public License as published by @@ -32,17 +32,23 @@ import org.hbase.async.PutRequest; import org.hbase.async.Scanner; +import com.google.common.base.Strings; +import com.stumbleupon.async.Callback; import com.stumbleupon.async.Deferred; +import net.opentsdb.core.AppendDataPoints; import net.opentsdb.core.Const; import net.opentsdb.core.IllegalDataException; import net.opentsdb.core.Internal; import net.opentsdb.core.Internal.Cell; import net.opentsdb.core.Query; +import net.opentsdb.core.RequestBuilder; import net.opentsdb.core.RowKey; import net.opentsdb.core.TSDB; import net.opentsdb.core.Tags; import net.opentsdb.meta.Annotation; +import net.opentsdb.rollup.RollupInterval; +import net.opentsdb.rollup.RollupUtils; import net.opentsdb.uid.NoSuchUniqueId; import net.opentsdb.uid.UniqueId; import net.opentsdb.utils.Config; @@ -54,14 +60,14 @@ * rows matching the query will be FSCK'd. Alternatively a full table scan can * be performed. *

    - * Scanning is done in three stages: + * Scanning is done in three stages: * 1) Each row key is parsed to make sure it's a valid OpenTSDB row. If it isn't * then the user can decide to delete it. If one or more UIDs cannot be resolved * to names (metric or tags) then the user can decide to purge it. - * 2) All key value pairs in a row are parsed to determine the type of object. - * If it's a single data point, it's added to a tree map based on the data point - * timestamp. If it's a compacted column, the data points are exploded and - * added to the data point map. If it's some other object it may be purged if + * 2) All key value pairs in a row are parsed to determine the type of object. + * If it's a single data point, it's added to a tree map based on the data point + * timestamp. If it's a compacted column, the data points are exploded and + * added to the data point map. If it's some other object it may be purged if * told to, or if it's a known type (e.g. annotations) simply ignored. * 3) If any data points were found, we iterate over each one looking for * duplicates, malformed encodings or potential value-length-encoding savings. @@ -79,19 +85,21 @@ */ final class Fsck { private static final Logger LOG = LoggerFactory.getLogger(Fsck.class); - + /** The TSDB to use for access */ - private final TSDB tsdb; + private final TSDB tsdb; /** Options to use while iterating over rows */ private final FsckOptions options; - + /** Counters incremented during processing. They have to be atomic counters * as we may be running multiple fsck threads. */ final AtomicLong kvs_processed = new AtomicLong(); final AtomicLong rows_processed = new AtomicLong(); final AtomicLong valid_datapoints = new AtomicLong(); final AtomicLong annotations = new AtomicLong(); + final AtomicLong append_dps = new AtomicLong(); + final AtomicLong append_dps_fixed = new AtomicLong(); final AtomicLong bad_key = new AtomicLong(); final AtomicLong bad_key_fixed = new AtomicLong(); final AtomicLong duplicates = new AtomicLong(); @@ -112,17 +120,17 @@ final class Fsck { final AtomicLong vle = new AtomicLong(); final AtomicLong vle_bytes = new AtomicLong(); final AtomicLong vle_fixed = new AtomicLong(); - + /** Length of the metric + timestamp for key validation */ - private static int key_prefix_length = TSDB.metrics_width() + - Const.TIMESTAMP_BYTES; - + private int key_prefix_length = Const.SALT_WIDTH() + + TSDB.metrics_width() + Const.TIMESTAMP_BYTES; + /** Length of a tagk + tagv pair for key validation */ - private static int key_tags_length = TSDB.tagk_width() + TSDB.tagv_width(); - + private int key_tags_length = TSDB.tagk_width() + TSDB.tagv_width(); + /** How often to report progress */ private static long report_rows = 10000; - + /** * Default Ctor * @param tsdb The TSDB to use for access @@ -132,7 +140,7 @@ public Fsck(final TSDB tsdb, final FsckOptions options) { this.tsdb = tsdb; this.options = options; } - + /** * Fetches the max metric ID and splits the data table up amongst threads on * a naive split. By default we execute cores * 2 threads but the user can @@ -142,38 +150,33 @@ public Fsck(final TSDB tsdb, final FsckOptions options) { public void runFullTable() throws Exception { LOG.info("Starting full table scan"); final long start_time = System.currentTimeMillis() / 1000; - final long max_id = CliUtils.getMaxMetricID(tsdb); - final int workers = options.threads() > 0 ? options.threads() : Runtime.getRuntime().availableProcessors() * 2; - final double quotient = (double)max_id / (double)workers; - LOG.info("Max metric ID is [" + max_id + "]"); - LOG.info("Spooling up [" + workers + "] worker threads"); - long index = 1; - final Thread[] threads = new Thread[workers]; - for (int i = 0; i < workers; i++) { - threads[i] = new FsckWorker(index, quotient, i); - threads[i].setName("Fsck #" + i); - threads[i].start(); - index += quotient; - if (index < max_id) { - index++; - } + + final List scanners = CliUtils.getDataTableScanners(tsdb, workers); + LOG.info("Spooling up [" + scanners.size() + "] worker threads"); + final List threads = new ArrayList(scanners.size()); + int i = 0; + for (final Scanner scanner : scanners) { + final FsckWorker worker = new FsckWorker(scanner, i++, this.options); + worker.setName("Fsck #" + i); + worker.start(); + threads.add(worker); } final Thread reporter = new ProgressReporter(); reporter.start(); - for (int i = 0; i < workers; i++) { - threads[i].join(); - LOG.info("Thread [" + i + "] Finished"); + for (final Thread thread : threads) { + thread.join(); + LOG.info("Thread [" + thread + "] Finished"); } reporter.interrupt(); - + logResults(); final long duration = (System.currentTimeMillis() / 1000) - start_time; LOG.info("Completed fsck in [" + duration + "] seconds"); } - + /** * Scans the rows matching one or more standard queries. An aggregator is still * required though it's ignored. @@ -182,63 +185,91 @@ public void runFullTable() throws Exception { */ public void runQueries(final List queries) throws Exception { final long start_time = System.currentTimeMillis() / 1000; - - // TODO - threadify it. We *could* have hundreds of queries and we don't + + // TODO - threadify it. We *could* have hundreds of queries and we don't // want to create that many threads. For now we'll just execute each one // serially final Thread reporter = new ProgressReporter(); reporter.start(); - + for (final Query query : queries) { - final FsckWorker worker = new FsckWorker(query, 0); - worker.run(); + final List scanners = Internal.getScanners(query); + final List threads = new ArrayList(scanners.size()); + int i = 0; + for (final Scanner scanner : scanners) { + final FsckWorker worker = new FsckWorker(scanner, i++, this.options); + worker.setName("Fsck #" + i); + worker.start(); + threads.add(worker); + } + + for (final Thread thread : threads) { + thread.join(); + LOG.info("Thread [" + thread + "] Finished"); + } } reporter.interrupt(); - + logResults(); final long duration = (System.currentTimeMillis() / 1000) - start_time; LOG.info("Completed fsck in [" + duration + "] seconds"); } - + /** @return The total number of errors detected during the run */ long totalErrors() { return bad_key.get() + duplicates.get() + orphans.get() + unknown.get() + - bad_values.get() + bad_compacted_columns.get() + + bad_values.get() + bad_compacted_columns.get() + fixable_compacted_columns.get() + value_encoding.get(); } - + /** @return The total number of errors fixed during the run */ long totalFixed() { return bad_key_fixed.get() + duplicates_fixed.get() + orphans_fixed.get() + - unknown_fixed.get() + value_encoding_fixed.get() + + unknown_fixed.get() + value_encoding_fixed.get() + bad_values_deleted.get(); } - + /** @return The total number of errors that could be (or may have been) fixed */ long correctable() { return bad_key.get() + duplicates.get() + orphans.get() + unknown.get() + - bad_values.get() + bad_compacted_columns.get() + + bad_values.get() + bad_compacted_columns.get() + fixable_compacted_columns.get() + value_encoding.get(); } - + /** - * A worker thread that takes a query or a chunk of the main data table and + * Log all Throwables + */ + final class GeneralErrCallBack implements Callback, Exception> { + private Object[] parameters; + + GeneralErrCallBack(Object... parameters) { + this.parameters = parameters; + } + + @Override + public Deferred call(Exception arg) throws Exception { + LOG.error("when: %s, something went wrong: %s", parameters, arg); + throw arg; + } + } + + /** + * A worker thread that takes a query or a chunk of the main data table and * performs the actual FSCK process. */ final class FsckWorker extends Thread { - /** Optional value of the first metric this worker should start on, should - * be >0 */ - final long start_id; - /** Value of the metric this worker should end on */ - final long end_id; /** Id of the thread this worker belongs to */ final int thread_id; /** Optional query to execute instead of a full table scan */ final Query query; + /** The scanner to use for iterating over a chunk of the table */ + final Scanner scanner; /** Set of TSUIDs this worker has seen. Used to avoid UID resolution for * previously processed row keys */ final Set tsuids = new HashSet(); - + + final FsckOptions options; + /** Shared flags and values for compiling a compacted column */ byte[] compact_qualifier = null; int qualifier_index = 0; @@ -247,54 +278,37 @@ final class FsckWorker extends Thread { boolean compact_row = false; int qualifier_bytes = 0; int value_bytes = 0; - + /** * Ctor for running a worker on a chunk of the data table - * @param start_id The first metric this worker should start on - * @param quotient How many metrics the worker should cover + * @param scanner The scanner to use for iterationg * @param thread_id Id of the thread this worker is assigned for logging */ - FsckWorker(final long start_id, final double quotient, final int thread_id) { - this.start_id = start_id; - this.end_id = start_id + (long) quotient + 1; // teensy bit of overlap + FsckWorker(final Scanner scanner, final int thread_id, final FsckOptions options) { + this.scanner = scanner; this.thread_id = thread_id; query = null; + this.options = options; } - - /** - * Ctor for running an FSCK over a specific query, scanning only rows that - * match the filter. - * @param query The query to execute - * @param thread_id Id of the thread this worker is assigned for logging - */ - FsckWorker(final Query query, final int thread_id) { - start_id = 0; - end_id = 0; - this.thread_id = thread_id; - this.query = query; - } - + /** * Determines the type of scanner to use, i.e. a specific query scanner or - * for a portion of the whole table. It then performs the actual scan, - * compiling a list of data points and fixing/compacting them when + * for a portion of the whole table. It then performs the actual scan, + * compiling a list of data points and fixing/compacting them when * appropriate. */ public void run() { - final Scanner scanner = query != null ? Internal.getScanner(query) : - CliUtils.getDataTableScanner(tsdb, start_id, end_id); - - // store every data point for the row in here - final TreeMap> datapoints = + // store every data point for the row in here + final TreeMap> datapoints = new TreeMap>(); byte[] last_key = null; ArrayList> rows; - + try { while ((rows = scanner.nextRows().joinUninterruptibly()) != null) { // keep in mind that with annotations and millisecond values, a row // can now have more than 4069 key values, the default for a scanner. - // Since we don't know how many values may actually be in a row, we + // Since we don't know how many values may actually be in a row, we // don't want to set the KV limit too high. Instead we'll just keep // working through the sets until we hit a different row key, then // process all of the data points. It puts more of a burden on fsck @@ -316,7 +330,7 @@ public void run() { fsckRow(row, datapoints); } } - + // handle the last row if (!datapoints.isEmpty()) { rows_processed.getAndIncrement(); @@ -328,54 +342,59 @@ public void run() { LOG.error("Shouldn't be here", e); } } - + /** * Parses the row of KeyValues. First it validates the row key, then parses - * each KeyValue to determine what kind of object it is. Data points are + * each KeyValue to determine what kind of object it is. Data points are * stored in the tree map and non-data point columns are handled per the * option flags * @param row The row of data to parse * @param datapoints The map of datapoints to append to. * @throws Exception If something goes pear shaped. */ - private void fsckRow(final ArrayList row, + private void fsckRow(final ArrayList row, final TreeMap> datapoints) throws Exception { // The data table should contain only rows with a metric, timestamp and - // one or more tag pairs. Future version may use different prefixes or - // key formats but for now, we can safely delete any rows with invalid + // one or more tag pairs. Future version may use different prefixes or + // key formats but for now, we can safely delete any rows with invalid // keys. This may check the same row key multiple times but that's good // as it will keep the data points from being pushed to the dp map if (!fsckKey(row.get(0).key())) { return; } - - final long base_time = Bytes.getUnsignedInt(row.get(0).key(), - TSDB.metrics_width()); - + + final long base_time = Bytes.getUnsignedInt(row.get(0).key(), + Const.SALT_WIDTH() + TSDB.metrics_width()); + + NextKV: for (final KeyValue kv : row) { kvs_processed.getAndIncrement(); // these are not final as they may be modified when fixing is enabled - byte[] value = kv.value(); + byte[] value = kv.value(); byte[] qual = kv.qualifier(); - + // all qualifiers must be at least 2 bytes long, i.e. a single data point if (qual.length < 2) { unknown.getAndIncrement(); LOG.error("Invalid qualifier, must be on 2 bytes or more.\n\t" + kv); if (options.fix() && options.deleteUnknownColumns()) { final DeleteRequest delete = new DeleteRequest(tsdb.dataTable(), kv); - tsdb.getClient().delete(delete); + Deferred operation_result = tsdb.getClient().delete(delete); + operation_result.addErrback(new GeneralErrCallBack(delete)); + if (options.fixInSync()) { + operation_result.join(options.getFixTimeout()); + } unknown_fixed.getAndIncrement(); } continue; } - - // All data point columns have an even number of bytes, so if we find - // one that has an odd length, it could be an OpenTSDB object or it + + // Almost all data point columns have an even number of bytes, so if we + // find one that has an odd length, it could be an OpenTSDB object or it // could be junk that made it into the table. if (qual.length % 2 != 0) { - // If this test fails, the column is not a TSDB object such as an - // annotation or blob. Future versions may be able to compact TSDB + // If this test fails, the column is not a TSDB object such as an + // annotation or blob. Future versions may be able to compact TSDB // objects so that their qualifier would be of a different length, but // for now we'll consider it an error. if (qual.length != 3 && qual.length != 5) { @@ -384,35 +403,152 @@ private void fsckRow(final ArrayList row, "of bytes.\n\t" + kv); if (options.fix() && options.deleteUnknownColumns()) { final DeleteRequest delete = new DeleteRequest(tsdb.dataTable(), kv); - tsdb.getClient().delete(delete); + Deferred operation_result = tsdb.getClient().delete(delete); + operation_result.addErrback(new GeneralErrCallBack(delete)); + if (options.fixInSync()) { + operation_result.join(options.getFixTimeout()); + } unknown_fixed.getAndIncrement(); } continue; } - + // TODO - create a list of TSDB objects and fsck them. Maybe a plugin // or interface. // TODO - perform validation of the annotation if (qual[0] == Annotation.PREFIX()) { + // TODO - don't increment if we have a rollup instead! annotations.getAndIncrement(); continue; + } else if (qual[0] == AppendDataPoints.APPEND_COLUMN_PREFIX) { + append_dps.getAndIncrement(); + try { + final AppendDataPoints adps = new AppendDataPoints(); + adps.parseKeyValue(tsdb, kv); + if (adps.repairedDeferred() != null) { + append_dps_fixed.incrementAndGet(); + } + } catch (RuntimeException e) { + LOG.error("Unexpected exception processing append data point: " + kv, e); + } + continue; + } else if (tsdb.getRollupConfig() != null) { + // it could be a rollup with numeric prefix. Maybe possibly. + int agg_id = qual[0]; + String aggregation = tsdb.getRollupConfig().getAggregatorForId(agg_id); + if (!Strings.isNullOrEmpty(aggregation)) { + StringBuilder buffer = null; + boolean marked_value_as_bad = false; + boolean marked_value_as_good = false; + for (RollupInterval interval : tsdb.getRollupConfig().getIntervals()) { + long offset = RollupUtils.getOffsetFromRollupQualifier(qual, 1, interval); + if (offset < 0 || offset >= interval.getIntervals()) { + // definitely not a rollup for this interval. + continue; + } + + // TODO - need to set the table if we add full support for walking + // the rollup tables too. + boolean wrong_table = !Bytes.equals(tsdb.dataTable(), interval.getTemporalTable()); + long timestamp = RollupUtils.getTimestampFromRollupQualifier(qual, base_time, interval, 1); + short len = Internal.getValueLengthFromQualifier(qual, 1); + len++; + short flags = Internal.getFlagsFromQualifier(qual, 1); + String err = null; + try { + if (Internal.isFloat(qual, 1)) { + Internal.extractFloatingPointValue(value, 0, (byte) flags); + } else { + Internal.extractIntegerValue(value, 0, (byte) flags); + } + if (!marked_value_as_good) { + valid_datapoints.incrementAndGet(); + marked_value_as_good = true; + } + } catch (IllegalDataException e) { + if (!marked_value_as_bad) { + bad_values.incrementAndGet(); + marked_value_as_bad = true; + } + err = e.getMessage(); + } + + // now see if something was wrong + boolean early_ts = timestamp / 1000 < base_time; + boolean late_ts = (timestamp / 1000) >= + base_time + (interval.getIntervals() * interval.getIntervalSeconds()); + + if (err == null && !wrong_table && !early_ts && !late_ts) { + // good rollup + valid_datapoints.incrementAndGet(); + continue NextKV; + } + + if (buffer == null) { + buffer = new StringBuilder(); + } + if (buffer.length() > 0) { + buffer.append("\n"); + } + buffer.append("\tPossible rollup"); + if (wrong_table) { + buffer.append(" in the wrong table [") + .append(tsdb.dataTable()) + .append("]"); + } + buffer.append(" Agg=") + .append(aggregation) + .append(" Interval=") + .append(interval) + .append(" Offset=") + .append(offset); + if (early_ts) { + buffer.append(" Timestamp (") + .append(timestamp) + .append(") earlier than row timestamps (") + .append(base_time) + .append(")"); + } else if (late_ts) { + buffer.append(" Timestamp (") + .append(timestamp) + .append(") later than next row timestamps (") + .append(base_time + + (interval.getIntervals() * interval.getIntervalSeconds())) + .append(")"); + } else { + buffer.append(" Timestamp=") + .append(timestamp); + } + buffer.append(" "); + parseUnknownDP(value, 0, len, buffer); + buffer.append("\n\t\t") + .append(kv); + } + + if (buffer != null) { + LOG.warn(buffer.toString()); + continue; + } + } + // otherwise fall through, not a rollup that's configured on this + // system. } LOG.warn("Found an object possibly from a future version of OpenTSDB\n\t" + kv); future.getAndIncrement(); continue; } - - // This is (hopefully) a compacted column with multiple data points. It + + // This is (hopefully) a compacted column with multiple data points. It // could have two points with second qualifiers or multiple points with // a mix of second and millisecond qualifiers if (qual.length == 4 && !Internal.inMilliseconds(qual[0]) || qual.length > 4) { if (value[value.length - 1] > Const.MS_MIXED_COMPACT) { - // TODO - figure out a way to fix these. Maybe lookup a row before + // TODO - figure out a way to fix these. Maybe lookup a row before // or after and try parsing this for values. If the values are // somewhat close to the others, then we could just set the last - // byte. Otherwise it could be a bad compaction and we'd need to + // byte. Otherwise it could be a bad compaction and we'd need to // toss it. bad_compacted_columns.getAndIncrement(); LOG.error("The last byte of a compacted should be 0 or 1. Either" @@ -420,12 +556,12 @@ private void fsckRow(final ArrayList row, + " future version of OpenTSDB.\n\t" + kv); continue; } - - // add every cell in the compacted column to the data point tree so + + // add every cell in the compacted column to the data point tree so // that we can scan for duplicate timestamps try { final ArrayList cells = Internal.extractDataPoints(kv); - + // the extractDataPoints() method will automatically fix up some // issues such as setting proper lengths on floats and sorting the // cells to be in order. Rather than reproduce the extraction code or @@ -445,11 +581,11 @@ private void fsckRow(final ArrayList row, dps.add(new DP(kv, cell)); qualifier_bytes += cell.qualifier().length; value_bytes += cell.value().length; - System.arraycopy(cell.qualifier(), 0, recompacted_qualifier, + System.arraycopy(cell.qualifier(), 0, recompacted_qualifier, qualifier_index, cell.qualifier().length); qualifier_index += cell.qualifier().length; } - + if (Bytes.memcmp(recompacted_qualifier, kv.qualifier()) != 0) { LOG.error("Compacted column was out of order or requires a " + "fixup: " + kv); @@ -461,16 +597,20 @@ private void fsckRow(final ArrayList row, LOG.error(e.getMessage()); if (options.fix() && options.deleteBadCompacts()) { final DeleteRequest delete = new DeleteRequest(tsdb.dataTable(), kv); - tsdb.getClient().delete(delete); + Deferred operation_result = tsdb.getClient().delete(delete); + operation_result.addErrback(new GeneralErrCallBack(delete)); + if (options.fixInSync()) { + operation_result.join(options.getFixTimeout()); + } bad_compacted_columns_deleted.getAndIncrement(); } } continue; } - - // at this point we *should* be dealing with a single data point encoded + + // at this point we *should* be dealing with a single data point encoded // in seconds or milliseconds. - final long timestamp = + final long timestamp = Internal.getTimestampFromQualifier(qual, base_time); ArrayList dps = datapoints.get(timestamp); if (dps == null) { @@ -484,7 +624,7 @@ private void fsckRow(final ArrayList row, } /** - * Validates the row key. It must match the format + * Validates the row key. It must match the format * {@code [...]}. If it doesn't, then * the row is considered an error. If the UIDs in a row key do not resolve * to a name, then the row is considered an orphan and the values contained @@ -500,22 +640,26 @@ private void fsckRow(final ArrayList row, * @throws Exception If something goes pear shaped. */ private boolean fsckKey(final byte[] key) throws Exception { - if (key.length < key_prefix_length || + if (key.length < key_prefix_length || (key.length - key_prefix_length) % key_tags_length != 0) { LOG.error("Invalid row key.\n\tKey: " + UniqueId.uidToString(key)); bad_key.getAndIncrement(); - + if (options.fix() && options.deleteBadRows()) { final DeleteRequest delete = new DeleteRequest(tsdb.dataTable(), key); - tsdb.getClient().delete(delete); + Deferred operation_result = tsdb.getClient().delete(delete); + operation_result.addErrback(new GeneralErrCallBack(delete)); + if (options.fixInSync()) { + operation_result.join(options.getFixTimeout()); + } bad_key_fixed.getAndIncrement(); } return false; } - + // Process the time series ID by resolving the UIDs to names if we haven't - // already seen this particular TSUID - final byte[] tsuid = UniqueId.getTSUIDFromKey(key, TSDB.metrics_width(), + // already seen this particular TSUID. Note that getTSUID accounts for salt + final byte[] tsuid = UniqueId.getTSUIDFromKey(key, TSDB.metrics_width(), Const.TIMESTAMP_BYTES); if (!tsuids.contains(tsuid)) { try { @@ -524,15 +668,19 @@ private boolean fsckKey(final byte[] key) throws Exception { LOG.error("Unable to resolve the metric from the row key.\n\tKey: " + UniqueId.uidToString(key) + "\n\t" + nsui.getMessage()); orphans.getAndIncrement(); - + if (options.fix() && options.deleteOrphans()) { final DeleteRequest delete = new DeleteRequest(tsdb.dataTable(), key); - tsdb.getClient().delete(delete); + Deferred operation_result = tsdb.getClient().delete(delete); + operation_result.addErrback(new GeneralErrCallBack(delete)); + if (options.fixInSync()) { + operation_result.join(options.getFixTimeout()); + } orphans_fixed.getAndIncrement(); } return false; } - + try { Tags.resolveIds(tsdb, (ArrayList) UniqueId.getTagPairsFromTSUID(tsuid)); @@ -540,10 +688,14 @@ private boolean fsckKey(final byte[] key) throws Exception { LOG.error("Unable to resolve the a tagk or tagv from the row key.\n\tKey: " + UniqueId.uidToString(key) + "\n\t" + nsui.getMessage()); orphans.getAndIncrement(); - + if (options.fix() && options.deleteOrphans()) { final DeleteRequest delete = new DeleteRequest(tsdb.dataTable(), key); - tsdb.getClient().delete(delete); + Deferred operation_result = tsdb.getClient().delete(delete); + operation_result.addErrback(new GeneralErrCallBack(delete)); + if (options.fixInSync()) { + operation_result.join(options.getFixTimeout()); + } orphans_fixed.getAndIncrement(); } return false; @@ -560,7 +712,7 @@ private boolean fsckKey(final byte[] key) throws Exception { * @param datapoints The list of data points parsed from the row * @throws Exception If something goes pear shaped. */ - private void fsckDataPoints(final Map> datapoints) + private void fsckDataPoints(final Map> datapoints) throws Exception { // store a unique set of qualifier/value columns to help us later when @@ -571,12 +723,13 @@ private void fsckDataPoints(final Map> datapoints) boolean has_milliseconds = false; boolean has_duplicates = false; boolean has_uncorrected_value_error = false; - + long timestamp = Long.MAX_VALUE; + for (final Map.Entry> time_map : datapoints.entrySet()) { if (key == null) { key = time_map.getValue().get(0).kv.key(); } - + if (time_map.getValue().size() < 2) { // there was only one data point for this timestamp, no conflicts final DP dp = time_map.getValue().get(0); @@ -594,9 +747,9 @@ private void fsckDataPoints(final Map> datapoints) // sort so we can figure out which one we're going to keep, i.e. oldest // or newest - Collections.sort(time_map.getValue()); + Collections.sort(time_map.getValue()); has_duplicates = true; - // We want to keep either the first or the last incoming datapoint + // We want to keep either the first or the last incoming datapoint // and ignore delete the middle. final StringBuilder buf = new StringBuilder(); @@ -628,6 +781,7 @@ private void fsckDataPoints(final Map> datapoints) } unique_columns.put(dp_to_keep.kv.qualifier(), dp_to_keep.kv.value()); + timestamp = Math.min(timestamp, dp_to_keep.kv.timestamp()); valid_datapoints.getAndIncrement(); has_uncorrected_value_error |= Internal.isFloat(dp_to_keep.qualifier()) ? fsckFloat(dp_to_keep) : fsckInteger(dp_to_keep); @@ -638,40 +792,48 @@ private void fsckDataPoints(final Map> datapoints) has_seconds = true; } - for (int dp_index = delete_range_start; dp_index < delete_range_stop; + for (int dp_index = delete_range_start; dp_index < delete_range_stop; dp_index++) { duplicates.getAndIncrement(); DP dp = time_map.getValue().get(dp_index); - final byte flags = (byte)Internal.getFlagsFromQualifier(dp.kv.qualifier()); - buf.append(" ") - .append("write time: (") - .append(dp.kv.timestamp()) - .append(" - ") - .append(new Date(dp.kv.timestamp())) - .append(") ") - .append(" compacted: (") - .append(dp.compacted) - .append(") qualifier: ") - .append(Arrays.toString(dp.kv.qualifier())) - .append(" value: ") - .append(Internal.isFloat(dp.kv.qualifier()) ? - Internal.extractFloatingPointValue(dp.value(), 0, flags) : - Internal.extractIntegerValue(dp.value(), 0, flags)) - .append("\n"); - unique_columns.put(dp.kv.qualifier(), dp.kv.value()); - if (options.fix() && options.resolveDupes()) { - if (compact_row) { - // Scheduled for deletion by compaction. - duplicates_fixed_comp.getAndIncrement(); - } else if (!dp.compacted) { - LOG.debug("Removing duplicate data point: " + dp.kv); - tsdb.getClient().delete( - new DeleteRequest( - tsdb.dataTable(), dp.kv.key(), dp.kv.family(), dp.qualifier() - ) - ); - duplicates_fixed.getAndIncrement(); + try { + final byte flags = (byte)Internal.getFlagsFromQualifier(dp.kv.qualifier()); + buf.append(" ") + .append("write time: (") + .append(dp.kv.timestamp()) + .append(" - ") + .append(new Date(dp.kv.timestamp())) + .append(") ") + .append(" compacted: (") + .append(dp.compacted) + .append(") qualifier: ") + .append(Arrays.toString(dp.kv.qualifier())) + .append(" value: ") + .append(Internal.isFloat(dp.kv.qualifier()) ? + Internal.extractFloatingPointValue(dp.value(), 0, flags) : + Internal.extractIntegerValue(dp.value(), 0, flags)) + .append("\n"); + unique_columns.put(dp.kv.qualifier(), dp.kv.value()); + if (options.fix() && options.resolveDupes()) { + if (compact_row) { + // Scheduled for deletion by compaction. + duplicates_fixed_comp.getAndIncrement(); + } else if (!dp.compacted) { + LOG.debug("Removing duplicate data point: " + dp.kv); + DeleteRequest delete = new DeleteRequest(tsdb.dataTable(), + dp.kv.key(), + dp.kv.family(), + dp.qualifier()); + Deferred operation_result = tsdb.getClient().delete(delete); + operation_result.addErrback(new GeneralErrCallBack(delete)); + if (options.fixInSync()) { + operation_result.join(options.getFixTimeout()); + } + duplicates_fixed.getAndIncrement(); + } } + } catch (Exception e) { + LOG.error("Unexpected exception processing DP: " + dp); } } if (options.lastWriteWins()) { @@ -679,20 +841,20 @@ private void fsckDataPoints(final Map> datapoints) } LOG.info(buf.toString()); } - + // if an error was found in this row that was not marked for repair, then // we should bail at this point and not write a new compacted column. - if ((has_duplicates && !options.resolveDupes()) || + if ((has_duplicates && !options.resolveDupes()) || (has_uncorrected_value_error && !options.deleteBadValues())) { LOG.warn("One or more errors found in row that were not marked for repair"); return; } - - if ((options.compact() || compact_row) && options.fix() + + if ((options.compact() || compact_row) && options.fix() && qualifier_index > 0) { - if (qualifier_index == 2 || (qualifier_index == 4 && + if (qualifier_index == 2 || (qualifier_index == 4 && Internal.inMilliseconds(compact_qualifier))) { - // we may have deleted all but one value from the row and that one + // we may have deleted all but one value from the row and that one // value may have a different qualifier than it originally had. We // can't write a compacted column with a single data point as the length // will be off due to the flag at the end. Therefore we just rollback @@ -703,13 +865,13 @@ private void fsckDataPoints(final Map> datapoints) compact_value[value_index] = 1; } value_index++; - final byte[] new_qualifier = Arrays.copyOfRange(compact_qualifier, 0, + final byte[] new_qualifier = Arrays.copyOfRange(compact_qualifier, 0, qualifier_index); - final byte[] new_value = Arrays.copyOfRange(compact_value, 0, + final byte[] new_value = Arrays.copyOfRange(compact_value, 0, value_index); - final PutRequest put = new PutRequest(tsdb.dataTable(), key, - TSDB.FAMILY(), new_qualifier, new_value); - + final PutRequest put = RequestBuilder.buildPutRequest(tsdb.getConfig(), tsdb.dataTable(), key, + TSDB.FAMILY(), new_qualifier, new_value, timestamp); + // it's *possible* that the hash of our new compacted qualifier is in // the delete list so double check before we delete everything if (unique_columns.containsKey(new_qualifier)) { @@ -745,11 +907,11 @@ private void fsckDataPoints(final Map> datapoints) // proceeding with the deletes. tsdb.getClient().put(put).joinUninterruptibly(); } - - final List> deletes = + + final List> deletes = new ArrayList>(unique_columns.size()); for (byte[] qualifier : unique_columns.keySet()) { - final DeleteRequest delete = new DeleteRequest(tsdb.dataTable(), key, + final DeleteRequest delete = new DeleteRequest(tsdb.dataTable(), key, TSDB.FAMILY(), qualifier); if (LOG.isDebugEnabled()) { final StringBuilder buf = new StringBuilder(); @@ -767,9 +929,9 @@ private void fsckDataPoints(final Map> datapoints) duplicates_fixed_comp.set(0); } } - + /** - * Handles validating a floating point value. Floats must be encoded on 4 + * Handles validating a floating point value. Floats must be encoded on 4 * bytes for a Float and 8 bytes for a Double. The qualifier is compared to * the actual length in the case of single data points. In previous versions * of OpenTSDB, the qualifier flag may have been on 4 bytes but the actual @@ -801,13 +963,17 @@ private boolean fsckFloat(final DP dp) throws Exception { final float value_as_float = Float.intBitsToFloat(Bytes.getInt(value, 4)); value = Bytes.fromInt( - Float.floatToRawIntBits((float)value_as_float)); + Float.floatToRawIntBits(value_as_float)); if (compact_row || options.compact()) { appendDP(qual, value, 4); } else if (!dp.compacted){ - final PutRequest put = new PutRequest(tsdb.dataTable(), + final PutRequest put = new PutRequest(tsdb.dataTable(), dp.kv.key(), dp.kv.family(), qual, value); - tsdb.getClient().put(put); + Deferred operation_result = tsdb.getClient().put(put); + operation_result.addErrback(new GeneralErrCallBack(put)); + if (options.fixInSync()) { + operation_result.join(options.getFixTimeout()); + } } else { LOG.error("SHOULDN'T be here as we didn't compact or fix a " + "single value"); @@ -824,9 +990,13 @@ private boolean fsckFloat(final DP dp) throws Exception { + " not zeroed\n\t" + dp); bad_values.getAndIncrement(); if (options.fix() && options.deleteBadValues() && !dp.compacted) { - final DeleteRequest delete = new DeleteRequest(tsdb.dataTable(), + final DeleteRequest delete = new DeleteRequest(tsdb.dataTable(), dp.kv); - tsdb.getClient().delete(delete); + Deferred operation_result = tsdb.getClient().delete(delete); + operation_result.addErrback(new GeneralErrCallBack(delete)); + if (options.fixInSync()) { + operation_result.join(options.getFixTimeout()); + } bad_values_deleted.getAndIncrement(); } else if (dp.compacted) { LOG.error("The value was in a compacted column. This should " @@ -845,13 +1015,17 @@ private boolean fsckFloat(final DP dp) throws Exception { final float value_as_float = Float.intBitsToFloat(Bytes.getInt(value, 4)); value = Bytes.fromInt( - Float.floatToRawIntBits((float)value_as_float)); + Float.floatToRawIntBits(value_as_float)); if (compact_row || options.compact()) { appendDP(qual, value, 4); } else if (!dp.compacted) { - final PutRequest put = new PutRequest(tsdb.dataTable(), + final PutRequest put = new PutRequest(tsdb.dataTable(), dp.kv.key(), dp.kv.family(), qual, value); - tsdb.getClient().put(put); + Deferred operation_result = tsdb.getClient().put(put); + operation_result.addErrback(new GeneralErrCallBack(put)); + if (options.fixInSync()) { + operation_result.join(options.getFixTimeout()); + } } else { LOG.error("SHOULDN'T be here as we didn't compact or fix a single value"); } @@ -868,7 +1042,11 @@ private boolean fsckFloat(final DP dp) throws Exception { + " was only " + value.length + " bytes.\n\t" + dp.kv); if (options.fix() && options.deleteBadValues() && !dp.compacted) { final DeleteRequest delete = new DeleteRequest(tsdb.dataTable(), dp.kv); - tsdb.getClient().delete(delete); + Deferred operation_result = tsdb.getClient().delete(delete); + operation_result.addErrback(new GeneralErrCallBack(delete)); + if (options.fixInSync()) { + operation_result.join(options.getFixTimeout()); + } bad_values_deleted.getAndIncrement(); } else if (dp.compacted) { LOG.error("The previous value was in a compacted column. This should " @@ -884,7 +1062,11 @@ private boolean fsckFloat(final DP dp) throws Exception { + " bytes.\n\t" + dp.kv); if (options.fix() && options.deleteBadValues() && !dp.compacted) { final DeleteRequest delete = new DeleteRequest(tsdb.dataTable(), dp.kv); - tsdb.getClient().delete(delete); + Deferred operation_result = tsdb.getClient().delete(delete); + operation_result.addErrback(new GeneralErrCallBack(delete)); + if (options.fixInSync()) { + operation_result.join(options.getFixTimeout()); + } bad_values_deleted.getAndIncrement(); } else if (dp.compacted) { LOG.error("The previous value was in a compacted column. This should " @@ -901,10 +1083,10 @@ private boolean fsckFloat(final DP dp) throws Exception { } return false; } - + /** * Handles validating an integer value. Integers must be encoded on 1, 2, 4 - * or 8 bytes. Older versions of OpenTSDB wrote all integers on 8 bytes + * or 8 bytes. Older versions of OpenTSDB wrote all integers on 8 bytes * regardless of value. If the --fix flag is specified, this method will * attempt to re-encode small values to save space (up to 7 bytes!!). It also * makes sure the value length matches the length specified in the qualifier @@ -916,7 +1098,7 @@ private boolean fsckFloat(final DP dp) throws Exception { private boolean fsckInteger(final DP dp) throws Exception { byte[] qual = dp.qualifier(); byte[] value = dp.value(); - + // this should be a single integer value. Check the encoding to make // sure it's the proper length, and if the flag is set to fix encoding // we can save space with VLE. @@ -929,7 +1111,11 @@ private boolean fsckInteger(final DP dp) throws Exception { + "should be " + length + " bytes.\n\t" + dp.kv); if (options.fix() && options.deleteBadValues()) { final DeleteRequest delete = new DeleteRequest(tsdb.dataTable(), dp.kv); - tsdb.getClient().delete(delete); + Deferred operation_result = tsdb.getClient().delete(delete); + operation_result.addErrback(new GeneralErrCallBack(delete)); + if (options.fixInSync()) { + operation_result.join(options.getFixTimeout()); + } bad_values_deleted.getAndIncrement(); } else if (dp.compacted) { LOG.error("The previous value was in a compacted column. This should " @@ -940,9 +1126,9 @@ private boolean fsckInteger(final DP dp) throws Exception { } return false; } - + // OpenTSDB had support for VLE decoding of integers but only wrote - // on 8 bytes originally. Lets see how much space we could save. + // on 8 bytes originally. Lets see how much space we could save. // We'll assume that a length other than 8 bytes is already VLE'd if (length == 8) { final long decoded = Bytes.getLong(value); @@ -954,7 +1140,7 @@ private boolean fsckInteger(final DP dp) throws Exception { vle.getAndIncrement(); vle_bytes.addAndGet(6); value = Bytes.fromShort((short) decoded); - } else if (Integer.MIN_VALUE <= decoded && + } else if (Integer.MIN_VALUE <= decoded && decoded <= Integer.MAX_VALUE) { vle.getAndIncrement(); vle_bytes.addAndGet(4); @@ -968,12 +1154,16 @@ private boolean fsckInteger(final DP dp) throws Exception { appendDP(new_qualifier, value, value.length); } else { // put the new value, THEN delete the old - final PutRequest put = new PutRequest(tsdb.dataTable(), - dp.kv.key(), dp.kv.family(), new_qualifier, value); + final PutRequest put = RequestBuilder.buildPutRequest(tsdb.getConfig(), tsdb.dataTable(), + dp.kv.key(), dp.kv.family(), new_qualifier, value, dp.kv.timestamp()); tsdb.getClient().put(put).joinUninterruptibly(); - final DeleteRequest delete = new DeleteRequest(tsdb.dataTable(), + final DeleteRequest delete = new DeleteRequest(tsdb.dataTable(), dp.kv.key(), dp.kv.family(), qual); - tsdb.getClient().delete(delete); + Deferred operation_result = tsdb.getClient().delete(delete); + operation_result.addErrback(new GeneralErrCallBack(delete)); + if (options.fixInSync()) { + operation_result.join(options.getFixTimeout()); + } } vle_fixed.getAndIncrement(); } // don't return true here as we don't consider a VLE an error. @@ -986,27 +1176,27 @@ private boolean fsckInteger(final DP dp) throws Exception { } /** - * Appends the given value to the running qualifier and value compaction + * Appends the given value to the running qualifier and value compaction * byte arrays. It doesn't take a {@code DP} as we may be changing the * arrays before they're re-written. * @param new_qual The qualifier to append * @param new_value The value to append * @param value_length How much of the value to append */ - private void appendDP(final byte[] new_qual, final byte[] new_value, + private void appendDP(final byte[] new_qual, final byte[] new_value, final int value_length) { System.arraycopy(new_qual, 0, compact_qualifier, qualifier_index, new_qual.length); qualifier_index += new_qual.length; System.arraycopy(new_value, 0, compact_value, value_index, value_length); - value_index += value_length; + value_index += value_length; } - + /** * Appends a representation of a datapoint to a string buffer * @param buf The buffer to modify * @param msg An optional message to append */ - private StringBuilder appendDatapointInfo(final StringBuilder buf, + private StringBuilder appendDatapointInfo(final StringBuilder buf, final DP dp, final String msg) { buf.append(" ") .append("write time: (") @@ -1021,7 +1211,7 @@ private StringBuilder appendDatapointInfo(final StringBuilder buf, } /** - * Resets the running compaction variables. This should be called AFTER a + * Resets the running compaction variables. This should be called AFTER a * {@link fsckDataPoints()} has been run and before the next row of values * is processed. Note that we may overallocate some memory when creating * the arrays. @@ -1050,7 +1240,7 @@ final class DP implements Comparable { boolean compacted; /** The specific data point qualifier/value if the data point was compacted */ Cell cell; - + /** * Default Ctor used for a single data point * @param kv The column where the value appeared. @@ -1059,7 +1249,7 @@ final class DP implements Comparable { this.kv = kv; compacted = false; } - + /** * Overload for a compacted data point * @param kv The column where the value appeared. @@ -1070,7 +1260,7 @@ final class DP implements Comparable { this.cell = cell; compacted = true; } - + /** * Compares data points. * @param dp The data point to compare to @@ -1081,20 +1271,20 @@ final class DP implements Comparable { public int compareTo(final DP dp) { if (kv.timestamp() == dp.kv.timestamp()) { return 0; - } + } return kv.timestamp() < dp.kv.timestamp() ? -1 : 1; } - + /** @return The qualifier of the data point (from the compaction or column) */ public byte[] qualifier() { return compacted ? cell.qualifier() : kv.qualifier(); } - + /** @return The value of the data point */ public byte[] value() { return compacted ? cell.value() : kv.value(); } - + /** @return The cell or key value string */ public String toString() { return compacted ? cell.toString() : kv.toString(); @@ -1102,6 +1292,38 @@ public String toString() { } } + private static void parseUnknownDP(final byte[] values, + final int offset, + final int length, + final StringBuilder buffer) { + if (length == 1) { + buffer.append((int) values[offset]) + .append(" 1b integer"); + } else if (length == 2) { + buffer.append(Bytes.getShort(values, offset)) + .append(" 2b integer"); + } else if (length == 4) { + int val = Bytes.getInt(values, offset); + buffer.append(val) + .append(" 4b integer || ") + .append(Float.intBitsToFloat(val)) + .append(" 4b float"); + } else if (length == 8) { + long val = Bytes.getLong(values, offset); + buffer.append(val) + .append(" 8b long || ") + .append(Double.longBitsToDouble(val)) + .append(" 8b double"); + } else { + buffer.append("[") + // TODO - ugly copy creating garbage. Could walk and print. + .append(Arrays.toString(Arrays.copyOfRange(values, offset, length))) + .append("] ") + .append(length) + .append("b unknown"); + } + } + /** * Silly little class to report the progress while fscking */ @@ -1117,8 +1339,8 @@ public void run() { processed_rows = (processed_rows - (processed_rows % report_rows)); if (processed_rows - last_progress >= report_rows) { last_progress = processed_rows; - LOG.info("Processed " + processed_rows + " rows, " + - valid_datapoints.get() + " valid datapoints"); + LOG.info("Processed " + processed_rows + " rows, " + + valid_datapoints.get() + " valid datapoints"); } Thread.sleep(1000); } catch (InterruptedException e) { @@ -1126,7 +1348,7 @@ public void run() { } } } - + /** Prints usage and exits with the given retval. */ private static void usage(final ArgP argp, final String errmsg, final int retval) { @@ -1164,19 +1386,19 @@ private void logResults() { LOG.info("Unparseable Datapoint Values: " + bad_values.get()); LOG.info("Unparseable Datapoint Values Deleted: " + bad_values_deleted.get()); LOG.info("Improperly Encoded Floating Point Values: " + value_encoding.get()); - LOG.info("Improperly Encoded Floating Point Values Fixed: " + + LOG.info("Improperly Encoded Floating Point Values Fixed: " + value_encoding_fixed.get()); LOG.info("Unparseable Compacted Columns: " + bad_compacted_columns.get()); - LOG.info("Unparseable Compacted Columns Deleted: " + + LOG.info("Unparseable Compacted Columns Deleted: " + bad_compacted_columns_deleted.get()); LOG.info("Datapoints Qualified for VLE : " + vle.get()); LOG.info("Datapoints Compressed with VLE: " + vle_fixed.get()); - LOG.info("Bytes Saved with VLE: " + vle_bytes.get()); + LOG.info("Bytes Saved with VLE: " + vle_bytes.get()); LOG.info("Total Errors: " + totalErrors()); LOG.info("Total Correctable Errors: " + correctable()); LOG.info("Total Errors Fixed: " + totalFixed()); } - + /** * The main class executed from the "tsdb" script * @param args Command line arguments to parse @@ -1204,7 +1426,7 @@ public static void main(String[] args) throws Exception { usage(argp, "Must supply a query or use the '--full-scan' flag", 1); } tsdb.checkNecessaryTablesExist().joinUninterruptibly(); - + argp = null; final Fsck fsck = new Fsck(tsdb, options); try { diff --git a/src/tools/FsckOptions.java b/src/tools/FsckOptions.java index 1f0a8b9cbb..481f799ff0 100644 --- a/src/tools/FsckOptions.java +++ b/src/tools/FsckOptions.java @@ -28,6 +28,8 @@ final class FsckOptions { private boolean delete_bad_rows; private boolean delete_bad_compacts; private int threads; + private long fix_timeout; // fix timeout for each operation results, time unit: milliseconds + private boolean fix_in_sync_mode = false; // wait for each fix operation to finish to continue /** * Default Ctor that sets the options based on command line flags and config @@ -52,6 +54,8 @@ public FsckOptions(final ArgP argp, final Config config) { argp.has("--fix-all"); delete_bad_compacts = argp.has("--delete-bad-compacts") || argp.has("--fix-all"); + fix_in_sync_mode = argp.has("--sync"); + if (argp.has("--threads")) { threads = Integer.parseInt(argp.get("--threads")); if (threads < 1) { @@ -92,6 +96,7 @@ public static void addDataOptions(final ArgP argp) { "Delete compacted columns that cannot be parsed."); argp.addOption("--threads", "NUMBER", "Number of threads to use when executing a full table scan."); + argp.addOption("--sync", "Wait for each fix operation to finish to continue."); } /** @return Whether or not to fix errors while processing. Does not affect @@ -134,7 +139,6 @@ public boolean deleteUnknownColumns() { public boolean deleteBadValues() { return delete_bad_values; } - /** @return Remove rows with invalid keys */ public boolean deleteBadRows() { @@ -157,7 +161,6 @@ public void setFix(final boolean fix) { this.fix = fix; } - /** @param compact Whether or not to compact rows while processing. Can cause * compaction without the --fix flag. Will skip rows with duplicate data * points unless --last-write-wins is also specified or set in the config @@ -171,27 +174,23 @@ public void setResolveDupes(final boolean fix_dupes) { this.resolve_dupes = fix_dupes; } - /** @param last_write_wins Accept data points with the most recent timestamp when duplicates * are found */ public void setLastWriteWins(final boolean last_write_wins) { this.last_write_wins = last_write_wins; } - /** @param delete_orphans Whether or not to delete rows where the UIDs failed to resolve * to a name */ public void setDeleteOrphans(final boolean delete_orphans) { this.delete_orphans = delete_orphans; } - /** @param delete_unknown_columns Delete columns that aren't recognized */ public void setDeleteUnknownColumns(final boolean delete_unknown_columns) { this.delete_unknown_columns = delete_unknown_columns; } - /** @param delete_bad_values Remove data points with bad values */ public void setDeleteBadValues(final boolean delete_bad_values) { this.delete_bad_values = delete_bad_values; @@ -221,4 +220,34 @@ public void setThreads(final int threads) { } this.threads = threads; } + + /** + * @param timeout The maximum time to wait in milliseconds. A value of 0 + * means no timeout. + */ + public void setFixTimeout(long timeout) { + this.fix_timeout = timeout; + } + + /** + * @return timeout + */ + public long getFixTimeout() { + return this.fix_timeout; + } + + /** + * @param flag Wheather to wait for the result of the fix operation. + */ + public void setFixInSync(boolean flag) { + this.fix_in_sync_mode = flag; + } + + /** + * Wait for each fix operation to finish to continue. + * @return true if in sync mode + */ + public boolean fixInSync() { + return this.fix_in_sync_mode; + } } diff --git a/src/tools/GnuplotInstaller.java b/src/tools/GnuplotInstaller.java new file mode 100644 index 0000000000..6356a1723d --- /dev/null +++ b/src/tools/GnuplotInstaller.java @@ -0,0 +1,108 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2010-2016 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.tools; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; + +import org.jboss.netty.buffer.ChannelBuffer; +import org.jboss.netty.buffer.DynamicChannelBuffer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + *

    Title: GnuplotInstaller

    + *

    Description: Installs the gnuplot invocation shell file

    + */ + +public class GnuplotInstaller { + private static final boolean IS_WINDOWS = + System.getProperty("os.name", "").contains("Windows"); + + /** Static class logger */ + private static final Logger LOG = LoggerFactory.getLogger(GnuplotInstaller.class); + + /** The name of the shell file */ + public static final String GP_BATCH_FILE_NAME = IS_WINDOWS ? "mygnuplot.bat" : "mygnuplot.sh"; + /** The name of the gnuplot executable */ + public static final String GP_NAME = IS_WINDOWS ? "gnuplot.exe" : "gnuplot"; + + /** The directory where shell file will be installed */ + public static final String GP_DIR = System.getProperty("java.io.tmpdir") + File.separator + ".tsdb" + File.separator + "tsdb-gnuplot"; + /** The java/io file representing the gnuplot shell file */ + public static final File GP_FILE = new File(GP_DIR, GP_BATCH_FILE_NAME); + /** Indicates if gnuplot was found on the path */ + public static final boolean FOUND_GP; + + static { + boolean found = false; + final String PATH = System.getenv("PATH"); + if(PATH!=null) { + final String[] paths = PATH.split(File.pathSeparator); + for(String path: paths) { + LOG.debug("Inspecting PATH for Gnuplot Exe: [{}]", path); + File dir = new File(path.trim()); + if(dir.exists() && dir.isDirectory()) { + File gp = new File(dir, GP_NAME); + if(gp.exists()) { + found = true; + LOG.info("Found gnuplot at [{}]", gp.getAbsolutePath()); + break; + } + + } + } + } + FOUND_GP = found; + if(!found) LOG.warn("Failed to locate Gnuplot executable"); + } + + private GnuplotInstaller() { + + } + + /** + * Installs the mygnuplot shell file + */ + public static void installMyGnuPlot() { + if(!FOUND_GP) { + LOG.warn("Skipping Gnuplot Shell Script Install since Gnuplot executable was not found"); + return; + } + if(!GP_FILE.exists()) { + if(!GP_FILE.getParentFile().exists()) { + GP_FILE.getParentFile().mkdirs(); + } + InputStream is = null; + FileOutputStream fos = null; + try { + is = GnuplotInstaller.class.getClassLoader().getResourceAsStream(GP_BATCH_FILE_NAME); + ChannelBuffer buff = new DynamicChannelBuffer(is.available()); + buff.writeBytes(is, is.available()); + is.close(); is = null; + fos = new FileOutputStream(GP_FILE); + buff.readBytes(fos, buff.readableBytes()); + fos.close(); fos = null; + GP_FILE.setExecutable(true); + } catch (Exception ex) { + throw new IllegalArgumentException("Failed to install mygnuplot", ex); + } finally { + if( is!=null ) try { is.close(); } catch (Exception x) { /* No Op */ } + if( fos!=null ) try { fos.close(); } catch (Exception x) { /* No Op */ } + } + } + } + + +} \ No newline at end of file diff --git a/src/tools/MetaSync.java b/src/tools/MetaSync.java index 75bf1b2957..5076ea2c4b 100644 --- a/src/tools/MetaSync.java +++ b/src/tools/MetaSync.java @@ -12,7 +12,6 @@ // see . package net.opentsdb.tools; -import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -28,7 +27,6 @@ import net.opentsdb.uid.UniqueId.UniqueIdType; import org.hbase.async.Bytes; -import org.hbase.async.HBaseException; import org.hbase.async.KeyValue; import org.hbase.async.Scanner; import org.slf4j.Logger; @@ -54,13 +52,7 @@ final class MetaSync extends Thread { /** TSDB to use for storage access */ final TSDB tsdb; - - /** The ID to start the sync with for this thread */ - final long start_id; - - /** The end of the ID block to work on */ - final long end_id; - + /** A shared list of TSUIDs that have been processed by this or other * threads. It stores hashes instead of the bytes or strings to save * on space */ @@ -78,22 +70,27 @@ final class MetaSync extends Thread { /** Diagnostic ID for this thread */ final int thread_id; + /** The scanner for this worker */ + final Scanner scanner; + /** * Constructor that sets local variables * @param tsdb The TSDB to process with - * @param start_id The starting ID of the block we'll work on - * @param quotient The total number of IDs in our block + * @param scanner The scanner to use for this worker + * @param processed_tsuids TSUIDs that have been processed already + * @param metric_uids List of metric UIDs + * @param tagk_uids List of tag key UIDs + * @param tagv_uids List of tag value UIDs * @param thread_id The ID of this thread (starts at 0) */ - public MetaSync(final TSDB tsdb, final long start_id, final double quotient, + public MetaSync(final TSDB tsdb, final Scanner scanner, final Set processed_tsuids, ConcurrentHashMap metric_uids, ConcurrentHashMap tagk_uids, ConcurrentHashMap tagv_uids, final int thread_id) { this.tsdb = tsdb; - this.start_id = start_id; - this.end_id = start_id + (long) quotient + 1; // teensy bit of overlap + this.scanner = scanner; this.processed_tsuids = processed_tsuids; this.metric_uids = metric_uids; this.tagk_uids = tagk_uids; @@ -105,11 +102,28 @@ public MetaSync(final TSDB tsdb, final long start_id, final double quotient, * Loops through the entire TSDB data set and exits when complete. */ public void run() { - // list of deferred calls used to act as a buffer final ArrayList> storage_calls = new ArrayList>(); final Deferred result = new Deferred(); + + final class ErrBack implements Callback { + @Override + public Object call(Exception e) throws Exception { + Throwable ex = e; + while (ex.getClass().equals(DeferredGroupException.class)) { + if (ex.getCause() == null) { + LOG.warn("Unable to get to the root cause of the DGE"); + break; + } + ex = ex.getCause(); + } + LOG.error("Sync thread failed with exception", ex); + result.callback(null); + return null; + } + } + final ErrBack err_back = new ErrBack(); /** * Called when we have encountered a previously un-processed UIDMeta object. @@ -342,17 +356,9 @@ public Deferred call(final Boolean exists) throws Exception { final class MetaScanner implements Callback>> { - private final Scanner scanner; private byte[] last_tsuid = null; private String tsuid_string = ""; - - /** - * Default constructor that initializes the data row scanner - */ - public MetaScanner() { - scanner = getScanner(); - } - + /** * Fetches the next set of rows from the scanner and adds this class as * a callback @@ -360,7 +366,7 @@ public MetaScanner() { * been processed. */ public Object scan() { - return scanner.nextRows().addCallback(this); + return scanner.nextRows().addCallback(this).addErrback(err_back); } @Override @@ -372,132 +378,136 @@ public Object call(ArrayList> rows) } for (final ArrayList row : rows) { - - final byte[] tsuid = UniqueId.getTSUIDFromKey(row.get(0).key(), - TSDB.metrics_width(), Const.TIMESTAMP_BYTES); - - // if the current tsuid is the same as the last, just continue - // so we save time - if (last_tsuid != null && Arrays.equals(last_tsuid, tsuid)) { - continue; - } - last_tsuid = tsuid; - - // see if we've already processed this tsuid and if so, continue - if (processed_tsuids.contains(Arrays.hashCode(tsuid))) { - continue; - } - tsuid_string = UniqueId.uidToString(tsuid); - - // add tsuid to the processed list - processed_tsuids.add(Arrays.hashCode(tsuid)); - - // we may have a new TSUID or UIDs, so fetch the timestamp of the - // row for use as the "created" time. Depending on speed we could - // parse datapoints, but for now the hourly row time is enough - final long timestamp = Bytes.getUnsignedInt(row.get(0).key(), - TSDB.metrics_width()); - - LOG.debug("[" + thread_id + "] Processing TSUID: " + tsuid_string + - " row timestamp: " + timestamp); - - // now process the UID metric meta data - final byte[] metric_uid_bytes = - Arrays.copyOfRange(tsuid, 0, TSDB.metrics_width()); - final String metric_uid = UniqueId.uidToString(metric_uid_bytes); - Long last_get = metric_uids.get(metric_uid); - - if (last_get == null || last_get == 0 || timestamp < last_get) { - // fetch and update. Returns default object if the meta doesn't - // exist, so we can just call sync on this to create a missing - // entry - final UidCB cb = new UidCB(UniqueIdType.METRIC, - metric_uid_bytes, timestamp); - final Deferred process_uid = UIDMeta.getUIDMeta(tsdb, - UniqueIdType.METRIC, metric_uid_bytes).addCallbackDeferring(cb); - storage_calls.add(process_uid); - metric_uids.put(metric_uid, timestamp); - } - - // loop through the tags and process their meta - final List tags = UniqueId.getTagsFromTSUID(tsuid_string); - int idx = 0; - for (byte[] tag : tags) { - final UniqueIdType type = (idx % 2 == 0) ? UniqueIdType.TAGK : - UniqueIdType.TAGV; - idx++; - final String uid = UniqueId.uidToString(tag); + try { + final byte[] tsuid = UniqueId.getTSUIDFromKey(row.get(0).key(), + TSDB.metrics_width(), Const.TIMESTAMP_BYTES); - // check the maps to see if we need to bother updating - if (type == UniqueIdType.TAGK) { - last_get = tagk_uids.get(uid); - } else { - last_get = tagv_uids.get(uid); + // if the current tsuid is the same as the last, just continue + // so we save time + if (last_tsuid != null && Arrays.equals(last_tsuid, tsuid)) { + continue; } - if (last_get != null && last_get != 0 && last_get <= timestamp) { + last_tsuid = tsuid; + + // see if we've already processed this tsuid and if so, continue + if (processed_tsuids.contains(Arrays.hashCode(tsuid))) { continue; } - - // fetch and update. Returns default object if the meta doesn't - // exist, so we can just call sync on this to create a missing - // entry - final UidCB cb = new UidCB(type, tag, timestamp); - final Deferred process_uid = UIDMeta.getUIDMeta(tsdb, type, tag) - .addCallbackDeferring(cb); - storage_calls.add(process_uid); - if (type == UniqueIdType.TAGK) { - tagk_uids.put(uid, timestamp); - } else { - tagv_uids.put(uid, timestamp); + tsuid_string = UniqueId.uidToString(tsuid); + + /** + * An error callback used to catch issues with a particular timeseries + * or UIDMeta such as a missing UID name. We want to continue + * processing when this happens so we'll just log the error and + * the user can issue a command later to clean up orphaned meta + * entries. + */ + final class RowErrBack implements Callback { + @Override + public Object call(Exception e) throws Exception { + Throwable ex = e; + while (ex.getClass().equals(DeferredGroupException.class)) { + if (ex.getCause() == null) { + LOG.warn("Unable to get to the root cause of the DGE"); + break; + } + ex = ex.getCause(); + } + if (ex.getClass().equals(IllegalStateException.class)) { + LOG.error("Invalid data when processing TSUID [" + + tsuid_string + "]: " + ex.getMessage()); + } else if (ex.getClass().equals(IllegalArgumentException.class)) { + LOG.error("Invalid data when processing TSUID [" + + tsuid_string + "]: " + ex.getMessage()); + } else if (ex.getClass().equals(NoSuchUniqueId.class)) { + LOG.warn("Timeseries [" + tsuid_string + + "] includes a non-existant UID: " + ex.getMessage()); + } else { + LOG.error("Unknown exception processing row: " + row, ex); + } + return null; + } } - } - - /** - * An error callback used to cache issues with a particular timeseries - * or UIDMeta such as a missing UID name. We want to continue - * processing when this happens so we'll just log the error and - * the user can issue a command later to clean up orphaned meta - * entries. - */ - final class ErrBack implements Callback, Exception> { - @Override - public Deferred call(Exception e) throws Exception { + // add tsuid to the processed list + processed_tsuids.add(Arrays.hashCode(tsuid)); + + // we may have a new TSUID or UIDs, so fetch the timestamp of the + // row for use as the "created" time. Depending on speed we could + // parse datapoints, but for now the hourly row time is enough + final long timestamp = Bytes.getUnsignedInt(row.get(0).key(), + Const.SALT_WIDTH() + TSDB.metrics_width()); + + LOG.debug("[" + thread_id + "] Processing TSUID: " + tsuid_string + + " row timestamp: " + timestamp); + + // now process the UID metric meta data + final byte[] metric_uid_bytes = + Arrays.copyOfRange(tsuid, 0, TSDB.metrics_width()); + final String metric_uid = UniqueId.uidToString(metric_uid_bytes); + Long last_get = metric_uids.get(metric_uid); + + if (last_get == null || last_get == 0 || timestamp < last_get) { + // fetch and update. Returns default object if the meta doesn't + // exist, so we can just call sync on this to create a missing + // entry + final UidCB cb = new UidCB(UniqueIdType.METRIC, + metric_uid_bytes, timestamp); + final Deferred process_uid = UIDMeta.getUIDMeta(tsdb, + UniqueIdType.METRIC, metric_uid_bytes) + .addCallbackDeferring(cb) + .addErrback(new RowErrBack()); + storage_calls.add(process_uid); + metric_uids.put(metric_uid, timestamp); + } + + // loop through the tags and process their meta + final List tags = UniqueId.getTagsFromTSUID(tsuid_string); + int idx = 0; + for (byte[] tag : tags) { + final UniqueIdType type = (idx % 2 == 0) ? UniqueIdType.TAGK : + UniqueIdType.TAGV; + idx++; + final String uid = UniqueId.uidToString(tag); - Throwable ex = e; - while (ex.getClass().equals(DeferredGroupException.class)) { - if (ex.getCause() == null) { - LOG.warn("Unable to get to the root cause of the DGE"); - break; - } - ex = ex.getCause(); + // check the maps to see if we need to bother updating + if (type == UniqueIdType.TAGK) { + last_get = tagk_uids.get(uid); + } else { + last_get = tagv_uids.get(uid); + } + if (last_get != null && last_get != 0 && last_get <= timestamp) { + continue; } - if (ex.getClass().equals(IllegalStateException.class)) { - LOG.error("Invalid data when processing TSUID [" + - tsuid_string + "]", ex); - } else if (ex.getClass().equals(IllegalArgumentException.class)) { - LOG.error("Invalid data when processing TSUID [" + - tsuid_string + "]", ex); - } else if (ex.getClass().equals(NoSuchUniqueId.class)) { - LOG.warn("Timeseries [" + tsuid_string + - "] includes a non-existant UID: " + ex.getMessage()); + + // fetch and update. Returns default object if the meta doesn't + // exist, so we can just call sync on this to create a missing + // entry + final UidCB cb = new UidCB(type, tag, timestamp); + final Deferred process_uid = + UIDMeta.getUIDMeta(tsdb, type, tag) + .addCallbackDeferring(cb) + .addErrback(new RowErrBack()); + storage_calls.add(process_uid); + if (type == UniqueIdType.TAGK) { + tagk_uids.put(uid, timestamp); } else { - LOG.error("Unmatched Exception: " + ex.getClass()); - throw e; + tagv_uids.put(uid, timestamp); } - - return Deferred.fromResult(false); } + // handle the timeseries meta last so we don't record it if one + // or more of the UIDs had an issue + final Deferred process_tsmeta = + TSMeta.getTSMeta(tsdb, tsuid_string) + .addCallbackDeferring(new TSMetaCB(tsuid, timestamp)) + .addErrback(new RowErrBack()); + storage_calls.add(process_tsmeta); + } catch (RuntimeException e) { + LOG.error("Processing row " + row + " failed with exception: " + + e.getMessage()); + LOG.debug("Row: " + row + " stack trace: ", e); } - - // handle the timeseries meta last so we don't record it if one - // or more of the UIDs had an issue - final Deferred process_tsmeta = - TSMeta.getTSMeta(tsdb, tsuid_string) - .addCallbackDeferring(new TSMetaCB(tsuid, timestamp)); - process_tsmeta.addErrback(new ErrBack()); - storage_calls.add(process_tsmeta); } /** @@ -558,26 +568,5 @@ public Object call(Exception e) throws Exception { throw new RuntimeException("[" + thread_id + "] Scanner exception", e); } } - - /** - * Returns a scanner set to scan the range configured for this thread - * @return A scanner on the "t" CF configured for the specified range - * @throws HBaseException if something goes boom - */ - private Scanner getScanner() throws HBaseException { - final short metric_width = TSDB.metrics_width(); - final byte[] start_row = - Arrays.copyOfRange(Bytes.fromLong(start_id), 8 - metric_width, 8); - final byte[] end_row = - Arrays.copyOfRange(Bytes.fromLong(end_id), 8 - metric_width, 8); - - LOG.debug("[" + thread_id + "] Start row: " + UniqueId.uidToString(start_row)); - LOG.debug("[" + thread_id + "] End row: " + UniqueId.uidToString(end_row)); - final Scanner scanner = tsdb.getClient().newScanner(tsdb.dataTable()); - scanner.setStartKey(start_row); - scanner.setStopKey(end_row); - scanner.setFamily("t".getBytes(Charset.forName("ISO-8859-1"))); - return scanner; - } } diff --git a/src/tools/OpenTSDBMain.java b/src/tools/OpenTSDBMain.java new file mode 100644 index 0000000000..6261949db2 --- /dev/null +++ b/src/tools/OpenTSDBMain.java @@ -0,0 +1,866 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.tools; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.InputStream; +import java.io.PrintStream; +import java.lang.management.ManagementFactory; +import java.lang.reflect.Method; +import java.net.HttpURLConnection; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.URL; +import java.net.URLConnection; +import java.util.Collections; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; + +import javax.management.ObjectName; + +import net.opentsdb.core.TSDB; +import net.opentsdb.tools.ConfigArgP.ConfigurationItem; +import net.opentsdb.tsd.PipelineFactory; +import net.opentsdb.utils.Config; +import net.opentsdb.utils.Threads; + +import org.jboss.netty.bootstrap.ServerBootstrap; +import org.jboss.netty.buffer.ChannelBuffer; +import org.jboss.netty.buffer.ChannelBuffers; +import org.jboss.netty.channel.socket.ServerSocketChannelFactory; +import org.jboss.netty.channel.socket.nio.NioServerBossPool; +import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory; +import org.jboss.netty.channel.socket.nio.NioWorkerPool; +import org.jboss.netty.channel.socket.oio.OioServerSocketChannelFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.joran.JoranConfigurator; +import ch.qos.logback.core.BasicStatusManager; +import ch.qos.logback.core.joran.spi.JoranException; +import ch.qos.logback.core.status.StatusListener; + +/** + *

    Title: Main

    + *

    Description: OpenTSDB fat-jar main entry point

    + */ + +public class OpenTSDBMain { + /** The platform EOL string */ + public static final String EOL = System.getProperty("line.separator", "\n"); + /** Static class logger */ + private static final Logger log = LoggerFactory.getLogger(OpenTSDBMain.class); + /** The content prefix */ + public static final String CONTENT_PREFIX = "queryui"; + + /** The default pid file directory which is ${user.home}/.tsdb/ */ + public static final File DEFAULT_PID_DIR = new File(System.getProperty("user.home") + File.separator + ".tsdb"); + /** The default pid file which is ${user.home}/.tsdb/opentsdb.pid */ + public static final File DEFAULT_PID_FILE = new File(DEFAULT_PID_DIR, "opentsdb.pid"); + + /** The default flush interval */ + public static final short DEFAULT_FLUSH_INTERVAL = 1000; + + /** Boot classes keyed by the commands we recognize */ + public static final Map> COMMANDS; + + static { + Map> tmp = new HashMap>(); + tmp.put("fsck", Fsck.class); + tmp.put("import", TextImporter.class); + tmp.put("mkmetric", UidManager.class); // -> shift --> set uid assign metrics "$@" + tmp.put("query", CliQuery.class); + tmp.put("tsd", OpenTSDBMain.class); + tmp.put("scan", DumpSeries.class); + tmp.put("uid", UidManager.class); + tmp.put("exportui", UIContentExporter.class); + tmp.put("help", HelpProcessor.class); + COMMANDS = Collections.unmodifiableMap(tmp); + } + + /** + * Prints the main usage banner + */ + public static void mainUsage(PrintStream ps) { + StringBuilder b = new StringBuilder("\nUsage: java -jar [opentsdb.jar] [command] [args]\nValid commands:") + .append("\n\ttsd: Starts a new TSDB instance") + .append("\n\tfsck: Searches for and optionally fixes corrupted data in a TSDB") + .append("\n\timport: Imports data from a file into HBase through a TSDB") + .append("\n\tmkmetric: Creates a new metric") + .append("\n\tquery: Queries time series data from a TSDB ") + .append("\n\tscan: Dumps data straight from HBase") + .append("\n\tuid: Provides various functions to search or modify information in the tsdb-uid table. ") + .append("\n\texportui: Exports the OpenTSDB UI static content") + .append("\n\n\tUse help for details on a command\n"); + ps.println(b); + } + + + /** + * The OpenTSDB fat-jar main entry point + * @param args See usage banner {@link OpenTSDBMain#mainUsage(PrintStream)} + */ + public static void main(String[] args) { + log.info("Starting."); + log.info(BuildData.revisionString()); + log.info(BuildData.buildString()); + try { + System.in.close(); // Release a FD we don't need. + } catch (Exception e) { + log.warn("Failed to close stdin", e); + } + if(args.length==0) { + log.error("No command supplied"); + mainUsage(System.err); + System.exit(-1); + } + // This is not normally needed since values passed on the CL are auto-trimmed, + // but since the Main may be called programatically in some embedded scenarios, + // let's save us some time and trim the values here. + for(int i = 0; i < args.length; i++) { + args[i] = args[i].trim(); + } + String targetTool = args[0].toLowerCase(); + if(!COMMANDS.containsKey(targetTool)) { + log.error("Command not recognized: [" + targetTool + "]"); + mainUsage(System.err); + System.exit(-1); + } + process(targetTool, shift(args)); + } + + /** + * Executes the target tool + * @param targetTool the name of the target tool to execute + * @param args The command line arguments minus the tool name + */ + private static void process(String targetTool, String[] args) { + if("mkmetric".equals(targetTool)) { + shift(args); + } + if(!"tsd".equals(targetTool)) { + try { + COMMANDS.get(targetTool).getDeclaredMethod("main", String[].class).invoke(null, new Object[] {args}); + } catch(Exception x) { + log.error("Failed to call [" + targetTool + "].", x); + System.exit(-1); + } + } else { + launchTSD(args); + } + } + + + /** + * Applies and processes the pre-tsd command line + * @param cap The main configuration wrapper + * @param argp The preped command line argument handler + */ + protected static void applyCommandLine(ConfigArgP cap, ArgP argp) { + // --config, --include-config, --help + if(argp.has("--help")) { + if(cap.hasNonOption("extended")) { + System.out.println(cap.getExtendedUsage("tsd extended usage:")); + } else { + System.out.println(cap.getDefaultUsage("tsd usage:")); + } + System.exit(0); + } + if(argp.has("--config")) { + loadConfigSource(cap, argp.get("--config").trim()); + } + if(argp.has("--include")) { + String[] sources = argp.get("--include").split(","); + for(String s: sources) { + loadConfigSource(cap, s.trim()); + } + } + } + + /** + * Applies the properties from the named source to the main configuration + * @param config the main configuration to apply to + * @param source the name of the source to apply properties from + */ + protected static void loadConfigSource(ConfigArgP config, String source) { + Properties p = loadConfig(source); + Config c = config.getConfig(); + for(String key: p.stringPropertyNames()) { + String value = p.getProperty(key); + ConfigurationItem ci = config.getConfigurationItem(key); + if(ci!=null) { // if we recognize the key, validate it + ci.setValue(value); + } + c.overrideConfig(key, value); + } + } + + /** + * Loads properties from a file or url with the passed name + * @param name The name of the file or URL + * @return the loaded properties + */ + protected static Properties loadConfig(String name) { + try { + URL url = new URL(name); + return loadConfig(url); + } catch (Exception ex) { + return loadConfig(new File(name)); + } + } + + /** + * Loads properties from the passed input stream + * @param source The name of the source the properties are being loaded from + * @param is The input stream to load from + * @return the loaded properties + */ + protected static Properties loadConfig(String source, InputStream is) { + try { + Properties p = new Properties(); + p.load(is); + // trim the value as it may have trailing white-space + Set keys = p.stringPropertyNames(); + for(String key: keys) { + p.setProperty(key, p.getProperty(key).trim()); + } + return p; + } catch (IllegalArgumentException iae) { + throw iae; + } catch (Exception ex) { + throw new IllegalArgumentException("Failed to load configuration from [" + source + "]"); + } + } + + /** + * Loads properties from the passed file + * @param file The file to load from + * @return the loaded properties + */ + protected static Properties loadConfig(File file) { + InputStream is = null; + try { + is = new FileInputStream(file); + return loadConfig(file.getAbsolutePath(), is); + } catch (Exception ex) { + throw new IllegalArgumentException("Failed to load configuration from [" + file.getAbsolutePath() + "]"); + }finally { + if(is!=null) try { is.close(); } catch (Exception ex) { /* No Op */ } + } + } + + /** + * Loads properties from the passed URL + * @param url The url to load from + * @return the loaded properties + */ + protected static Properties loadConfig(URL url) { + InputStream is = null; + try { + URLConnection connection = url.openConnection(); + if(connection instanceof HttpURLConnection) { + ((HttpURLConnection)connection).setConnectTimeout(2000); + } + is = connection.getInputStream(); + return loadConfig(url.toString(), is); + } catch (Exception ex) { + throw new IllegalArgumentException("Failed to load configuration from [" + url + "]"); + }finally { + if(is!=null) try { is.close(); } catch (Exception ex) { /* No Op */ } + } + } + + /** Prints usage and exits with the given retval. */ + static void usage(final ArgP argp, final String errmsg, final int retval) { + System.err.println(errmsg); + System.err.println(new ConfigArgP().getDefaultUsage()); + if (argp != null) { + System.err.print(argp.usage()); + } + System.exit(retval); + } + + + + /** + * Starts the TSD. + * @param args The command line arguments + */ + private static void launchTSD(String[] args) { + ConfigArgP cap = new ConfigArgP(args); + Config config = cap.getConfig(); + ArgP argp = cap.getArgp(); + applyCommandLine(cap, argp); + config.loadStaticVariables(); + // All options are now correctly set in config + setJVMName(config.getInt("tsd.network.port"), config.getString("tsd.network.bind")); + // Configure the logging + if(config.hasProperty("tsd.logback.file")) { + final String logBackFile = config.getString("tsd.logback.file"); + final String rollPattern = config.hasProperty("tsd.logback.rollpattern") ? config.getString("tsd.logback.rollpattern") : null; + final boolean keepConsoleOpen = config.hasProperty("tsd.logback.console") ? config.getBoolean("tsd.logback.console") : false; + log.info("\n\t===================================\n\tReconfiguring logback. Logging to file:\n\t{}\n\t===================================\n", logBackFile); + setLogbackInternal(logBackFile, rollPattern, keepConsoleOpen); + } else { + final String logBackConfig; + if(config.hasProperty("tsd.logback.config")) { + logBackConfig = config.getString("tsd.logback.config"); + } else { + logBackConfig = System.getProperty("tsd.logback.config", null); + } + if(logBackConfig!=null && !logBackConfig.trim().isEmpty() && new File(logBackConfig.trim()).canRead()) { + setLogbackExternal(logBackConfig.trim()); + } + } + if(config.auto_metric()) { + log.info("\n\t==========================================\n\tAuto-Metric Enabled\n\t==========================================\n"); + } else { + log.warn("\n\t==========================================\n\tAuto-Metric Disabled\n\t==========================================\n"); + } + try { + // Write the PID file + writePid(config.getString("tsd.process.pid.file"), config.getBoolean("tsd.process.pid.ignore.existing")); + // Export the UI content + if(!config.getBoolean("tsd.ui.noexport")) { + loadContent(config.getString("tsd.http.staticroot")); + } + // Create the cache dir if it does not exist + File cacheDir = new File(config.getString("tsd.http.cachedir")); + if(cacheDir.exists()) { + if(!cacheDir.isDirectory()) { + throw new IllegalArgumentException("The http cache directory [" + cacheDir + "] is not a directory, but a file, which is bad"); + } + } else { + if(!cacheDir.mkdirs()) { + throw new IllegalArgumentException("Failed to create the http cache directory [" + cacheDir + "]"); + } + } + } catch (Exception ex) { + log.error("Failed to process tsd configuration", ex); + System.exit(-1); + } + + // ===================================================================== + // Command line processing complete, ready to start TSD. + // The code from here to the end of the method is an exact duplicate + // of {@link TSDMain#main(String[])} once configuration is complete. + // At the time of this writing, this is at line 123 starting with the + // code: final ServerSocketChannelFactory factory; + // ===================================================================== + + log.info("Configuration complete. Starting TSDB"); + final ServerSocketChannelFactory factory; + if (config.getBoolean("tsd.network.async_io")) { + int workers = Runtime.getRuntime().availableProcessors() * 2; + if (config.hasProperty("tsd.network.worker_threads")) { + try { + workers = config.getInt("tsd.network.worker_threads"); + } catch (NumberFormatException nfe) { + usage(argp, "Invalid worker thread count", 1); + } + } + final Executor executor = Executors.newCachedThreadPool(); + final NioServerBossPool boss_pool = + new NioServerBossPool(executor, 1, new Threads.BossThreadNamer()); + final NioWorkerPool worker_pool = new NioWorkerPool(executor, + workers, new Threads.WorkerThreadNamer()); + factory = new NioServerSocketChannelFactory(boss_pool, worker_pool); + } else { + factory = new OioServerSocketChannelFactory( + Executors.newCachedThreadPool(), Executors.newCachedThreadPool()); + } + + TSDB tsdb = null; + try { + tsdb = new TSDB(config); + tsdb.initializePlugins(true); + + // Make sure we don't even start if we can't find our tables. + tsdb.checkNecessaryTablesExist().joinUninterruptibly(); + + registerShutdownHook(tsdb); + final ServerBootstrap server = new ServerBootstrap(factory); + + server.setPipelineFactory(new PipelineFactory(tsdb)); + if (config.hasProperty("tsd.network.backlog")) { + server.setOption("backlog", config.getInt("tsd.network.backlog")); + } + server.setOption("child.tcpNoDelay", + config.getBoolean("tsd.network.tcp_no_delay")); + server.setOption("child.keepAlive", + config.getBoolean("tsd.network.keep_alive")); + server.setOption("reuseAddress", + config.getBoolean("tsd.network.reuse_address")); + + // null is interpreted as the wildcard address. + InetAddress bindAddress = null; + if (config.hasProperty("tsd.network.bind")) { + bindAddress = InetAddress.getByName(config.getString("tsd.network.bind")); + } + + // we validated the network port config earlier + final InetSocketAddress addr = new InetSocketAddress(bindAddress, + config.getInt("tsd.network.port")); + server.bind(addr); + log.info("Ready to serve on " + addr); + } catch (Throwable e) { + factory.releaseExternalResources(); + try { + if (tsdb != null) + tsdb.shutdown().joinUninterruptibly(); + } catch (Exception e2) { + log.error("Failed to shutdown HBase client", e2); + } + throw new RuntimeException("Initialization failed", e); + } + // The server is now running in separate threads, we can exit main. + } + + + /** + * Attempts to set the vm agent property that identifies the vm's display name. + * This is the name displayed for tools such as jconsole and jps when using auto-dicsovery. + * When using a fat-jar, this provides a much more identifiable name + * @param port The listening port + * @param iface The bound interface + */ + protected static void setJVMName(final int port, final String iface) { + final Properties p = getAgentProperties(); + if(p!=null) { + final String ifc = (iface==null || iface.trim().isEmpty()) ? "" : (iface.trim() + ":"); + final String name = "opentsdb[" + ifc + port + "]"; + p.setProperty("sun.java.command", name); + p.setProperty("sun.rt.javaCommand", name); + System.setProperty("sun.java.command", name); + System.setProperty("sun.rt.javaCommand", name); + } + } + + /** + * Returns the agent properties + * @return the agent properties or null if reflective call failed + */ + protected static Properties getAgentProperties() { + try { + Class clazz = Class.forName("sun.misc.VMSupport"); + Method m = clazz.getDeclaredMethod("getAgentProperties"); + m.setAccessible(true); + Properties p = (Properties)m.invoke(null); + return p; + } catch (Throwable t) { + return null; + } + } + + /** + * Sets the logback system property, tsdb.logback.file and then + * reloads the internal file based logging configuration. The property will + * not be set if it was already set, allowing the value to be set by a standard + * command line set system property. + * @param logFileName The name of the file logback will log to. + * @param rollPattern The pattern specifying the rolling file pattern, + * inserted between the file name base and extension. So for a log file name of + * /var/log/opentsdb.log and a pattern of _%d{yyyy-MM-dd}.%i, + * the configured fileNamePattern would be /var/log/opentsdb_%d{yyyy-MM-dd}.%i.log + * @param keepConsoleOpen If true, will keep the console appender open, + * otherwise closes it and logs exclusively to the file. + */ + protected static void setLogbackInternal(final String logFileName, final String rollPattern, final boolean keepConsoleOpen) { + if(System.getProperty("tsdb.logback.file", null) == null) { + System.setProperty("tsdb.logback.file", logFileName); + } + System.setProperty("tsd.logback.rollpattern", insertPattern(logFileName, rollPattern)); + BasicStatusManager bsm = new BasicStatusManager(); + for(StatusListener listener: bsm.getCopyOfStatusListenerList()) { + log.info("Status Listener: {}", listener); + } + try { + final URL url = OpenTSDBMain.class.getClassLoader().getResource("file-logback.xml"); + final LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory(); + try { + final JoranConfigurator configurator = new JoranConfigurator(); + configurator.setContext(context); + configurator.doConfigure(url); + if(!keepConsoleOpen) { + final ch.qos.logback.classic.Logger root = (ch.qos.logback.classic.Logger)LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME); + root.detachAppender("STDOUT"); + } + log.info("Set internal logback config with file [{}] and roll pattern [{}]", logFileName, rollPattern); + } catch (JoranException je) { + System.err.println("Failed to configure internal logback"); + je.printStackTrace(System.err); + } + } catch (Exception ex) { + log.warn("Failed to set internal logback config with file [{}]", logFileName, ex); + } + } + + /** + * Merges the roll pattern name into the file name + * @param logFileName The log file name + * @param rollPattern The rolling log file appender roll pattern + * @return the merged file pattern + */ + protected static String insertPattern(final String logFileName, final String rollPattern) { + int index = logFileName.lastIndexOf('.'); + if(index==-1) return logFileName + rollPattern; + return logFileName.substring(0, index) + rollPattern + logFileName.substring(index); + } + + + /** + * Reloads the logback configuration to an external file + * @param fileName The logback configuration file + */ + protected static void setLogbackExternal(final String fileName) { + try { + final ObjectName logbackObjectName = new ObjectName("ch.qos.logback.classic:Name=default,Type=ch.qos.logback.classic.jmx.JMXConfigurator"); + ManagementFactory.getPlatformMBeanServer().invoke(logbackObjectName, "reloadByFileName", new Object[]{fileName}, new String[]{String.class.getName()}); + log.info("Set external logback config to [{}]", fileName); + } catch (Exception ex) { + log.warn("Failed to set external logback config to [{}]", fileName, ex); + } + } + + + /** + * Drops the first array item in the passed array. + * If the passed array is null or empty, returns an empty array + * @param args The array to shift + * @return the shifted array + */ + private static String[] shift(String[] args) { + if(args==null || args.length==0 | args.length==1) return new String[0]; + String[] newArgs = new String[args.length-1]; + System.arraycopy(args, 1, newArgs, 0, newArgs.length); + return newArgs; + } + + + private static void registerShutdownHook(final TSDB tsdb) { + final class TSDBShutdown extends Thread { + public TSDBShutdown() { + super("TSDBShutdown"); + } + public void run() { + try { + tsdb.shutdown().join(); + } catch (Exception e) { + LoggerFactory.getLogger(TSDBShutdown.class) + .error("Uncaught exception during shutdown", e); + } + } + } + Runtime.getRuntime().addShutdownHook(new TSDBShutdown()); + } + + + /** + *

    Title: HelpProcessor

    + *

    Description: Command line help processor

    + */ + public static class HelpProcessor { + /** + * Entry point for invoking the ui content exporter + * @param args see the ArgP + */ + public static void main(String[] args) { + if(args==null || args.length==0) { + mainUsage(System.out); + } else { + Class command = COMMANDS.get(args[0].trim()); + if(command==null) { + System.err.println("\nUnrecognized command [" + args[0] + "]\n"); + mainUsage(System.err); + } else { + try { + if(args[0].equals("tsd")) { + ConfigArgP cap = new ConfigArgP(); + if(args.length>1 && args[1].trim().equals("extended")) { + System.out.println(cap.getExtendedUsage("tsd extended usage:")); + } else { + System.out.println(cap.getExtendedUsage("tsd usage:")); + } + } else { + Method m = null; + ArgP fake = new ArgP(); + try { + m = command.getDeclaredMethod("usage", ArgP.class, String.class); + m.setAccessible(true); + m.invoke(null, new Object[] {fake, "\nHelp for [" + args[0] + "] command\n"}); + } catch (NoSuchMethodException ne) { + try { + m = command.getDeclaredMethod("usage", ArgP.class, String.class, int.class); + m.setAccessible(true); + m.invoke(null, new Object[] {fake, "\nHelp for [" + args[0] + "] command\n", 1}); + } catch (NoSuchMethodException ne2) { + m = command.getDeclaredMethod("usage", ArgP.class, int.class); + m.setAccessible(true); + m.invoke(null, new Object[] {fake, 1}); + } + } + } + } catch(Exception x) { + log.error("Failed to invoke help for [" + args[0] + "].", x); + System.exit(-1); + } + } + } + System.exit(0); + } + + /** + * Prints help usage + * @param argp Ignored + * @param errmsg Ignored + */ + static void usage(final ArgP argp, final String errmsg) { + System.out.println("help: Prints the main command line help"); + System.out.println("help: Prints help for the specified command"); + } + + } + + /** + *

    Title: UIContentExporter

    + *

    Description: Exports the queryui content from the jar to the specified directory

    + */ + public static class UIContentExporter { + private static final ArgP uiexOptions = new ArgP(); + + static { + uiexOptions.addOption("--d", "DIR", "The directory to export the UI content to"); + uiexOptions.addOption("--p", "Create the directory if it does not exist"); + } + + + /** + * Usage banner, args are not used, just approximating a consistent signature + * @param ignored ignored + * @param alsoIgnored ignored + */ + public static void usage(ArgP ignored, int alsoIgnored) { + System.out.println("Usage: java -jar exportui --d [--p]\n" + + uiexOptions.usage() ); + + } + + + /** + * Entry point for invoking the ui content exporter + * @param args see the ArgP + */ + public static void main(String[] args) { + uiexOptions.parse(args); + String dirName = uiexOptions.get("--d"); + boolean createIfNotExists = uiexOptions.has("--p"); + if(dirName==null) { + log.error("Missing argument for target directory. Usage: java -jar exportui --d [--p]"); + System.exit(1); + } + File f = new File(dirName); + if(!f.exists()) { + if(createIfNotExists) { + if(f.mkdirs()) { + log.info("Created exportui directory [{}]", f); + } else { + log.error("Failed to create target directory [{}]", f); + System.exit(1); + } + } else { + log.error("Specified target directory [{}] does not exist. You could use the --p option, or create the directory", f); + System.exit(1); + } + } else { + if(!f.isDirectory()) { + log.error("Specified target [{}] is not a directory, but is a file. exportui cannot contine", f); + System.exit(1); + } + } + loadContent(f.getAbsolutePath()); + System.exit(0); + } + } + /** + * Loads the Static UI content files from the classpath JAR to the configured static root directory + * @param the name of the content directory to write the content to + */ + private static void loadContent(String contentDirectory) { + File gpDir = new File(contentDirectory); + final long startTime = System.currentTimeMillis(); + int filesLoaded = 0; + int fileFailures = 0; + int fileNewer = 0; + long bytesLoaded = 0; + String codeSourcePath = TSDMain.class.getProtectionDomain().getCodeSource().getLocation().getPath(); + File file = new File(codeSourcePath); + if( codeSourcePath.endsWith(".jar") && file.exists() && file.canRead() ) { + JarFile jar = null; + ChannelBuffer contentBuffer = ChannelBuffers.dynamicBuffer(300000); + try { + jar = new JarFile(file); + final Enumeration entries = jar.entries(); + while(entries.hasMoreElements()) { + JarEntry entry = entries.nextElement(); + final String name = entry.getName(); + if (name.startsWith(CONTENT_PREFIX + "/")) { + final int contentSize = (int)entry.getSize(); + final long contentTime = entry.getTime(); + if(entry.isDirectory()) { + new File(gpDir, name).mkdirs(); + continue; + } + File contentFile = new File(gpDir, name.replace(CONTENT_PREFIX + "/", "")); + if( !contentFile.getParentFile().exists() ) { + contentFile.getParentFile().mkdirs(); + } + if( contentFile.exists() ) { + if( contentFile.lastModified() >= contentTime ) { + log.debug("File in directory was newer [{}]", name); + fileNewer++; + continue; + } + contentFile.delete(); + } + log.debug("Writing content file [{}]", contentFile ); + contentFile.createNewFile(); + if( !contentFile.canWrite() ) { + log.warn("Content file [{}] not writable", contentFile); + fileFailures++; + continue; + } + FileOutputStream fos = null; + InputStream jis = null; + try { + fos = new FileOutputStream(contentFile); + jis = jar.getInputStream(entry); + contentBuffer.writeBytes(jis, contentSize); + contentBuffer.readBytes(fos, contentSize); + fos.flush(); + jis.close(); jis = null; + fos.close(); fos = null; + filesLoaded++; + bytesLoaded += contentSize; + log.debug("Wrote content file [{}] + with size [{}]", contentFile, contentSize ); + } finally { + if( jis!=null ) try { jis.close(); } catch (Exception ex) {} + if( fos!=null ) try { fos.close(); } catch (Exception ex) {} + } + } // not content + } // end of while loop + final long elapsed = System.currentTimeMillis()-startTime; + StringBuilder b = new StringBuilder("\n\n\t===================================================\n\tStatic Root Directory:[").append(contentDirectory).append("]"); + b.append("\n\tTotal Files Written:").append(filesLoaded); + b.append("\n\tTotal Bytes Written:").append(bytesLoaded); + b.append("\n\tFile Write Failures:").append(fileFailures); + b.append("\n\tExisting File Newer Than Content:").append(fileNewer); + b.append("\n\tElapsed (ms):").append(elapsed); + b.append("\n\t===================================================\n"); + log.info(b.toString()); + } catch (Exception ex) { + log.error("Failed to export ui content", ex); + } finally { + if( jar!=null ) try { jar.close(); } catch (Exception x) { /* No Op */} + } + } else { // end of was-not-a-jar + log.warn("\n\tThe OpenTSDB classpath is not a jar file, so there is no content to unload.\n\tBuild the OpenTSDB jar and run 'java -jar --d '."); + } + } + + /** + * Writes the PID to the file at the passed location + * @param file The fully qualified pid file name + * @param ignorePidFile If true, an existing pid file will be ignored after a warning log + */ + private static void writePid(String file, boolean ignorePidFile) { + File pidFile = new File(file); + if(pidFile.exists()) { + Long oldPid = getPid(pidFile); + if(oldPid==null) { + pidFile.delete(); + } else { + log.warn("\n\t==================================\n\tThe OpenTSDB PID file [" + file + "] already exists for PID [" + oldPid + "]. \n\tOpenTSDB might already be running.\n\t==================================\n"); + if(!ignorePidFile) { + log.warn("Exiting due to existing pid file. Start with option --ignore-existing-pid to overwrite"); + System.exit(-1); + } else { + log.warn("Deleting existing pid file [" + file + "]"); + pidFile.delete(); + } + } + } + pidFile.deleteOnExit(); + File pidDir = pidFile.getParentFile(); + FileOutputStream fos = null; + try { + if(!pidDir.exists()) { + if(!pidDir.mkdirs()) { + throw new Exception("Failed to create PID directory [" + file + "]"); + } + } + fos = new FileOutputStream(pidFile); + String PID = ManagementFactory.getRuntimeMXBean().getName().split("@")[0]; + fos.write(String.format("%s%s", PID, EOL).getBytes()); + fos.flush(); + fos.close(); + fos = null; + log.info("PID [" + PID + "] written to pid file [" + file + "]"); + } catch (Exception ex) { + log.error("Failed to write PID file to [" + file + "]", ex); + throw new IllegalArgumentException("Failed to write PID file to [" + file + "]", ex); + } finally { + if(fos!=null) try { fos.close(); } catch (Exception ex) { /* No Op */ } + } + } + + /** + * Reads the pid from the specified pid file + * @param pidFile The pid file to read from + * @return The read pid or possibly null / blank if failed to read + */ + private static Long getPid(File pidFile) { + FileReader reader = null; + BufferedReader lineReader = null; + String pidLine = null; + try { + reader = new FileReader(pidFile); + lineReader = new BufferedReader(reader); + pidLine = lineReader.readLine(); + if(pidLine!=null) { + pidLine = pidLine.trim(); + } + } catch (Exception ex) { + log.error("Failed to read PID from file [" + pidFile.getAbsolutePath() + "]", ex); + } finally { + if(reader!=null) try { reader.close(); } catch (Exception ex) { /* No Op */ } + } + try { + return Long.parseLong(pidLine); + } catch (Exception ex) { + return null; + } + } + +} \ No newline at end of file diff --git a/src/tools/StartupPlugin.java b/src/tools/StartupPlugin.java new file mode 100644 index 0000000000..e8aae0d5e5 --- /dev/null +++ b/src/tools/StartupPlugin.java @@ -0,0 +1,86 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.tools; + +import com.stumbleupon.async.Deferred; + +import net.opentsdb.utils.Config; +import net.opentsdb.core.TSDB; +import net.opentsdb.stats.StatsCollector; + +/** + * The StartupPlugin allows users to interact with the OpenTSDB configuration + * as soon as it is completely parsed, just before OpenTSDB begins to use it. + *

    + * Note: Implementations must have a parameterless constructor. The + * {@link #initialize(Config)} method will be called immediately after the plugin is + * instantiated and before any other methods are called. + * + * @since 2.3 + */ +public abstract class StartupPlugin { + + /** + * Called by TSDB to initialize the plugin + * Implementations are responsible for setting up any IO they need as well + * as starting any required background threads. + * Note: Implementations should throw exceptions if they can't start + * up properly. The TSD will then shutdown so the operator can fix the + * problem. Please use IllegalArgumentException for configuration issues. + * @param config The OpenTSDDB config object. + * @return A reference to the same configuration object passed in the parameters + * on success. + * @throws IllegalArgumentException if required configuration parameters are + * missing + */ + public abstract Config initialize(Config config); + + /** + * Called when the TSD is fully initialized and ready to handle traffic. + */ + public abstract void setReady(final TSDB tsdb); + + /** + * Called to gracefully shutdown the plugin. Implementations should close + * any IO they have open + * @return A deferred object that indicates the completion of the request. + * The {@link Object} has not special meaning and can be {@code null} + * (think of it as {@code Deferred}). + */ + public abstract Deferred shutdown(); + + /** + * Should return the version of this plugin in the format: + * MAJOR.MINOR.MAINT, e.g. "2.0.1". The MAJOR version should match the major + * version of OpenTSDB the plugin is meant to work with. + * @return A version string used to log the loaded version + */ + public abstract String version(); + + /** + * Should return the version of this plugin in the format: + * MAJOR.MINOR.MAINT, e.g. "2.0.1". The MAJOR version should match the major + * version of OpenTSDB the plugin is meant to work with. + * @return A version string used to log the loaded version + */ + public abstract String getType(); + + /** + * Called by the TSD when a request for statistics collection has come in. The + * implementation may provide one or more statistics. If no statistics are + * available for the implementation, simply stub the method. + * @param collector The collector used for emitting statistics + */ + public abstract void collectStats(final StatsCollector collector); + +} \ No newline at end of file diff --git a/src/tools/TSDMain.java b/src/tools/TSDMain.java index 82dd37741d..a5bda860a0 100644 --- a/src/tools/TSDMain.java +++ b/src/tools/TSDMain.java @@ -12,26 +12,36 @@ // see . package net.opentsdb.tools; -import java.io.File; import java.io.IOException; +import java.lang.reflect.Constructor; + import java.net.InetAddress; import java.net.InetSocketAddress; +import java.util.concurrent.Executor; import java.util.concurrent.Executors; +import java.util.HashMap; +import java.util.Map; +import org.jboss.netty.bootstrap.ServerBootstrap; import org.jboss.netty.channel.socket.ServerSocketChannelFactory; +import org.jboss.netty.channel.socket.nio.NioServerBossPool; +import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory; +import org.jboss.netty.channel.socket.nio.NioWorkerPool; import org.jboss.netty.channel.socket.oio.OioServerSocketChannelFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.jboss.netty.bootstrap.ServerBootstrap; -import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory; -import net.opentsdb.BuildData; +import net.opentsdb.tools.BuildData; import net.opentsdb.core.TSDB; import net.opentsdb.core.Const; import net.opentsdb.tsd.PipelineFactory; +import net.opentsdb.tsd.RpcManager; import net.opentsdb.utils.Config; import net.opentsdb.utils.FileSystem; -import net.opentsdb.graph.Plot; +import net.opentsdb.utils.Pair; +import net.opentsdb.utils.PluginLoader; +import net.opentsdb.utils.Threads; + /** * Main class of the TSD, the Time Series Daemon. */ @@ -49,8 +59,15 @@ static void usage(final ArgP argp, final String errmsg, final int retval) { System.exit(retval); } - private static final short DEFAULT_FLUSH_INTERVAL = 1000; + /** A map of configured filters for use in querying */ + private static Map, Constructor>> + startupPlugin_filter_map = new HashMap, Constructor>>(); + private static final short DEFAULT_FLUSH_INTERVAL = 1000; + + private static TSDB tsdb = null; + public static void main(String[] args) throws IOException { Logger log = LoggerFactory.getLogger(TSDMain.class); log.info("Starting."); @@ -74,12 +91,21 @@ public static void main(String[] args) throws IOException { "Number for async io workers (default: cpu * 2)."); argp.addOption("--async-io", "true|false", "Use async NIO (default true) or traditional blocking io"); + argp.addOption("--read-only", "true|false", + "Set tsd.mode to ro (default false)"); + argp.addOption("--disable-ui", "true|false", + "Set tsd.core.enable_ui to false (default true)"); + argp.addOption("--disable-api", "true|false", + "Set tsd.core.enable_api to false (default true)"); argp.addOption("--backlog", "NUM", "Size of connection attempt queue (default: 3072 or kernel" + " somaxconn."); + argp.addOption("--max-connections", "NUM", + "Maximum number of connections to accept"); argp.addOption("--flush-interval", "MSEC", "Maximum time for which a new data point can be buffered" + " (default: " + DEFAULT_FLUSH_INTERVAL + ")."); + argp.addOption("--statswport", "Force all stats to include the port"); CliOptions.addAutoMetricFlag(argp); args = CliOptions.parse(argp, args); args = null; // free(). @@ -119,6 +145,12 @@ public static void main(String[] args) throws IOException { } final ServerSocketChannelFactory factory; + int connections_limit = 0; + try { + connections_limit = config.getInt("tsd.core.connections.limit"); + } catch (NumberFormatException nfe) { + usage(argp, "Invalid connections limit", 1); + } if (config.getBoolean("tsd.network.async_io")) { int workers = Runtime.getRuntime().availableProcessors() * 2; if (config.hasProperty("tsd.network.worker_threads")) { @@ -128,26 +160,48 @@ public static void main(String[] args) throws IOException { usage(argp, "Invalid worker thread count", 1); } } - factory = new NioServerSocketChannelFactory( - Executors.newCachedThreadPool(), Executors.newCachedThreadPool(), - workers); + final Executor executor = Executors.newCachedThreadPool(); + final NioServerBossPool boss_pool = + new NioServerBossPool(executor, 1, new Threads.BossThreadNamer()); + final NioWorkerPool worker_pool = new NioWorkerPool(executor, + workers, new Threads.WorkerThreadNamer()); + factory = new NioServerSocketChannelFactory(boss_pool, worker_pool); } else { factory = new OioServerSocketChannelFactory( - Executors.newCachedThreadPool(), Executors.newCachedThreadPool()); + Executors.newCachedThreadPool(), Executors.newCachedThreadPool(), + new Threads.PrependThreadNamer()); } - - TSDB tsdb = null; + + StartupPlugin startup = null; + try { + startup = loadStartupPlugins(config); + } catch (IllegalArgumentException e) { + usage(argp, e.getMessage(), 3); + } catch (Exception e) { + throw new RuntimeException("Initialization failed", e); + } + try { tsdb = new TSDB(config); + if (startup != null) { + tsdb.setStartupPlugin(startup); + } tsdb.initializePlugins(true); + if (config.getBoolean("tsd.storage.hbase.prefetch_meta")) { + tsdb.preFetchHBaseMeta(); + } // Make sure we don't even start if we can't find our tables. tsdb.checkNecessaryTablesExist().joinUninterruptibly(); - - registerShutdownHook(tsdb); + + registerShutdownHook(); final ServerBootstrap server = new ServerBootstrap(factory); + + // This manager is capable of lazy init, but we force an init + // here to fail fast. + final RpcManager manager = RpcManager.instance(tsdb); - server.setPipelineFactory(new PipelineFactory(tsdb)); + server.setPipelineFactory(new PipelineFactory(tsdb, manager, connections_limit)); if (config.hasProperty("tsd.network.backlog")) { server.setOption("backlog", config.getInt("tsd.network.backlog")); } @@ -168,6 +222,9 @@ public static void main(String[] args) throws IOException { final InetSocketAddress addr = new InetSocketAddress(bindAddress, config.getInt("tsd.network.port")); server.bind(addr); + if (startup != null) { + startup.setReady(tsdb); + } log.info("Ready to serve on " + addr); } catch (Throwable e) { factory.releaseExternalResources(); @@ -182,14 +239,60 @@ public static void main(String[] args) throws IOException { // The server is now running in separate threads, we can exit main. } - private static void registerShutdownHook(final TSDB tsdb) { + private static StartupPlugin loadStartupPlugins(Config config) { + Logger log = LoggerFactory.getLogger(TSDMain.class); + + // load the startup plugin if enabled + StartupPlugin startup = null; + + if (config.getBoolean("tsd.startup.enable")) { + log.debug("Startup Plugin is Enabled"); + final String plugin_path = config.getString("tsd.core.plugin_path"); + final String plugin_class = config.getString("tsd.startup.plugin"); + + log.debug("Plugin Path: " + plugin_path); + try { + TSDB.loadPluginPath(plugin_path); + } catch (Exception e) { + log.error("Error loading plugins from plugin path: " + plugin_path, e); + } + + log.debug("Attempt to Load: " + plugin_class); + startup = PluginLoader.loadSpecificPlugin(plugin_class, StartupPlugin.class); + if (startup == null) { + throw new IllegalArgumentException("Unable to locate startup plugin: " + + config.getString("tsd.startup.plugin")); + } + try { + startup.initialize(config); + } catch (Exception e) { + throw new RuntimeException("Failed to initialize startup plugin", e); + } + log.info("Successfully initialized startup plugin [" + + startup.getClass().getCanonicalName() + "] version: " + + startup.version()); + } else { + startup = null; + } + + return startup; + } + + private static void registerShutdownHook() { final class TSDBShutdown extends Thread { public TSDBShutdown() { super("TSDBShutdown"); } public void run() { try { - tsdb.shutdown().join(); + if (RpcManager.isInitialized()) { + // Check that its actually been initialized. We don't want to + // create a new instance only to shutdown! + RpcManager.instance(tsdb).shutdown().join(); + } + if (tsdb != null) { + tsdb.shutdown().join(); + } } catch (Exception e) { LoggerFactory.getLogger(TSDBShutdown.class) .error("Uncaught exception during shutdown", e); diff --git a/src/tools/TextImporter.java b/src/tools/TextImporter.java index fb501f4957..2f33f009be 100644 --- a/src/tools/TextImporter.java +++ b/src/tools/TextImporter.java @@ -1,5 +1,5 @@ // This file is part of OpenTSDB. -// Copyright (C) 2010-2012 The OpenTSDB Authors. +// Copyright (C) 2010-2014 The OpenTSDB Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU Lesser General Public License as published by @@ -53,24 +53,27 @@ public static void main(String[] args) throws Exception { ArgP argp = new ArgP(); CliOptions.addCommon(argp); CliOptions.addAutoMetricFlag(argp); + argp.addOption("--skip-errors", "Whether or not to skip exceptions " + + "during processing"); args = CliOptions.parse(argp, args); if (args == null) { usage(argp, 1); } else if (args.length < 1) { usage(argp, 2); } - + // get a config object Config config = CliOptions.getConfig(argp); - + final TSDB tsdb = new TSDB(config); + final boolean skip_errors = argp.has("--skip-errors"); tsdb.checkNecessaryTablesExist().joinUninterruptibly(); argp = null; try { int points = 0; final long start_time = System.nanoTime(); for (final String path : args) { - points += importFile(tsdb.getClient(), tsdb, path); + points += importFile(tsdb.getClient(), tsdb, path, skip_errors); } final double time_delta = (System.nanoTime() - start_time) / 1000000000.0; LOG.info(String.format("Total: imported %d data points in %.3fs" @@ -97,7 +100,8 @@ public final void emit(final String line) { private static int importFile(final HBaseClient client, final TSDB tsdb, - final String path) throws IOException { + final String path, + final boolean skip_errors) throws IOException { final long start_time = System.nanoTime(); long ping_start_time = start_time; final BufferedReader in = open(path); @@ -126,63 +130,116 @@ public String toString() { } }; final Errback errback = new Errback(); + LOG.info("reading from file:" + path); while ((line = in.readLine()) != null) { final String[] words = Tags.splitString(line, ' '); final String metric = words[0]; if (metric.length() <= 0) { - throw new RuntimeException("invalid metric: " + metric); + if (skip_errors) { + LOG.error("invalid metric: " + metric); + LOG.error("error while processing file " + + path + " line=" + line + "... Continuing"); + continue; + } else { + throw new RuntimeException("invalid metric: " + metric); + } } - final long timestamp = Tags.parseLong(words[1]); - if (timestamp <= 0) { - throw new RuntimeException("invalid timestamp: " + timestamp); + final long timestamp; + try { + timestamp = Tags.parseLong(words[1]); + if (timestamp <= 0) { + if (skip_errors) { + LOG.error("invalid timestamp: " + timestamp); + LOG.error("error while processing file " + + path + " line=" + line + "... Continuing"); + continue; + } else { + throw new RuntimeException("invalid timestamp: " + timestamp); + } + } + } catch (final RuntimeException e) { + if (skip_errors) { + LOG.error("invalid timestamp: " + e.getMessage()); + LOG.error("error while processing file " + + path + " line=" + line + "... Continuing"); + continue; + } else { + throw e; + } } + final String value = words[2]; if (value.length() <= 0) { - throw new RuntimeException("invalid value: " + value); - } - final HashMap tags = new HashMap(); - for (int i = 3; i < words.length; i++) { - if (!words[i].isEmpty()) { - Tags.parse(tags, words[i]); + if (skip_errors) { + LOG.error("invalid value: " + value); + LOG.error("error while processing file " + + path + " line=" + line + "... Continuing"); + continue; + } else { + throw new RuntimeException("invalid value: " + value); } } - final WritableDataPoints dp = getDataPoints(tsdb, metric, tags); - Deferred d; - if (Tags.looksLikeInteger(value)) { - d = dp.addPoint(timestamp, Tags.parseLong(value)); - } else { // floating point value - d = dp.addPoint(timestamp, Float.parseFloat(value)); - } - d.addErrback(errback); - points++; - if (points % 1000000 == 0) { - final long now = System.nanoTime(); - ping_start_time = (now - ping_start_time) / 1000000; - LOG.info(String.format("... %d data points in %dms (%.1f points/s)", - points, ping_start_time, - (1000000 * 1000.0 / ping_start_time))); - ping_start_time = now; - } - if (throttle) { - LOG.info("Throttling..."); - long throttle_time = System.nanoTime(); - try { - d.joinUninterruptibly(); - } catch (Exception e) { - throw new RuntimeException("Should never happen", e); + + try { + final HashMap tags = new HashMap(); + for (int i = 3; i < words.length; i++) { + if (!words[i].isEmpty()) { + Tags.parse(tags, words[i]); + } + } + + final WritableDataPoints dp = getDataPoints(tsdb, metric, tags); + Deferred d; + if (Tags.looksLikeInteger(value)) { + d = dp.addPoint(timestamp, Tags.parseLong(value)); + } else { // floating point value + d = dp.addPoint(timestamp, Float.parseFloat(value)); } - throttle_time = System.nanoTime() - throttle_time; - if (throttle_time < 1000000000L) { - LOG.info("Got throttled for only " + throttle_time + "ns, sleeping a bit now"); - try { Thread.sleep(1000); } catch (InterruptedException e) { throw new RuntimeException("interrupted", e); } + d.addErrback(errback); + points++; + if (points % 1000000 == 0) { + final long now = System.nanoTime(); + ping_start_time = (now - ping_start_time) / 1000000; + LOG.info(String.format("... %d data points in %dms (%.1f points/s)", + points, ping_start_time, + (1000000 * 1000.0 / ping_start_time))); + ping_start_time = now; + } + if (throttle) { + LOG.info("Throttling..."); + long throttle_time = System.nanoTime(); + try { + d.joinUninterruptibly(); + } catch (final Exception e) { + throw new RuntimeException("Should never happen", e); + } + throttle_time = System.nanoTime() - throttle_time; + if (throttle_time < 1000000000L) { + LOG.info("Got throttled for only " + throttle_time + + "ns, sleeping a bit now"); + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new RuntimeException("interrupted", e); + } + } + LOG.info("Done throttling..."); + throttle = false; + } + } catch (final RuntimeException e) { + if (skip_errors) { + LOG.error("Exception: " + e.getMessage()); + LOG.error("error while processing file " + + path + " line=" + line + "... Continuing"); + continue; + } else { + throw e; } - LOG.info("Done throttling..."); - throttle = false; } } } catch (RuntimeException e) { LOG.error("Exception caught while processing file " - + path + " line=" + line); + + path + " line=[" + line + "]", e); throw e; } finally { in.close(); @@ -202,6 +259,10 @@ public String toString() { * @throws IOException when shit happens. */ private static BufferedReader open(final String path) throws IOException { + if (path.equals("-")) { + return new BufferedReader(new InputStreamReader(System.in)); + } + InputStream is = new FileInputStream(path); if (path.endsWith(".gz")) { is = new GZIPInputStream(is); diff --git a/src/tools/UidManager.java b/src/tools/UidManager.java index 3e8a001ef6..1eead8bf10 100644 --- a/src/tools/UidManager.java +++ b/src/tools/UidManager.java @@ -17,6 +17,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeSet; @@ -24,7 +25,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import org.hbase.async.AtomicIncrementRequest; import org.hbase.async.Bytes; import org.hbase.async.DeleteRequest; @@ -40,6 +40,7 @@ import net.opentsdb.uid.NoSuchUniqueId; import net.opentsdb.uid.NoSuchUniqueName; import net.opentsdb.uid.UniqueId; +import net.opentsdb.uid.UniqueId.UniqueIdType; import net.opentsdb.utils.Config; /** @@ -64,6 +65,7 @@ static void usage(final ArgP argp, final String errmsg) { + " assign [names]:" + " Assign an ID for the given name(s).\n" + " rename : Renames this UID.\n" + + " delete : Deletes this UID.\n" + " fsck: [fix] [delete_unknown] Checks the consistency of UIDs.\n" + " fix - Fix errors. By default errors are logged.\n" + " delete_unknown - Remove columns with unknown qualifiers.\n" @@ -125,6 +127,7 @@ public static void main(String[] args) throws Exception { rc = runCommand(tsdb, table, idwidth, ignorecase, args); } finally { try { + LOG.info("Shutting down TSD...."); tsdb.getClient().shutdown().joinUninterruptibly(); LOG.info("Gracefully shutdown the TSD"); } catch (Exception e) { @@ -157,13 +160,25 @@ private static int runCommand(final TSDB tsdb, usage("Wrong number of arguments"); return 2; } - return assign(tsdb.getClient(), table, idwidth, args); + return assign(tsdb, table, idwidth, args); } else if (args[0].equals("rename")) { if (nargs != 4) { usage("Wrong number of arguments"); return 2; } return rename(tsdb.getClient(), table, idwidth, args); + } else if (args[0].equals("delete")) { + if (nargs != 3) { + usage("Wrong number of arguments"); + return 2; + } + + try { + return delete(tsdb, table, args); + } catch (Exception e) { + LOG.error("Unexpected exception", e); + return 4; + } } else if (args[0].equals("fsck")) { boolean fix = false; boolean fix_unknowns = false; @@ -315,6 +330,9 @@ private static int grep(final HBaseClient client, private static boolean printResult(final ArrayList row, final byte[] family, final boolean formard) { + if (null == row || row.isEmpty()) { + return false; + } final byte[] key = row.get(0).key(); String name = formard ? CliUtils.fromBytes(key) : null; String id = formard ? null : Arrays.toString(key); @@ -336,22 +354,27 @@ private static boolean printResult(final ArrayList row, /** * Implements the {@code assign} subcommand. - * @param client The HBase client to use. + * @param tsdb The TSDB to use. * @param table The name of the HBase table to use. * @param idwidth Number of bytes on which the UIDs should be. * @param args Command line arguments ({@code assign name [names]}). * @return The exit status of the command (0 means success). */ - private static int assign(final HBaseClient client, + private static int assign(final TSDB tsdb, final byte[] table, final short idwidth, final String[] args) { - final UniqueId uid = new UniqueId(client, table, args[1], (int) idwidth); + boolean randomize = false; + if (UniqueId.stringToUniqueIdType(args[1]) == UniqueIdType.METRIC) { + randomize = tsdb.getConfig().getBoolean("tsd.core.uid.random_metrics"); + } + final UniqueId uid = new UniqueId(tsdb.getClient(), table, args[1], + (int) idwidth, randomize); for (int i = 2; i < args.length; i++) { try { uid.getOrCreateId(args[i]); // Lookup again the ID we've just created and print it. - extactLookupName(client, table, idwidth, args[1], args[i]); + extactLookupName(tsdb.getClient(), table, idwidth, args[1], args[i]); } catch (HBaseException e) { LOG.error("error while processing " + args[i], e); return 3; @@ -390,6 +413,30 @@ private static int rename(final HBaseClient client, return 0; } + /** + * Implements the {@code delete} subcommand. + * @param client The HBase client to use. + * @param table The name of the HBase table to use. + * @param args Command line arguments ({@code assign name [names]}). + * @return The exit status of the command (0 means success). + */ + private static int delete(final TSDB tsdb, final byte[] table, + final String[] args) throws Exception { + final String kind = args[1]; + final String name = args[2]; + try { + tsdb.deleteUidAsync(kind, name).join(); + } catch (HBaseException e) { + LOG.error("error while processing delete " + name, e); + return 3; + } catch (NoSuchUniqueName e) { + LOG.error(e.getMessage()); + return 1; + } + LOG.info("UID " + kind + ' ' + name + " deleted."); + return 0; + } + /** * Implements the {@code fsck} subcommand. * @param client The HBase client to use. @@ -955,11 +1002,9 @@ private static int extactLookupName(final HBaseClient client, */ private static int metaSync(final TSDB tsdb) throws Exception { final long start_time = System.currentTimeMillis() / 1000; - final long max_id = CliUtils.getMaxMetricID(tsdb); - + // now figure out how many IDs to divy up between the workers final int workers = Runtime.getRuntime().availableProcessors() * 2; - final double quotient = (double)max_id / (double)workers; final Set processed_tsuids = Collections.synchronizedSet(new HashSet()); final ConcurrentHashMap metric_uids = @@ -968,29 +1013,24 @@ private static int metaSync(final TSDB tsdb) throws Exception { new ConcurrentHashMap(); final ConcurrentHashMap tagv_uids = new ConcurrentHashMap(); - - long index = 1; - - LOG.info("Max metric ID is [" + max_id + "]"); - LOG.info("Spooling up [" + workers + "] worker threads"); - final Thread[] threads = new Thread[workers]; - for (int i = 0; i < workers; i++) { - threads[i] = new MetaSync(tsdb, index, quotient, processed_tsuids, - metric_uids, tagk_uids, tagv_uids, i); - threads[i].setName("MetaSync # " + i); - threads[i].start(); - index += quotient; - if (index < max_id) { - index++; - } + + final List scanners = CliUtils.getDataTableScanners(tsdb, workers); + LOG.info("Spooling up [" + scanners.size() + "] worker threads"); + final List threads = new ArrayList(scanners.size()); + int i = 0; + for (final Scanner scanner : scanners) { + final MetaSync worker = new MetaSync(tsdb, scanner, processed_tsuids, + metric_uids, tagk_uids, tagv_uids, i++); + worker.setName("Sync #" + i); + worker.start(); + threads.add(worker); } - - // wait till we're all done - for (int i = 0; i < workers; i++) { - threads[i].join(); - LOG.info("[" + i + "] Finished"); + + for (final Thread thread : threads) { + thread.join(); + LOG.info("Thread [" + thread + "] Finished"); } - + LOG.info("All metasync threads have completed"); // make sure buffered data is flushed to storage before exiting tsdb.flush().joinUninterruptibly(); diff --git a/src/tree/Leaf.java b/src/tree/Leaf.java index 3614930468..9d1c0817be 100644 --- a/src/tree/Leaf.java +++ b/src/tree/Leaf.java @@ -33,6 +33,7 @@ import com.stumbleupon.async.Deferred; import net.opentsdb.core.TSDB; +import net.opentsdb.uid.NoSuchUniqueId; import net.opentsdb.uid.UniqueId; import net.opentsdb.uid.UniqueId.UniqueIdType; import net.opentsdb.utils.JSON; diff --git a/src/tree/Tree.java b/src/tree/Tree.java index 834555d571..017fdd10c8 100644 --- a/src/tree/Tree.java +++ b/src/tree/Tree.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; +import java.util.regex.PatternSyntaxException; import net.opentsdb.core.TSDB; import net.opentsdb.uid.UniqueId; diff --git a/src/tree/TreeRule.java b/src/tree/TreeRule.java index a864e42325..76d255ad8b 100644 --- a/src/tree/TreeRule.java +++ b/src/tree/TreeRule.java @@ -17,6 +17,7 @@ import java.util.HashMap; import java.util.Map; import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; import org.hbase.async.Bytes; import org.hbase.async.DeleteRequest; diff --git a/src/tsd/AbstractHttpQuery.java b/src/tsd/AbstractHttpQuery.java new file mode 100644 index 0000000000..388127e9d6 --- /dev/null +++ b/src/tsd/AbstractHttpQuery.java @@ -0,0 +1,532 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2010-2014 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.tsd; + +import java.nio.charset.Charset; +import java.nio.charset.UnsupportedCharsetException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import com.google.common.base.Objects; +import com.stumbleupon.async.Deferred; + +import org.jboss.netty.buffer.ChannelBuffer; +import org.jboss.netty.buffer.ChannelBuffers; +import org.jboss.netty.channel.Channel; +import org.jboss.netty.channel.ChannelFuture; +import org.jboss.netty.channel.ChannelFutureListener; +import org.jboss.netty.handler.codec.http.DefaultHttpResponse; +import org.jboss.netty.handler.codec.http.HttpHeaders; +import org.jboss.netty.handler.codec.http.HttpMethod; +import org.jboss.netty.handler.codec.http.HttpRequest; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.jboss.netty.handler.codec.http.HttpVersion; +import org.jboss.netty.handler.codec.http.QueryStringDecoder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import net.opentsdb.core.Const; +import net.opentsdb.core.TSDB; +import net.opentsdb.stats.QueryStats; + +/** + * Abstract base class for HTTP queries. + * + * @since 2.2 + */ +public abstract class AbstractHttpQuery { + private static final Logger LOG = LoggerFactory.getLogger(AbstractHttpQuery.class); + + /** When the query was started (useful for timing). */ + private final long start_time = System.nanoTime(); + + /** The request in this HTTP query. */ + private final HttpRequest request; + + /** The channel on which the request was received. */ + private final Channel chan; + + /** Shortcut to the request method */ + private final HttpMethod method; + + /** Parsed query string (lazily built on first access). */ + private Map> querystring; + + /** Deferred result of this query, to allow asynchronous processing. + * (Optional.) */ + protected final Deferred deferred = new Deferred(); + + /** The response object we'll fill with data */ + private final DefaultHttpResponse response = + new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK); + + /** The {@code TSDB} instance we belong to */ + protected final TSDB tsdb; + + /** Used for recording query statistics */ + protected QueryStats stats; + + /** + * Set up required internal state. For subclasses. + * + * @param request the incoming HTTP request + * @param chan the {@link Channel} the request was received on + */ + protected AbstractHttpQuery(final TSDB tsdb, final HttpRequest request, final Channel chan) { + this.tsdb = tsdb; + this.request = request; + this.chan = chan; + this.method = request.getMethod(); + } + + /** + * Returns the underlying Netty {@link HttpRequest} of this query. + */ + public HttpRequest request() { + return request; + } + + /** Returns the HTTP method/verb for the request */ + public HttpMethod method() { + return this.method; + } + + /** Returns the response object, allowing serializers to set headers */ + public DefaultHttpResponse response() { + return this.response; + } + + /** + * Returns the underlying Netty {@link Channel} of this query. + */ + public Channel channel() { + return chan; + } + + /** @return The remote address and port in the format <ip>:<port> */ + public String getRemoteAddress() { + return chan.getRemoteAddress().toString(); + } + + /** + * Copies the header list and obfuscates the "cookie" header in case it + * contains auth tokens, etc. Note that it flattens duplicate headers keys + * as comma separated lists per the RFC + * @return The full set of headers for this query with the cookie obfuscated + */ + public Map getPrintableHeaders() { + final Map headers = new HashMap( + request.headers().entries().size()); + for (final Entry header : request.headers().entries()) { + if (header.getKey().toLowerCase().equals("cookie")) { + // null out the cookies + headers.put(header.getKey(), "*******"); + } else { + // http://tools.ietf.org/html/rfc2616#section-4.2 + if (headers.containsKey(header.getKey())) { + headers.put(header.getKey(), + headers.get(header.getKey()) + "," + header.getValue()); + } else { + headers.put(header.getKey(), header.getValue()); + } + } + } + return headers; + } + + /** + * Copies the header list so modifications won't affect the original set. + * Note that it flattens duplicate headers keys as comma separated lists + * per the RFC + * @return The full set of headers for this query + */ + public Map getHeaders() { + final Map headers = new HashMap( + request.headers().entries().size()); + for (final Entry header : request.headers().entries()) { + // http://tools.ietf.org/html/rfc2616#section-4.2 + if (headers.containsKey(header.getKey())) { + headers.put(header.getKey(), + headers.get(header.getKey()) + "," + header.getValue()); + } else { + headers.put(header.getKey(), header.getValue()); + } + } + return headers; + } + + /** + * Return the value of the given HTTP Header + * first match wins + * @return Header value as string + */ + public String getHeaderValue(final String headerName) { + if (headerName == null) { return null; } + return request.headers().get(headerName); + } + + /** @param stats The stats object to mark after writing is complete */ + public void setStats(final QueryStats stats) { + this.stats = stats; + } + + /** Return the time in nanoseconds that this query object was + * created. + */ + public long startTimeNanos() { + return start_time; + } + + /** Returns how many ms have elapsed since this query was created. */ + public int processingTimeMillis() { + return (int) ((System.nanoTime() - start_time) / 1000000); + } + + /** + * Returns the query string parameters passed in the URI. + */ + public Map> getQueryString() { + if (querystring == null) { + try { + querystring = new QueryStringDecoder(request.getUri()).getParameters(); + } catch (IllegalArgumentException e) { + throw new BadRequestException("Bad query string: " + e.getMessage()); + } + } + return querystring; + } + + /** + * Returns the value of the given query string parameter. + *

    + * If this parameter occurs multiple times in the URL, only the last value + * is returned and others are silently ignored. + * @param paramname Name of the query string parameter to get. + * @return The value of the parameter or {@code null} if this parameter + * wasn't passed in the URI. + */ + public String getQueryStringParam(final String paramname) { + final List params = getQueryString().get(paramname); + return params == null ? null : params.get(params.size() - 1); + } + + /** + * Returns the non-empty value of the given required query string parameter. + *

    + * If this parameter occurs multiple times in the URL, only the last value + * is returned and others are silently ignored. + * @param paramname Name of the query string parameter to get. + * @return The value of the parameter. + * @throws BadRequestException if this query string parameter wasn't passed + * or if its last occurrence had an empty value ({@code &a=}). + */ + public String getRequiredQueryStringParam(final String paramname) + throws BadRequestException { + final String value = getQueryStringParam(paramname); + if (value == null || value.isEmpty()) { + throw BadRequestException.missingParameter(paramname); + } + return value; + } + + /** + * Returns whether or not the given query string parameter was passed. + * @param paramname Name of the query string parameter to get. + * @return {@code true} if the parameter + */ + public boolean hasQueryStringParam(final String paramname) { + return getQueryString().get(paramname) != null; + } + + /** + * Returns all the values of the given query string parameter. + *

    + * In case this parameter occurs multiple times in the URL, this method is + * useful to get all the values. + * @param paramname Name of the query string parameter to get. + * @return The values of the parameter or {@code null} if this parameter + * wasn't passed in the URI. + */ + public List getQueryStringParams(final String paramname) { + return getQueryString().get(paramname); + } + + /** + * Returns only the path component of the URI as a string + * This call strips the protocol, host, port and query string parameters + * leaving only the path e.g. "/path/starts/here" + *

    + * Note that for slightly quicker performance you can call request().getUri() + * to get the full path as a string but you'll have to strip query string + * parameters manually. + * @return The path component of the URI + * @throws NullPointerException if the URI is null + */ + public String getQueryPath() { + return new QueryStringDecoder(request.getUri()).getPath(); + } + + /** + * Returns the path component of the URI as an array of strings, split on the + * forward slash + * Similar to the {@link #getQueryPath} call, this returns only the path + * without the protocol, host, port or query string params. E.g. + * "/path/starts/here" will return an array of {"path", "starts", "here"} + *

    + * Note that for maximum speed you may want to parse the query path manually. + * @return An array with 1 or more components, note the first item may be + * an empty string. + * @throws BadRequestException if the URI is empty or does not start with a + * slash + * @throws NullPointerException if the URI is null + */ + public String[] explodePath() { + final String path = getQueryPath(); + if (path.isEmpty()) { + throw new BadRequestException("Query path is empty"); + } + if (path.charAt(0) != '/') { + throw new BadRequestException("Query path doesn't start with a slash"); + } + // split may be a tad slower than other methods, but since the URIs are + // usually pretty short and not every request will make this call, we + // probably don't need any premature optimization + return path.substring(1).split("/"); + } + + /** + * Parses the query string to determine the base route for handing a query + * off to an RPC handler. + * @return the base route + * @throws BadRequestException if some necessary part of the query cannot + * be parsed. + */ + public abstract String getQueryBaseRoute(); + + /** + * Attempts to parse the character set from the request header. If not set + * defaults to UTF-8 + * @return A Charset object + * @throws UnsupportedCharsetException if the parsed character set is invalid + */ + public Charset getCharset() { + // RFC2616 3.7 + for (String type : this.request.headers().getAll("Content-Type")) { + int idx = type.toUpperCase().indexOf("CHARSET="); + if (idx > 1) { + String charset = type.substring(idx+8); + return Charset.forName(charset); + } + } + return Charset.forName("UTF-8"); + } + + /** @return True if the request has content, false if not. */ + public boolean hasContent() { + return this.request.getContent() != null && + this.request.getContent().readable(); + } + + /** + * Decodes the request content to a string using the appropriate character set + * @return Decoded content or an empty string if the request did not include + * content + * @throws UnsupportedCharsetException if the parsed character set is invalid + */ + public String getContent() { + return this.request.getContent().toString(this.getCharset()); + } + + /** + * Method to call after writing the HTTP response to the wire. The default + * is to simply log the request info. Can be overridden by subclasses. + */ + public void done() { + final int processing_time = processingTimeMillis(); + final String url = request.getUri(); + final String msg = String.format("HTTP %s done in %d ms", url, processing_time); + if (url.startsWith("/api/put") && LOG.isDebugEnabled()) { + // NOTE: Suppresses too many log lines from /api/put. + LOG.debug(msg); + } else { + logInfo(msg); + } + logInfo("HTTP " + request.getUri() + " done in " + processing_time + "ms"); + } + + /** + * Sends 500/Internal Server Error to the client. + * @param cause The unexpected exception that caused this error. + */ + public void internalError(final Exception cause) { + logError("Internal Server Error on " + request().getUri(), cause); + sendBuffer(HttpResponseStatus.INTERNAL_SERVER_ERROR, + ChannelBuffers.wrappedBuffer( + cause.toString().getBytes(Const.UTF8_CHARSET)), + "text/plain"); + } + + /** + * Sends 400/Bad Request status to the client. + * @param exception The exception that was thrown + */ + public void badRequest(final BadRequestException exception) { + logWarn("Bad Request on " + request().getUri() + ": " + exception.getMessage()); + sendBuffer(HttpResponseStatus.BAD_REQUEST, + ChannelBuffers.wrappedBuffer( + exception.toString().getBytes(Const.UTF8_CHARSET)), + "text/plain"); + } + + /** + * Sends 404/Not Found to the client. + */ + public void notFound() { + logWarn("Not Found: " + request().getUri()); + sendStatusOnly(HttpResponseStatus.NOT_FOUND); + } + + /** + * Send just the status code without a body, used for 204 or 304 + * @param status The response code to reply with + */ + public void sendStatusOnly(final HttpResponseStatus status) { + if (!chan.isConnected()) { + if(stats != null) { + stats.markSendFailed(); + } + done(); + return; + } + + response.setStatus(status); + final boolean keepalive = HttpHeaders.isKeepAlive(request); + if (keepalive) { + HttpHeaders.setContentLength(response, 0); + } + final ChannelFuture future = chan.write(response); + if (stats != null) { + future.addListener(new SendSuccess()); + } + if (!keepalive) { + future.addListener(ChannelFutureListener.CLOSE); + } + done(); + } + + /** + * Sends an HTTP reply to the client. + * @param status The status of the request (e.g. 200 OK or 404 Not Found). + * @param buf The content of the reply to send. + */ + public void sendBuffer(final HttpResponseStatus status, + final ChannelBuffer buf, + final String contentType) { + if (!chan.isConnected()) { + if(stats != null) { + stats.markSendFailed(); + } + done(); + return; + } + response.headers().set(HttpHeaders.Names.CONTENT_TYPE, contentType); + + // TODO(tsuna): Server, X-Backend, etc. headers. + // only reset the status if we have the default status, otherwise the user + // already set it + response.setStatus(status); + response.setContent(buf); + final boolean keepalive = HttpHeaders.isKeepAlive(request); + if (keepalive) { + HttpHeaders.setContentLength(response, buf.readableBytes()); + } + final ChannelFuture future = chan.write(response); + if (stats != null) { + future.addListener(new SendSuccess()); + } + if (!keepalive) { + future.addListener(ChannelFutureListener.CLOSE); + } + done(); + } + + /** A simple class that marks a query as complete when the stats are set */ + private class SendSuccess implements ChannelFutureListener { + @Override + public void operationComplete(final ChannelFuture future) throws Exception { + if(future.isSuccess()) { + stats.markSent();} + else + stats.markSendFailed(); + } + } + + /** @return Information about the query */ + public String toString() { + return Objects.toStringHelper(this) + .add("start_time", start_time) + .add("request", request) + .add("chan", chan) + .add("querystring", querystring) + .toString(); + } + + // ---------------- // + // Logging helpers. // + // ---------------- // + + /** + * Logger for the query instance. + */ + protected Logger logger() { + return LOG; + } + + protected final String logChannel() { + if (request.headers().contains("X-Forwarded-For")) { + String inetAddress; + String proxyChain = request.headers().get("X-Forwarded-For"); + int firstComma = proxyChain.indexOf(','); + if (firstComma != -1) { + inetAddress = proxyChain.substring(0, proxyChain.indexOf(',')); + } else { + inetAddress = proxyChain; + } + return "[id: 0x" + Integer.toHexString(chan.hashCode()) + + ", /" + inetAddress + " => " + chan.getLocalAddress() + ']'; + } else { + return chan.toString(); + } + } + + protected final void logInfo(final String msg) { + if (logger().isInfoEnabled()) { + logger().info(logChannel() + ' ' + msg); + } + } + + protected final void logWarn(final String msg) { + if (logger().isWarnEnabled()) { + logger().warn(logChannel() + ' ' + msg); + } + } + + protected final void logError(final String msg, final Exception e) { + if (logger().isErrorEnabled()) { + logger().error(logChannel() + ' ' + msg, e); + } + } + +} diff --git a/src/tsd/AnnotationRpc.java b/src/tsd/AnnotationRpc.java index d4c72050d5..c6dd803092 100644 --- a/src/tsd/AnnotationRpc.java +++ b/src/tsd/AnnotationRpc.java @@ -47,7 +47,9 @@ final class AnnotationRpc implements HttpRpc { */ public void execute(final TSDB tsdb, HttpQuery query) throws IOException { final HttpMethod method = query.getAPIMethod(); - + + RpcUtil.allowedMethods(method, HttpMethod.GET.getName(), HttpMethod.POST.getName(), HttpMethod.DELETE.getName(), HttpMethod.PUT.getName()); + final String[] uri = query.explodeAPIPath(); final String endpoint = uri.length > 1 ? uri[1] : ""; if (endpoint != null && endpoint.toLowerCase().endsWith("bulk")) { @@ -65,14 +67,11 @@ public void execute(final TSDB tsdb, HttpQuery query) throws IOException { // GET if (method == HttpMethod.GET) { try { - final Annotation stored_annotation = - Annotation.getAnnotation(tsdb, note.getTSUID(), note.getStartTime()) - .joinUninterruptibly(); - if (stored_annotation == null) { - throw new BadRequestException(HttpResponseStatus.NOT_FOUND, - "Unable to locate annotation in storage"); + if ("annotations".toLowerCase().equals(uri[0])) { + fetchMultipleAnnotations(tsdb, note, query); + } else { + fetchSingleAnnotation(tsdb, note, query); } - query.sendReply(query.serializer().formatAnnotationV1(stored_annotation)); } catch (BadRequestException e) { throw e; } catch (Exception e) { @@ -128,11 +127,6 @@ public Deferred call(Boolean success) throws Exception { throw new RuntimeException(e); } query.sendStatusOnly(HttpResponseStatus.NO_CONTENT); - - } else { - throw new BadRequestException(HttpResponseStatus.METHOD_NOT_ALLOWED, - "Method not allowed", "The HTTP method [" + method.getName() + - "] is not permitted for this endpoint"); } } @@ -144,14 +138,12 @@ public Deferred call(Boolean success) throws Exception { * @param query The query to parse and respond to */ void executeBulk(final TSDB tsdb, final HttpMethod method, HttpQuery query) { + RpcUtil.allowedMethods(query.method(), HttpMethod.PUT.getName(), HttpMethod.POST.getName(), HttpMethod.DELETE.getName()); + if (method == HttpMethod.POST || method == HttpMethod.PUT) { executeBulkUpdate(tsdb, method, query); } else if (method == HttpMethod.DELETE) { executeBulkDelete(tsdb, query); - } else { - throw new BadRequestException(HttpResponseStatus.METHOD_NOT_ALLOWED, - "Method not allowed", "The HTTP method [" + query.method().getName() + - "] is not permitted for this endpoint"); } } @@ -345,6 +337,33 @@ private Annotation parseQS(final HttpQuery query) { return note; } + private void fetchSingleAnnotation(final TSDB tsdb, final Annotation note, + final HttpQuery query) throws Exception { + final Annotation stored_annotation = + Annotation.getAnnotation(tsdb, note.getTSUID(), note.getStartTime()) + .joinUninterruptibly(); + if (stored_annotation == null) { + throw new BadRequestException(HttpResponseStatus.NOT_FOUND, + "Unable to locate annotation in storage"); + } + query.sendReply(query.serializer().formatAnnotationV1(stored_annotation)); + } + + private void fetchMultipleAnnotations(final TSDB tsdb, final Annotation note, + final HttpQuery query) throws Exception { + if (note.getEndTime() == 0) { + note.setEndTime(System.currentTimeMillis()); + } + final List annotations = + Annotation.getGlobalAnnotations(tsdb, note.getStartTime(), note.getEndTime()) + .joinUninterruptibly(); + if (annotations == null) { + throw new BadRequestException(HttpResponseStatus.NOT_FOUND, + "Unable to locate annotations in storage"); + } + query.sendReply(query.serializer().formatAnnotationsV1(annotations)); + } + /** * Parses a query string for a bulk delet request * @param query The query to parse diff --git a/src/tsd/BadRequestException.java b/src/tsd/BadRequestException.java index b221a3c9da..24975e7338 100644 --- a/src/tsd/BadRequestException.java +++ b/src/tsd/BadRequestException.java @@ -22,7 +22,7 @@ * optional detailed response. The default "message" field is still used for * short error descriptions, typically one sentence long. */ -final class BadRequestException extends RuntimeException { +public final class BadRequestException extends RuntimeException { /** The HTTP status code to return to the user * @since 2.0 */ diff --git a/src/tsd/ConnectionManager.java b/src/tsd/ConnectionManager.java index 35c3288bad..902c31315b 100644 --- a/src/tsd/ConnectionManager.java +++ b/src/tsd/ConnectionManager.java @@ -14,12 +14,14 @@ import java.io.IOException; import java.nio.channels.ClosedChannelException; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.jboss.netty.channel.Channel; import org.jboss.netty.channel.ChannelEvent; +import org.jboss.netty.channel.ChannelException; import org.jboss.netty.channel.ChannelHandlerContext; import org.jboss.netty.channel.ChannelStateEvent; import org.jboss.netty.channel.ExceptionEvent; @@ -37,10 +39,20 @@ final class ConnectionManager extends SimpleChannelHandler { private static final Logger LOG = LoggerFactory.getLogger(ConnectionManager.class); private static final AtomicLong connections_established = new AtomicLong(); + private static final AtomicLong connections_rejected = new AtomicLong(); private static final AtomicLong exceptions_unknown = new AtomicLong(); private static final AtomicLong exceptions_closed = new AtomicLong(); private static final AtomicLong exceptions_reset = new AtomicLong(); private static final AtomicLong exceptions_timeout = new AtomicLong(); + + /** Max connections can be serviced by tsd, if over limit, tsd will refuse + * new connections. */ + private final int connections_limit; + + /** A counter used for determining how many channels are open. Something odd + * happens with the DefaultChannelGroup in that .size() doesn't return the + * actual number of open connections. TODO - find out why. */ + private final AtomicInteger open_connections = new AtomicInteger(); private static final DefaultChannelGroup channels = new DefaultChannelGroup("all-channels"); @@ -49,8 +61,21 @@ static void closeAllConnections() { channels.close().awaitUninterruptibly(); } - /** Constructor. */ + /** + * Default Ctor with no concurrent connection limit. + */ public ConnectionManager() { + connections_limit = 0; + } + + /** + * CTor for setting a limit on concurrent connections. + * @param connections_limit The maximum number of concurrent connections allowed. + * @since 2.3 + */ + public ConnectionManager(final int connections_limit) { + LOG.info("TSD concurrent connection limit set to: " + connections_limit); + this.connections_limit = connections_limit; } /** @@ -59,6 +84,8 @@ public ConnectionManager() { */ public static void collectStats(final StatsCollector collector) { collector.record("connectionmgr.connections", channels.size(), "type=open"); + collector.record("connectionmgr.connections", connections_rejected, + "type=rejected"); collector.record("connectionmgr.connections", connections_established, "type=total"); collector.record("connectionmgr.exceptions", exceptions_closed, @@ -73,11 +100,25 @@ public static void collectStats(final StatsCollector collector) { @Override public void channelOpen(final ChannelHandlerContext ctx, - final ChannelStateEvent e) { + final ChannelStateEvent e) throws IOException { + if (connections_limit > 0) { + final int channel_size = open_connections.incrementAndGet(); + if (channel_size > connections_limit) { + throw new ConnectionRefusedException("Channel size (" + channel_size + ") exceeds total " + + "connection limit (" + connections_limit + ")"); + // exceptionCaught will close the connection and increment the counter. + } + } channels.add(e.getChannel()); connections_established.incrementAndGet(); } + @Override + public void channelClosed(final ChannelHandlerContext ctx, + final ChannelStateEvent e) throws IOException { + open_connections.decrementAndGet(); + } + @Override public void handleUpstream(final ChannelHandlerContext ctx, final ChannelEvent e) throws Exception { @@ -109,6 +150,13 @@ public void exceptionCaught(final ChannelHandlerContext ctx, // in Java. Like, people have been bitching about errno for years, // and Java managed to do something *far* worse. That's quite a feat. return; + } else if (cause instanceof ConnectionRefusedException) { + connections_rejected.incrementAndGet(); + if (LOG.isDebugEnabled()) { + LOG.debug("Refusing connection from " + chan, e.getCause()); + } + chan.close(); + return; } } if (cause instanceof CodecEmbedderException) { @@ -122,4 +170,17 @@ public void exceptionCaught(final ChannelHandlerContext ctx, e.getChannel().close(); } + /** Simple exception for refusing a connection. */ + private static class ConnectionRefusedException extends ChannelException { + + /** + * Default ctor with a message. + * @param message A descriptive message for the exception. + */ + public ConnectionRefusedException(final String message) { + super(message); + } + + private static final long serialVersionUID = 5348377149312597939L; + } } diff --git a/src/tsd/DropCachesRpc.java b/src/tsd/DropCachesRpc.java new file mode 100644 index 0000000000..9cd59d15e9 --- /dev/null +++ b/src/tsd/DropCachesRpc.java @@ -0,0 +1,88 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2013 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.tsd; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Pattern; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.Atomics; +import com.stumbleupon.async.Callback; +import com.stumbleupon.async.Deferred; + +import org.jboss.netty.channel.Channel; +import org.jboss.netty.handler.codec.http.HttpMethod; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import net.opentsdb.tools.BuildData; +import net.opentsdb.core.Aggregators; +import net.opentsdb.core.TSDB; +import net.opentsdb.query.filter.TagVFilter; +import net.opentsdb.stats.StatsCollector; +import net.opentsdb.utils.Config; +import net.opentsdb.utils.JSON; +import net.opentsdb.utils.PluginLoader; + +import java.io.IOException; + +/** The "dropcaches" command. */ +public final class DropCachesRpc implements TelnetRpc, HttpRpc { + private static final Logger LOG = LoggerFactory.getLogger(DropCachesRpc.class); + + public Deferred execute(final TSDB tsdb, final Channel chan, + final String[] cmd) { + dropCaches(tsdb, chan); + chan.write("Caches dropped.\n"); + return Deferred.fromResult(null); + } + + public void execute(final TSDB tsdb, final HttpQuery query) + throws IOException { + + // only accept GET/DELETE + RpcUtil.allowedMethods(query.method(), HttpMethod.GET.getName(), HttpMethod.DELETE.getName()); + + dropCaches(tsdb, query.channel()); + + if (query.apiVersion() > 0) { + final HashMap response = new HashMap(); + response.put("status", "200"); + response.put("message", "Caches dropped"); + query.sendReply(query.serializer().formatDropCachesV1(response)); + } else { // deprecated API + query.sendReply("Caches dropped.\n"); + } + } + + /** Drops in memory caches. */ + private void dropCaches(final TSDB tsdb, final Channel chan) { + LOG.warn(chan + " Dropping all in-memory caches."); + tsdb.dropCaches(); + } +} \ No newline at end of file diff --git a/src/tsd/GraphHandler.java b/src/tsd/GraphHandler.java index 0eb6a6ee83..0ae7fd8d97 100644 --- a/src/tsd/GraphHandler.java +++ b/src/tsd/GraphHandler.java @@ -1,5 +1,5 @@ // This file is part of OpenTSDB. -// Copyright (C) 2010-2012 The OpenTSDB Authors. +// Copyright (C) 2010-2021 The OpenTSDB Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU Lesser General Public License as published by @@ -19,40 +19,51 @@ import java.io.IOException; import java.io.PrintWriter; import java.net.URL; +import java.net.URLDecoder; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.NoSuchElementException; +import java.util.Set; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicInteger; +import java.util.regex.Pattern; + import static java.util.concurrent.TimeUnit.MILLISECONDS; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; +import com.google.common.base.Strings; +import com.google.common.collect.Sets; + +import org.jboss.netty.handler.codec.http.HttpMethod; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import net.opentsdb.core.Aggregator; -import net.opentsdb.core.Aggregators; import net.opentsdb.core.Const; import net.opentsdb.core.DataPoint; import net.opentsdb.core.DataPoints; import net.opentsdb.core.Query; -import net.opentsdb.core.RateOptions; import net.opentsdb.core.TSDB; +import net.opentsdb.core.TSQuery; import net.opentsdb.core.Tags; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import net.opentsdb.graph.Plot; +import net.opentsdb.meta.Annotation; import net.opentsdb.stats.Histogram; import net.opentsdb.stats.StatsCollector; -import net.opentsdb.uid.NoSuchUniqueName; +import net.opentsdb.tools.GnuplotInstaller; import net.opentsdb.utils.DateTime; import net.opentsdb.utils.JSON; +import com.stumbleupon.async.Callback; + /** * Stateless handler of HTTP graph requests (the {@code /q} endpoint). */ @@ -64,6 +75,18 @@ final class GraphHandler implements HttpRpc { private static final boolean IS_WINDOWS = System.getProperty("os.name", "").contains("Windows"); + private static final String RANGE_COMPONENT = "\\\"?-?\\d*\\.?(\\d+)?([eE]-?\\d+)?\\\"?"; + private static Pattern RANGE_VALIDATOR = Pattern.compile( + "^\\["+RANGE_COMPONENT+":"+RANGE_COMPONENT+"]$"); + private static Pattern LABEL_VALIDATOR = Pattern.compile("^[a-zA-z0-9 \\-_]+$"); + private static Pattern KEY_VALIDATOR = Pattern.compile( + "^out|left|top|center|right|horiz|box|bottom$"); + private static Pattern STYLE_VALIDATOR = Pattern.compile("^linespoint|points|circles|dots$"); + private static Pattern COLOR_VALIDATOR = Pattern.compile("^(x|X)[a-fA-F0-9]{6}$"); + private static Pattern SMOOTH_VALIDATOR = Pattern.compile("^unique|frequency|fnormal|cumulative|cnormal|bins|csplines|acsplines|mcsplines|bezier|sbezier|unwrap|zsort$"); + // NOTE: This one should be tightened for only time based formatters. + private static Pattern FORMAT_VALIDATOR = Pattern.compile("^[%0-9.a-zA-Z \\-]+$"); + private static Pattern WXH_VALIDATOR = Pattern.compile("^\\d+x\\d+$"); /** Number of times we had to do all the work up to running Gnuplot. */ private static final AtomicInteger graphs_generated = new AtomicInteger(); @@ -105,6 +128,10 @@ public GraphHandler() { } public void execute(final TSDB tsdb, final HttpQuery query) { + + // only accept GET/POST + RpcUtil.allowedMethods(query.method(), HttpMethod.GET.getName(), HttpMethod.POST.getName()); + if (!query.hasQueryStringParam("json") && !query.hasQueryStringParam("png") && !query.hasQueryStringParam("ascii")) { @@ -126,6 +153,10 @@ public void execute(final TSDB tsdb, final HttpQuery query) { } } + // TODO(HugoMFernandes): Most of this (query-related) logic is implemented in + // net.opentsdb.tsd.QueryRpc.java (which actually does this asynchronously), + // so we should refactor both classes to split the actual logic used to + // generate the data from the actual visualization (removing all duped code). private void doGraph(final TSDB tsdb, final HttpQuery query) throws IOException { final String basepath = getGnuplotBasePath(tsdb, query); @@ -157,10 +188,39 @@ private void doGraph(final TSDB tsdb, final HttpQuery query) if (!nocache && isDiskCacheHit(query, end_time, max_age, basepath)) { return; } - Query[] tsdbqueries; - List options; - tsdbqueries = parseQuery(tsdb, query); - options = query.getQueryStringParams("o"); + + // Parse TSQuery from HTTP query + final TSQuery tsquery = QueryRpc.parseQuery(tsdb, query); + tsquery.validateAndSetQuery(); + + // Build the queries for the parsed TSQuery + Query[] tsdbqueries = tsquery.buildQueries(tsdb); + + List options = null; + final String options_allow_list = tsdb.getConfig().getString( + "tsd.gnuplot.options.allowlist"); + if (!Strings.isNullOrEmpty(options_allow_list)) { + String[] allow_list_strings = options_allow_list.split(";"); + Set allow_list = Sets.newHashSet(); + for (int i = 0; i < allow_list_strings.length; i++) { + String allow = allow_list_strings[i]; + if (allow != null) { + allow = URLDecoder.decode(allow.trim()); + allow_list.add(allow); + } + } + + options = query.getQueryStringParams("o"); + if (!(options == null)) { + for (int i = 0; i < options.size(); i++) { + if (!allow_list.contains(options.get(i))) { + throw new BadRequestException("Query option at index " + i + + " was not in the allow list."); + } + } + } + } + if (options == null) { options = new ArrayList(tsdbqueries.length); for (int i = 0; i < tsdbqueries.length; i++) { @@ -215,9 +275,37 @@ private void doGraph(final TSDB tsdb, final HttpQuery query) return; } + final RunGnuplot rungnuplot = new RunGnuplot(query, max_age, plot, basepath, + aggregated_tags, npoints); + + class ErrorCB implements Callback { + public Object call(final Exception e) throws Exception { + LOG.warn("Failed to retrieve global annotations: ", e); + throw e; + } + } + + class GlobalCB implements Callback> { + public Object call(final List global_annotations) throws Exception { + rungnuplot.plot.setGlobals(global_annotations); + execGnuplot(rungnuplot, query); + + return null; + } + } + + // Fetch global annotations, if needed + if (!tsquery.getNoAnnotations() && tsquery.getGlobalAnnotations()) { + Annotation.getGlobalAnnotations(tsdb, start_time, end_time) + .addCallback(new GlobalCB()).addErrback(new ErrorCB()); + } else { + execGnuplot(rungnuplot, query); + } + } + + private void execGnuplot(RunGnuplot rungnuplot, HttpQuery query) { try { - gnuplot.execute(new RunGnuplot(query, max_age, plot, basepath, - aggregated_tags, npoints)); + gnuplot.execute(rungnuplot); } catch (RejectedExecutionException e) { query.internalError(new Exception("Too many requests pending," + " please try again later", e)); @@ -357,7 +445,7 @@ private String getGnuplotBasePath(final TSDB tsdb, final HttpQuery query) { qs.remove("png"); qs.remove("json"); qs.remove("ascii"); - return tsdb.getConfig().getDirectoryName("tsd.http.cachedir") + + return tsdb.getConfig().getDirectoryName("tsd.http.cachedir") + Integer.toHexString(qs.hashCode()); } @@ -587,8 +675,14 @@ private HashMap loadCachedJson(final HttpQuery query, /** Parses the {@code wxh} query parameter to set the graph dimension. */ static void setPlotDimensions(final HttpQuery query, final Plot plot) { - final String wxh = query.getQueryStringParam("wxh"); + String wxh = query.getQueryStringParam("wxh"); if (wxh != null && !wxh.isEmpty()) { + wxh = URLDecoder.decode(wxh.trim()); + validateString("wxh", wxh); + if (!WXH_VALIDATOR.matcher(wxh).find()) { + throw new IllegalArgumentException("'wxh' was invalid. " + + "Must satisfy the pattern " + WXH_VALIDATOR.toString()); + } final int wxhlength = wxh.length(); if (wxhlength < 7) { // 100x100 minimum. throw new BadRequestException("Parameter wxh too short: " + wxh); @@ -633,13 +727,23 @@ private static String stringify(final String s) { * @return {@code null} if the parameter wasn't passed, otherwise the * value of the last occurrence of the parameter. */ - private static String popParam(final Map> querystring, - final String param) { + public static String popParam(final Map> querystring, + final String param) { final List params = querystring.remove(param); if (params == null) { return null; } - return params.get(params.size() - 1); + String given = params.get(params.size() - 1); + if (given != null) { + given = URLDecoder.decode(given.trim()); + } + // TODO - far from perfect, should help a little. + if (given.contains("`") || given.contains("%60") || + given.contains("`")) { + throw new BadRequestException("Parameter " + param + " contained a " + + "back-tick. That's a no-no."); + } + return given; } /** @@ -652,24 +756,59 @@ static void setPlotParams(final HttpQuery query, final Plot plot) { final Map> querystring = query.getQueryString(); String value; if ((value = popParam(querystring, "yrange")) != null) { + validateString("yrange", value, "[:]"); + if (!RANGE_VALIDATOR.matcher(value).find()) { + throw new BadRequestException("'yrange' was invalid. " + + "Must be in the format [min:max]."); + } params.put("yrange", value); } if ((value = popParam(querystring, "y2range")) != null) { + validateString("y2range", value, "[:]"); + if (!RANGE_VALIDATOR.matcher(value).find()) { + throw new BadRequestException("'y2range' was invalid. " + + "Must be in the format [min:max]."); + } params.put("y2range", value); } if ((value = popParam(querystring, "ylabel")) != null) { + validateString("ylabel", value, " "); + if (!LABEL_VALIDATOR.matcher(value).find()) { + throw new BadRequestException("'ylabel' was invalid. Must " + + "satisfy the pattern " + LABEL_VALIDATOR.toString()); + } params.put("ylabel", stringify(value)); } if ((value = popParam(querystring, "y2label")) != null) { + validateString("y2label", value, " "); + if (!LABEL_VALIDATOR.matcher(value).find()) { + throw new BadRequestException("'y2label' was invalid. Must " + + "satisfy the pattern " + LABEL_VALIDATOR.toString()); + } params.put("y2label", stringify(value)); } if ((value = popParam(querystring, "yformat")) != null) { + validateString("yformat", value, "% "); + if (!FORMAT_VALIDATOR.matcher(value).find()) { + throw new BadRequestException("'yformat' was invalid. Must " + + "satisfy the pattern " + FORMAT_VALIDATOR.toString()); + } params.put("format y", stringify(value)); } if ((value = popParam(querystring, "y2format")) != null) { + validateString("y2format", value, "% "); + if (!FORMAT_VALIDATOR.matcher(value).find()) { + throw new BadRequestException("'y2format' was invalid. Must " + + "satisfy the pattern " + FORMAT_VALIDATOR.toString()); + } params.put("format y2", stringify(value)); } if ((value = popParam(querystring, "xformat")) != null) { + validateString("xformat", value, "% "); + if (!FORMAT_VALIDATOR.matcher(value).find()) { + throw new BadRequestException("'xformat' was invalid. Must " + + "satisfy the pattern " + FORMAT_VALIDATOR.toString()); + } params.put("format x", stringify(value)); } if ((value = popParam(querystring, "ylog")) != null) { @@ -679,20 +818,53 @@ static void setPlotParams(final HttpQuery query, final Plot plot) { params.put("logscale y2", ""); } if ((value = popParam(querystring, "key")) != null) { + validateString("key", value); + if (!KEY_VALIDATOR.matcher(value).find()) { + throw new BadRequestException("'key' was invalid. Must " + + "satisfy the pattern " + KEY_VALIDATOR.toString()); + } params.put("key", value); } if ((value = popParam(querystring, "title")) != null) { + validateString("title", value, " "); + if (!LABEL_VALIDATOR.matcher(value).find()) { + throw new BadRequestException("'title' was invalid. Must " + + "satisfy the pattern " + LABEL_VALIDATOR.toString()); + } params.put("title", stringify(value)); } if ((value = popParam(querystring, "bgcolor")) != null) { + validateString("bgcolor", value); + if (!COLOR_VALIDATOR.matcher(value).find()) { + throw new BadRequestException("'bgcolor' was invalid. Must " + + "be a hex value e.g. 'xFFFFFF'"); + } params.put("bgcolor", value); } if ((value = popParam(querystring, "fgcolor")) != null) { + validateString("fgcolor", value); + if (!COLOR_VALIDATOR.matcher(value).find()) { + throw new BadRequestException("'fgcolor' was invalid. Must " + + "be a hex value e.g. 'xFFFFFF'"); + } params.put("fgcolor", value); } if ((value = popParam(querystring, "smooth")) != null) { + validateString("smooth", value); + if (!SMOOTH_VALIDATOR.matcher(value).find()) { + throw new BadRequestException("'smooth' was invalid. Must " + + "satisfy the pattern " + SMOOTH_VALIDATOR.toString()); + } params.put("smooth", value); } + if ((value = popParam(querystring, "style")) != null) { + validateString("style", value); + if (!STYLE_VALIDATOR.matcher(value).find()) { + throw new BadRequestException("'style' was invalid. Must " + + "satisfy the pattern " + STYLE_VALIDATOR.toString()); + } + params.put("style", value); + } // This must remain after the previous `if' in order to properly override // any previous `key' parameter if a `nokey' parameter is given. if ((value = popParam(querystring, "nokey")) != null) { @@ -793,20 +965,27 @@ private static void respondAsciiQuery(final HttpQuery query, .append('=').append(tag.getValue()); } for (final DataPoint d : dp) { - asciifile.print(metric); - asciifile.print(' '); - asciifile.print((d.timestamp() / 1000)); - asciifile.print(' '); if (d.isInteger()) { + printMetricHeader(asciifile, metric, d.timestamp()); asciifile.print(d.longValue()); } else { + // Doubles require extra processing. final double value = d.doubleValue(); - if (value != value || Double.isInfinite(value)) { - throw new IllegalStateException("NaN or Infinity:" + value + + // Value might be NaN or infinity. + if (Double.isInfinite(value)) { + // Infinity is invalid. + throw new IllegalStateException("Infinity:" + value + " d=" + d + ", query=" + query); + } else if (Double.isNaN(value)) { + // NaNs should be skipped. + continue; } + + printMetricHeader(asciifile, metric, d.timestamp()); asciifile.print(value); } + asciifile.print(tagbuf); asciifile.print('\n'); } @@ -822,81 +1001,17 @@ private static void respondAsciiQuery(final HttpQuery query, } /** - * Parses the {@code /q} query in a list of {@link Query} objects. - * @param tsdb The TSDB to use. - * @param query The HTTP query for {@code /q}. - * @return The corresponding {@link Query} objects. - * @throws BadRequestException if the query was malformed. - * @throws IllegalArgumentException if the metric or tags were malformed. - */ - private static Query[] parseQuery(final TSDB tsdb, final HttpQuery query) { - final List ms = query.getQueryStringParams("m"); - if (ms == null) { - throw BadRequestException.missingParameter("m"); - } - final Query[] tsdbqueries = new Query[ms.size()]; - int nqueries = 0; - for (final String m : ms) { - // m is of the following forms: - // agg:[interval-agg:][rate[{counter[,[countermax][,resetvalue]]}]:] - // metric[{tag=value,...}] - // Where the parts in square brackets `[' .. `]' are optional. - final String[] parts = Tags.splitString(m, ':'); - int i = parts.length; - if (i < 2 || i > 4) { - throw new BadRequestException("Invalid parameter m=" + m + " (" - + (i < 2 ? "not enough" : "too many") + " :-separated parts)"); - } - final Aggregator agg = getAggregator(parts[0]); - i--; // Move to the last part (the metric name). - final HashMap parsedtags = new HashMap(); - final String metric = Tags.parseWithMetric(parts[i], parsedtags); - final boolean rate = parts[--i].startsWith("rate"); - final RateOptions rate_options = QueryRpc.parseRateOptions(rate, parts[i]); - if (rate) { - i--; // Move to the next part. - } - final Query tsdbquery = tsdb.newQuery(); - try { - tsdbquery.setTimeSeries(metric, parsedtags, agg, rate, rate_options); - } catch (NoSuchUniqueName e) { - throw new BadRequestException(e.getMessage()); - } - // downsampling function & interval. - if (i > 0) { - final int dash = parts[1].indexOf('-', 1); // 1st char can't be `-'. - if (dash < 0) { - throw new BadRequestException("Invalid downsampling specifier '" - + parts[1] + "' in m=" + m); - } - Aggregator downsampler; - try { - downsampler = Aggregators.get(parts[1].substring(dash + 1)); - } catch (NoSuchElementException e) { - throw new BadRequestException("No such downsampling function: " - + parts[1].substring(dash + 1)); - } - final long interval = DateTime.parseDuration(parts[1].substring(0, dash)); - tsdbquery.downsample(interval, downsampler); - } else { - tsdbquery.downsample(1000, agg); - } - tsdbqueries[nqueries++] = tsdbquery; - } - return tsdbqueries; - } - - /** - * Returns the aggregator with the given name. - * @param name Name of the aggregator to get. - * @throws BadRequestException if there's no aggregator with this name. + * Helper method to write metric name and timestamp. + * @param writer The writer to which to write. + * @param metric The metric name. + * @param timestamp The timestamp. */ - private static final Aggregator getAggregator(final String name) { - try { - return Aggregators.get(name); - } catch (NoSuchElementException e) { - throw new BadRequestException("No such aggregation function: " + name); - } + private static void printMetricHeader(final PrintWriter writer, final String metric, + final long timestamp) { + writer.print(metric); + writer.print(' '); + writer.print(timestamp / 1000L); + writer.print(' '); } private static final PlotThdFactory thread_factory = new PlotThdFactory(); @@ -924,6 +1039,18 @@ public Thread newThread(final Runnable r) { * @return The path to the wrapper script. */ private static String findGnuplotHelperScript() { + if(!GnuplotInstaller.FOUND_GP) { + LOG.warn("Skipping Gnuplot Shell Script Install since Gnuplot executable was not found"); + return null; + } + if(!GnuplotInstaller.GP_FILE.exists()) { + GnuplotInstaller.installMyGnuPlot(); + } + if(GnuplotInstaller.GP_FILE.exists() && GnuplotInstaller.GP_FILE.canExecute()) { + LOG.info("Auto Installed Gnuplot Invoker at [{}]", GnuplotInstaller.GP_FILE.getAbsolutePath()); + return GnuplotInstaller.GP_FILE.getAbsolutePath(); + } + final URL url = GraphHandler.class.getClassLoader().getResource(WRAPPER); if (url == null) { throw new RuntimeException("Couldn't find " + WRAPPER + " on the" @@ -969,4 +1096,26 @@ static void logError(final HttpQuery query, final String msg, LOG.error(query.channel().toString() + ' ' + msg, e); } + static void validateString(final String what, final String s) { + validateString(what, s, ""); + } + + public static void validateString(final String what, final String s, String specials) { + if (s == null) { + throw new BadRequestException("Invalid " + what + ": null"); + } else if ("".equals(s)) { + throw new BadRequestException("Invalid " + what + ": empty string"); + } + final int n = s.length(); + for (int i = 0; i < n; i++) { + final char c = s.charAt(i); + if (!(('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') + || ('0' <= c && c <= '9') || c == '-' || c == '_' || c == '.' + || c == '/' || Character.isLetter(c) || specials.indexOf(c) != -1)) { + throw new BadRequestException("Invalid " + what + + " (\"" + s + "\"): illegal character: " + c); + } + } + } + } diff --git a/src/tsd/HistogramDataPointRpc.java b/src/tsd/HistogramDataPointRpc.java new file mode 100644 index 0000000000..20cf29d386 --- /dev/null +++ b/src/tsd/HistogramDataPointRpc.java @@ -0,0 +1,248 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.tsd; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.jboss.netty.handler.codec.http.HttpMethod; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; +import com.stumbleupon.async.Deferred; + +import net.opentsdb.core.Histogram; +import net.opentsdb.core.HistogramPojo; +import net.opentsdb.core.IncomingDataPoint; +import net.opentsdb.core.SimpleHistogram; +import net.opentsdb.core.SimpleHistogramDecoder; +import net.opentsdb.core.TSDB; +import net.opentsdb.core.Tags; +import net.opentsdb.utils.Config; + +/** + * The class responsible for writing histograms from Telnet calls or HTTP + * requests. + * + * @since 2.4 + */ +public class HistogramDataPointRpc extends PutDataPointRpc + implements TelnetRpc, HttpRpc { + + /** Type ref for the histo pojo. */ + private static final TypeReference> TYPE_REF = + new TypeReference>() {}; + + /** Whether or not histograms are enabled. */ + private final boolean enabled; + + /** + * Default ctor. Checks the "tsd.core.histograms.config" value to see if + * histograms are enabled. If they are not, then exceptions are returned. + * @param config A non-null config to pull data from. + */ + public HistogramDataPointRpc(final Config config) { + super(config); + // drats, since we can't look at the manager we'll look at the settings. + final String histo_config = config.getString("tsd.core.histograms.config"); + if (Strings.isNullOrEmpty(histo_config)) { + enabled = false; + } else { + enabled = true; + } + } + + @Override + public void execute(final TSDB tsdb, final HttpQuery query) throws IOException { + http_requests.incrementAndGet(); + + if (!enabled) { + throw new BadRequestException(HttpResponseStatus.SERVICE_UNAVAILABLE, + "Histogram storage has not been enabled. Check the " + + "'tsd.core.histograms.config' configuration."); + } + + // only accept POST + if (query.method() != HttpMethod.POST) { + throw new BadRequestException(HttpResponseStatus.METHOD_NOT_ALLOWED, + "Method not allowed", "The HTTP method [" + query.method().getName() + + "] is not permitted for this endpoint"); + } + + final List dps = query.serializer() + .parsePutV1(HistogramPojo.class, TYPE_REF); + processDataPoint(tsdb, query, dps); + } + + @Override + protected Deferred importDataPoint(final TSDB tsdb, + final String[] words) { + if (!enabled) { + throw new IllegalArgumentException( + "Histogram storage has not been enabled. Check the " + + "'tsd.core.histograms.config' configuration."); + } + + words[0] = null; // Ditch the "histogram". + if (words.length < 5) { // Need at least: metric timestamp value tag + // ^ 5 and not 4 because words[0] is "histogram". + throw new IllegalArgumentException("not enough arguments" + + " (need least 5, got " + + (words.length - 1) + ')'); + } + final String metric = words[1]; + if (metric.length() <= 0) { + throw new IllegalArgumentException("empty metric name"); + } + final long timestamp; + if (words[2].contains(".")) { + timestamp = Tags.parseLong(words[2].replace(".", "")); + } else { + timestamp = Tags.parseLong(words[2]); + } + if (timestamp <= 0) { + throw new IllegalArgumentException("invalid timestamp: " + timestamp); + } + + boolean has_id = false; + int id = 0; + try { + id = Integer.parseInt(words[3]); + has_id = true; + } catch (NumberFormatException e) { } + final String value; + if (has_id) { + value = words[4]; + } else { + // it's a simple Id + id = tsdb.histogramManager().getCodec(SimpleHistogramDecoder.class); + value = words[3]; + } + if (value.length() <= 0) { + throw new IllegalArgumentException("empty histogram value"); + } + final HashMap tags = new HashMap(); + for (int i = has_id ? 5 : 4; i < words.length; i++) { + if (!words[i].isEmpty()) { + Tags.parse(tags, words[i]); + } + } + + // validation and prepend the ID. + try { + final Histogram dp; + if (has_id) { + dp = tsdb.histogramManager().decode(id, + HistogramPojo.base64StringToBytes(value), false); + } else { + dp = parseTelnet(tsdb, value); + } + return tsdb.addHistogramPoint(metric, timestamp, + tsdb.histogramManager().encode(id, dp, true), tags); + } catch (Exception e) { + return Deferred.fromError(e); + } + } + + @Override + protected IncomingDataPoint getDataPointFromString(final TSDB tsdb, + final String[] words) { + final long timestamp; + if (words[2].contains(".")) { + timestamp = Tags.parseLong(words[2].replace(".", "")); + } else { + timestamp = Tags.parseLong(words[2]); + } + if (timestamp <= 0) { + throw new IllegalArgumentException("invalid timestamp: " + timestamp); + } + + boolean has_id = false; + int id = 0; + try { + id = Integer.parseInt(words[3]); + has_id = true; + } catch (NumberFormatException e) { } + final String value; + if (has_id) { + value = words[4]; + } else { + // it's a simple Id + id = tsdb.histogramManager().getCodec(SimpleHistogramDecoder.class); + value = words[3]; + } + + final HistogramPojo dp = new HistogramPojo(); + dp.setMetric(words[1]); + dp.setTimestamp(timestamp); + dp.setId(id); + if (has_id) { + dp.setValue(value); + } else { + dp.setValue(HistogramPojo.bytesToBase64String( + parseTelnet(tsdb, value).histogram(false))); + } + final HashMap tags = new HashMap(); + for (int i = has_id ? 5 : 4; i < words.length; i++) { + if (!words[i].isEmpty()) { + Tags.parse(tags, words[i]); + } + } + dp.setTags(tags); + return dp; + } + + SimpleHistogram parseTelnet(final TSDB tsdb, final String encoded) { + final SimpleHistogram shdp = new SimpleHistogram(tsdb.histogramManager() + .getCodec(SimpleHistogramDecoder.class)); + + final String[] buckets = Tags.splitString(encoded, ':'); + if (buckets.length < 1) { + throw new IllegalArgumentException("Must have at least one bucket in the " + + "histogram."); + } + + for (final String bucket : buckets) { + final String[] kv = Tags.splitString(bucket, '='); + if (kv.length != 2) { + throw new IllegalArgumentException("Improperly formatted bucket: " + + bucket); + } + + kv[0] = kv[0].toLowerCase(); + if (kv[0].equals("u")) { + shdp.setUnderflow(Long.parseLong(kv[1])); + } else if (kv[0].equals("o")) { + shdp.setOverflow(Long.parseLong(kv[1])); + } else { + final String[] bounds = Tags.splitString(kv[0], ','); + if (bounds.length != 2) { + throw new IllegalArgumentException("Improperly formatted bounds: " + + bucket); + } + shdp.addBucket(Float.parseFloat(bounds[0]), Float.parseFloat(bounds[1]), + Long.parseLong(kv[1])); + } + } + return shdp; + } + + @VisibleForTesting + boolean enabled() { + return enabled; + } +} diff --git a/src/tsd/HttpJsonSerializer.java b/src/tsd/HttpJsonSerializer.java index e82d4cf6f0..f0371cd44f 100644 --- a/src/tsd/HttpJsonSerializer.java +++ b/src/tsd/HttpJsonSerializer.java @@ -1,5 +1,5 @@ // This file is part of OpenTSDB. -// Copyright (C) 2013 The OpenTSDB Authors. +// Copyright (C) 2013-2023 The OpenTSDB Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU Lesser General Public License as published by @@ -20,34 +20,43 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.LinkedHashSet; import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; import org.jboss.netty.buffer.ChannelBuffer; import org.jboss.netty.buffer.ChannelBufferOutputStream; import org.jboss.netty.buffer.ChannelBuffers; import org.jboss.netty.handler.codec.http.HttpResponseStatus; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.type.TypeReference; +import com.stumbleupon.async.Callback; import com.stumbleupon.async.Deferred; +import net.opentsdb.core.Const; import net.opentsdb.core.DataPoint; import net.opentsdb.core.DataPoints; +import net.opentsdb.core.FillPolicy; import net.opentsdb.core.IncomingDataPoint; +import net.opentsdb.core.QueryException; import net.opentsdb.core.TSDB; import net.opentsdb.core.TSQuery; +import net.opentsdb.core.TSSubQuery; import net.opentsdb.meta.Annotation; import net.opentsdb.meta.TSMeta; import net.opentsdb.meta.UIDMeta; +import net.opentsdb.rollup.RollUpDataPoint; import net.opentsdb.search.SearchQuery; +import net.opentsdb.stats.QueryStats; +import net.opentsdb.stats.QueryStats.QueryStat; import net.opentsdb.tree.Branch; import net.opentsdb.tree.Tree; import net.opentsdb.tree.TreeRule; import net.opentsdb.tsd.AnnotationRpc.AnnotationBulkDelete; import net.opentsdb.tsd.QueryRpc.LastPointQuery; import net.opentsdb.utils.Config; +import net.opentsdb.utils.DateTime; import net.opentsdb.utils.JSON; /** @@ -59,13 +68,15 @@ * @since 2.0 */ class HttpJsonSerializer extends HttpSerializer { - private static final Logger LOG = - LoggerFactory.getLogger(HttpJsonSerializer.class); - + /** Type reference for incoming data points */ - private static TypeReference> TR_INCOMING = + static TypeReference> TR_INCOMING = new TypeReference>() {}; + /** Type reference for rollup data points */ + public static TypeReference> TR_ROLLUP = + new TypeReference>() {}; + /** Type reference for uid assignments */ private static TypeReference>> UID_ASSIGN = new TypeReference>>() {}; @@ -148,6 +159,41 @@ public List parsePutV1() { } } + /** + * Parses one or more data points for storage + * @return an array of data points to process for storage + * @throws JSONException if parsing failed + * @throws BadRequestException if the content was missing or parsing failed + * @since 2.4 + */ + @Override + public List parsePutV1( + final Class type, final TypeReference> typeReference) { + if (!query.hasContent()) { + throw new BadRequestException("Missing request content"); + } + + // convert to a string so we can handle character encoding properly + final String content = query.getContent().trim(); + final int firstbyte = content.charAt(0); + try { + if (firstbyte == '{') { + final T dp = + JSON.parseToObject(content, type); + final ArrayList dps = + new ArrayList(1); + dps.add(dp); + return dps; + } else if (firstbyte == '[') { + return JSON.parseToObject(content, typeReference); + } else { + throw new BadRequestException("The JSON must start as an object or an array"); + } + } catch (IllegalArgumentException iae) { + throw new BadRequestException("Unable to parse the given JSON", iae); + } + } + /** * Parses a suggestion query * @return a hash map of key/value pairs @@ -190,6 +236,26 @@ public HashMap> parseUidAssignV1() { } } + /** + * Parses metric, tagk or tagv, and name to rename UID + * @return as hash map of type and name + * @throws JSONException if parsing failed + * @throws BadRequestException if the content was missing or parsing failed + */ + public HashMap parseUidRenameV1() { + final String json = query.getContent(); + if (json == null || json.isEmpty()) { + throw new BadRequestException(HttpResponseStatus.BAD_REQUEST, + "Missing message content", + "Supply valid JSON formatted data in the body of your request"); + } + try { + return JSON.parseToObject(json, TR_HASH_MAP); + } catch (IllegalArgumentException iae) { + throw new BadRequestException("Unable to parse the given JSON", iae); + } + } + /** * Parses a timeseries data query * @return A TSQuery with data ready to validate @@ -204,7 +270,12 @@ public TSQuery parseQueryV1() { "Supply valid JSON formatted data in the body of your request"); } try { - return JSON.parseToObject(json, TSQuery.class); + TSQuery data_query = JSON.parseToObject(json, TSQuery.class); + // Filter out duplicate queries + Set query_set = new LinkedHashSet(data_query.getQueries()); + data_query.getQueries().clear(); + data_query.getQueries().addAll(query_set); + return data_query; } catch (IllegalArgumentException iae) { throw new BadRequestException("Unable to parse the given JSON", iae); } @@ -531,6 +602,15 @@ public ChannelBuffer formatUidAssignV1(final return this.serializeJSON(response); } + /** + * Format a response from the Uid Rename RPC + * @param response A map of result and error of the rename + * @return A JSON structure + * @throws JSONException if serialization failed + */ + public ChannelBuffer formatUidRenameV1(final Map response) { + return this.serializeJSON(response); + } /** * Format the results from a timeseries data query * @param data_query The TSQuery object used to fetch the results @@ -540,31 +620,132 @@ public ChannelBuffer formatUidAssignV1(final */ public ChannelBuffer formatQueryV1(final TSQuery data_query, final List results, final List globals) { + try { + return formatQueryAsyncV1(data_query, results, globals) + .joinUninterruptibly(); + } catch (QueryException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Shouldn't be here", e); + } + } + + /** + * Format the results from a timeseries data query + * @param data_query The TSQuery object used to fetch the results + * @param results The data fetched from storage + * @param globals An optional list of global annotation objects + * @return A Deferred object to pass on to the caller + * @throws IOException if serialization failed + * @since 2.2 + */ + public Deferred formatQueryAsyncV1(final TSQuery data_query, + final List results, final List globals) + throws IOException { + final long start = DateTime.currentTimeMillis(); final boolean as_arrays = this.query.hasQueryStringParam("arrays"); final String jsonp = this.query.getQueryStringParam("jsonp"); - // todo - this should be streamed at some point since it could be HUGE + // buffers and an array list to stored the deferreds final ChannelBuffer response = ChannelBuffers.dynamicBuffer(); final OutputStream output = new ChannelBufferOutputStream(response); - try { - // don't forget jsonp - if (jsonp != null && !jsonp.isEmpty()) { - output.write((jsonp + "(").getBytes(query.getCharset())); + // too bad an inner class can't modify a primitive. This is a work around + final List timeout_flag = new ArrayList(1); + timeout_flag.add(false); + + // start with JSONp if we're told to + if (jsonp != null && !jsonp.isEmpty()) { + output.write((jsonp + "(").getBytes(query.getCharset())); + } + + // start the JSON generator and write the opening array + final JsonGenerator json = JSON.getFactory().createGenerator(output); + json.writeStartArray(); + + /** + * Every individual data point set (the result of a query and possibly a + * group by) will initiate an asynchronous metric/tag UID to name resolution + * and then print to the buffer. + * NOTE that because this is asynchronous, the order of results is + * indeterminate. + */ + class DPsResolver implements Callback, Object> { + /** Has to be final to be shared with the nested classes */ + final StringBuilder metric = new StringBuilder(256); + /** Resolved tags */ + final Map tags = new ConcurrentHashMap(); + /** Resolved aggregated tags */ + final List agg_tags = new ArrayList(); + /** A list storing the metric and tag resolve calls */ + final List> resolve_deferreds = + new ArrayList>(); + /** The data points to serialize */ + final DataPoints dps; + /** Starting time in nanos when we sent the UID resolution queries off */ + long uid_start; + + public DPsResolver(final DataPoints dps) { + this.dps = dps; } - JsonGenerator json = JSON.getFactory().createGenerator(output); - json.writeStartArray(); - for (DataPoints[] separate_dps : results) { - for (DataPoints dps : separate_dps) { - json.writeStartObject(); + /** Resolves the metric UID to a name*/ + class MetricResolver implements Callback { + public Object call(final String metric) throws Exception { + DPsResolver.this.metric.append(metric); + return null; + } + } + + /** Resolves the tag UIDs to a key/value string set */ + class TagResolver implements Callback> { + public Object call(final Map tags) throws Exception { + DPsResolver.this.tags.putAll(tags); + return null; + } + } + + /** Resolves aggregated tags */ + class AggTagResolver implements Callback> { + public Object call(final List tags) throws Exception { + DPsResolver.this.agg_tags.addAll(tags); + return null; + } + } + + /** After the metric and tags have been resolved, this will print the + * results to the output buffer in the proper format. + */ + class WriteToBuffer implements Callback> { + final DataPoints dps; + + /** + * Default ctor that takes a data point set + * @param dps Datapoints to print + */ + public WriteToBuffer(final DataPoints dps) { + this.dps = dps; + } + + /** + * Handles writing the data to the output buffer. The results of the + * deferreds don't matter as they will be stored in the class final + * variables. + */ + public Object call(final ArrayList deferreds) throws Exception { + data_query.getQueryStats().addStat(dps.getQueryIndex(), + QueryStat.UID_TO_STRING_TIME, (DateTime.nanoTime() - uid_start)); + final long local_serialization_start = DateTime.nanoTime(); + final TSSubQuery orig_query = data_query.getQueries() + .get(dps.getQueryIndex()); - json.writeStringField("metric", dps.metricName()); + json.writeStartObject(); + json.writeStringField("metric", metric.toString()); json.writeFieldName("tags"); json.writeStartObject(); if (dps.getTags() != null) { - for (Map.Entry tag : dps.getTags().entrySet()) { + for (Map.Entry tag : tags.entrySet()) { json.writeStringField(tag.getKey(), tag.getValue()); } } @@ -573,12 +754,16 @@ public ChannelBuffer formatQueryV1(final TSQuery data_query, json.writeFieldName("aggregateTags"); json.writeStartArray(); if (dps.getAggregatedTags() != null) { - for (String atag : dps.getAggregatedTags()) { + for (String atag : agg_tags) { json.writeString(atag); } } json.writeEndArray(); + if (data_query.getShowQuery()) { + json.writeObjectField("query", orig_query); + } + if (data_query.getShowTSUIDs()) { json.writeFieldName("tsuids"); json.writeStartArray(); @@ -596,6 +781,13 @@ public ChannelBuffer formatQueryV1(final TSQuery data_query, Collections.sort(annotations); json.writeArrayFieldStart("annotations"); for (Annotation note : annotations) { + long ts = note.getStartTime(); + if (!((ts & Const.SECOND_MASK) != 0)) { + ts *= 1000; + } + if (ts < data_query.startTime() || ts > data_query.endTime()) { + continue; + } json.writeObject(note); } json.writeEndArray(); @@ -605,17 +797,27 @@ public ChannelBuffer formatQueryV1(final TSQuery data_query, Collections.sort(globals); json.writeArrayFieldStart("globalAnnotations"); for (Annotation note : globals) { + long ts = note.getStartTime(); + if (!((ts & Const.SECOND_MASK) != 0)) { + ts *= 1000; + } + if (ts < data_query.startTime() || ts > data_query.endTime()) { + continue; + } json.writeObject(note); } json.writeEndArray(); } } - // now the fun stuff, dump the data + // now the fun stuff, dump the data and time just the iteration over + // the data points + final long dps_start = DateTime.nanoTime(); json.writeFieldName("dps"); + long counter = 0; // default is to write a map, otherwise write arrays - if (as_arrays) { + if (!timeout_flag.get(0) && as_arrays) { json.writeStartArray(); for (final DataPoint dp : dps) { if (dp.timestamp() < data_query.startTime() || @@ -629,12 +831,20 @@ public ChannelBuffer formatQueryV1(final TSQuery data_query, if (dp.isInteger()) { json.writeNumber(dp.longValue()); } else { - json.writeNumber(dp.doubleValue()); + // Report missing intervals as null or NaN. + final double value = dp.doubleValue(); + if (Double.isNaN(value) && + orig_query.fillPolicy() == FillPolicy.NULL) { + json.writeNull(); + } else { + json.writeNumber(dp.doubleValue()); + } } json.writeEndArray(); + ++counter; } json.writeEndArray(); - } else { + } else if (!timeout_flag.get(0)) { json.writeStartObject(); for (final DataPoint dp : dps) { if (dp.timestamp() < (data_query.startTime()) || @@ -646,29 +856,137 @@ public ChannelBuffer formatQueryV1(final TSQuery data_query, if (dp.isInteger()) { json.writeNumberField(Long.toString(timestamp), dp.longValue()); } else { - json.writeNumberField(Long.toString(timestamp), dp.doubleValue()); + // Report missing intervals as null or NaN. + final double value = dp.doubleValue(); + if (Double.isNaN(value) && + orig_query.fillPolicy() == FillPolicy.NULL) { + json.writeNullField(Long.toString(timestamp)); + } else { + json.writeNumberField(Long.toString(timestamp), dp.doubleValue()); + } } + ++counter; } json.writeEndObject(); + + } else { + // skipping data points all together due to timeout + json.writeStartObject(); + json.writeEndObject(); + } + + final long agg_time = DateTime.nanoTime() - dps_start; + data_query.getQueryStats().addStat(dps.getQueryIndex(), + QueryStat.AGGREGATION_TIME, agg_time); + data_query.getQueryStats().addStat(dps.getQueryIndex(), + QueryStat.AGGREGATED_SIZE, counter); + + // yeah, it's a little early but we need to dump it out with the results. + data_query.getQueryStats().addStat(dps.getQueryIndex(), + QueryStat.SERIALIZATION_TIME, + DateTime.nanoTime() - local_serialization_start); + if (!timeout_flag.get(0) && data_query.getShowStats()) { + int query_index = (dps == null) ? -1 : dps.getQueryIndex(); + QueryStats stats = data_query.getQueryStats(); + + if (query_index >= 0) { + json.writeFieldName("stats"); + final Map s = stats.getQueryStats(query_index, false); + if (s != null) { + json.writeObject(s); + } else { + json.writeStringField("ERROR", "NO STATS FOUND"); + } + } } // close the results for this particular query json.writeEndObject(); + return null; } } - - // close - json.writeEndArray(); - json.close(); - if (jsonp != null && !jsonp.isEmpty()) { - output.write(")".getBytes()); + /** + * When called, initiates a resolution of metric and tag UIDs to names, + * then prints to the output buffer once they are completed. + */ + public Deferred call(final Object obj) throws Exception { + this.uid_start = DateTime.nanoTime(); + + resolve_deferreds.add(dps.metricNameAsync() + .addCallback(new MetricResolver())); + resolve_deferreds.add(dps.getTagsAsync() + .addCallback(new TagResolver())); + resolve_deferreds.add(dps.getAggregatedTagsAsync() + .addCallback(new AggTagResolver())); + return Deferred.group(resolve_deferreds) + .addCallback(new WriteToBuffer(dps)); + } + + } + + // We want the serializer to execute serially so we need to create a callback + // chain so that when one DPsResolver is finished, it triggers the next to + // start serializing. + final int LIMIT = 1 << 13; + int counter = 0; + Deferred cb_chain = new Deferred(); + + for (DataPoints[] separate_dps : results) { + for (DataPoints dps : separate_dps) { + try { + cb_chain.addCallback(new DPsResolver(dps)); + } catch (Exception e) { + throw new RuntimeException("Unexpected error durring resolution", e); + } + if (++counter >= LIMIT) { + counter = 0; + // trigger the callback chain chunk here + cb_chain.callback(null); + try { + cb_chain.joinUninterruptibly(); + } catch (Exception e1) { + // chain already joined + } + cb_chain = new Deferred(); + } } - return response; - } catch (IOException e) { - LOG.error("Unexpected exception", e); - throw new RuntimeException(e); } + + /** Final callback to close out the JSON array and return our results */ + class FinalCB implements Callback { + public ChannelBuffer call(final Object obj) + throws Exception { + + // Call this here so we rollup sub metrics into a summary. It's not + // completely accurate, of course, because we still have to write the + // summary and close the writer. But it's close. + data_query.getQueryStats().markSerializationSuccessful(); + + // dump overall stats as an extra object in the array + // TODO - yeah, I've heard this sucks, we need to figure out a better way. + if (data_query.getShowSummary()) { + final QueryStats stats = data_query.getQueryStats(); + json.writeStartObject(); + json.writeFieldName("statsSummary"); + json.writeObject(stats.getStats(true, true)); + json.writeEndObject(); + } + + // IMPORTANT Make sure the close the JSON array and the generator + json.writeEndArray(); + json.close(); + + if (jsonp != null && !jsonp.isEmpty()) { + output.write(")".getBytes()); + } + return response; + } + } + + // trigger the callback chain here - will be joined from outside + cb_chain.callback(null); + return cb_chain.addCallback(new FinalCB()); } /** @@ -823,6 +1141,50 @@ public ChannelBuffer formatStatsV1(final List stats) { return serializeJSON(stats); } + /** + * Format a list of thread statistics + * @param stats The thread statistics list to format + * @return A ChannelBuffer object to pass on to the caller + * @throws JSONException if serialization failed + * @since 2.2 + */ + public ChannelBuffer formatThreadStatsV1(final List> stats) { + return serializeJSON(stats); + } + + /** + * format a list of region client statistics + * @param stats The list of region client stats to format + * @return A ChannelBuffer object to pass on to the caller + * @throws JSONException if serialization failed + * @since 2.2 + */ + public ChannelBuffer formatRegionStatsV1(final List> stats) { + return serializeJSON(stats); + } + + /** + * Format a list of JVM statistics + * @param stats The JVM stats map to format + * @return A ChannelBuffer object to pass on to the caller + * @throws JSONException if serialization failed + * @since 2.2 + */ + public ChannelBuffer formatJVMStatsV1(final Map> stats) { + return serializeJSON(stats); + } + + /** + * Format the query stats + * @param query_stats Map of query statistics + * @return A ChannelBuffer object to pass on to the caller + * @throws BadRequestException if the plugin has not implemented this method + * @since 2.2 + */ + public ChannelBuffer formatQueryStatsV1(final Map query_stats) { + return serializeJSON(query_stats); + } + /** * Format the response from a search query * @param note The query (hopefully filled with results) to serialize @@ -849,6 +1211,17 @@ public ChannelBuffer formatConfigV1(final Config config) { return serializeJSON(map); } + /** + * Format the loaded filter configurations + * @param config The filters to serialize + * @return A ChannelBuffer object to pass on to the caller + * @throws JSONException if serialization failed + */ + public ChannelBuffer formatFilterConfigV1( + final Map> config) { + return serializeJSON(config); + } + /** * Helper object for the format calls to wrap the JSON response in a JSONP * function if requested. Used for code dedupe. diff --git a/src/tsd/HttpQuery.java b/src/tsd/HttpQuery.java index f1308c5e2f..10df901840 100644 --- a/src/tsd/HttpQuery.java +++ b/src/tsd/HttpQuery.java @@ -20,20 +20,18 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; -import java.nio.charset.Charset; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; -import ch.qos.logback.classic.spi.ThrowableProxy; -import ch.qos.logback.classic.spi.ThrowableProxyUtil; - -import com.stumbleupon.async.Deferred; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import com.google.common.html.HtmlEscapers; +import net.opentsdb.core.Const; +import net.opentsdb.core.TSDB; +import net.opentsdb.graph.Plot; +import net.opentsdb.stats.Histogram; +import net.opentsdb.stats.StatsCollector; +import net.opentsdb.utils.PluginLoader; import org.jboss.netty.buffer.ChannelBuffer; import org.jboss.netty.buffer.ChannelBuffers; @@ -41,22 +39,18 @@ import org.jboss.netty.channel.ChannelFuture; import org.jboss.netty.channel.ChannelFutureListener; import org.jboss.netty.channel.DefaultFileRegion; -import org.jboss.netty.handler.codec.http.DefaultHttpResponse; import org.jboss.netty.handler.codec.http.HttpHeaders; import org.jboss.netty.handler.codec.http.HttpMethod; import org.jboss.netty.handler.codec.http.HttpRequest; import org.jboss.netty.handler.codec.http.HttpResponseStatus; -import org.jboss.netty.handler.codec.http.HttpVersion; -import org.jboss.netty.handler.codec.http.QueryStringDecoder; import org.jboss.netty.util.CharsetUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import net.opentsdb.core.Const; -import net.opentsdb.core.TSDB; -import net.opentsdb.graph.Plot; -import net.opentsdb.stats.Histogram; -import net.opentsdb.stats.StatsCollector; -import net.opentsdb.tsd.HttpSerializer; -import net.opentsdb.utils.PluginLoader; +import ch.qos.logback.classic.spi.ThrowableProxy; +import ch.qos.logback.classic.spi.ThrowableProxyUtil; + +import com.stumbleupon.async.Deferred; /** * Binds together an HTTP request and the channel on which it was received. @@ -64,7 +58,7 @@ * It makes it easier to provide a few utility methods to respond to the * requests. */ -final class HttpQuery { +final class HttpQuery extends AbstractHttpQuery { private static final Logger LOG = LoggerFactory.getLogger(HttpQuery.class); @@ -90,37 +84,12 @@ final class HttpQuery { /** Caches serializer implementation information for user access */ private static ArrayList> serializer_status = null; - /** When the query was started (useful for timing). */ - private final long start_time = System.nanoTime(); - - /** The request in this HTTP query. */ - private final HttpRequest request; - - /** The channel on which the request was received. */ - private final Channel chan; - - /** Shortcut to the request method */ - private final HttpMethod method; - - /** Parsed query string (lazily built on first access). */ - private Map> querystring; - /** API version parsed from the incoming request */ private int api_version = 0; /** The serializer to use for parsing input and responding */ private HttpSerializer serializer = null; - /** Deferred result of this query, to allow asynchronous processing. */ - private final Deferred deferred = new Deferred(); - - /** The response object we'll fill with data */ - private final DefaultHttpResponse response = - new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK); - - /** The {@code TSDB} instance we belong to */ - private final TSDB tsdb; - /** Whether or not to show stack traces in the output */ private final boolean show_stack_trace; @@ -130,12 +99,9 @@ final class HttpQuery { * @param chan The channel on which the request was received. */ public HttpQuery(final TSDB tsdb, final HttpRequest request, final Channel chan) { - this.tsdb = tsdb; - this.request = request; - this.chan = chan; + super(tsdb, request, chan); this.show_stack_trace = tsdb.getConfig().getBoolean("tsd.http.show_stack_trace"); - this.method = request.getMethod(); this.serializer = new HttpJsonSerializer(this); } @@ -147,30 +113,6 @@ public static void collectStats(final StatsCollector collector) { collector.record("http.latency", httplatency, "type=all"); } - /** - * Returns the underlying Netty {@link HttpRequest} of this query. - */ - public HttpRequest request() { - return request; - } - - /** Returns the HTTP method/verb for the request */ - public HttpMethod method() { - return this.method; - } - - /** Returns the response object, allowing serializers to set headers */ - public DefaultHttpResponse response() { - return this.response; - } - - /** - * Returns the underlying Netty {@link Channel} of this query. - */ - public Channel channel() { - return chan; - } - /** * Returns the version for an API request. If the request was for a deprecated * API call (such as /q, /suggest, /logs) this value will be 0. If the request @@ -194,131 +136,12 @@ public Deferred getDeferred() { return deferred; } - /** Returns how many ms have elapsed since this query was created. */ - public int processingTimeMillis() { - return (int) ((System.nanoTime() - start_time) / 1000000); - } - /** @return The selected seralizer. Will return null if {@link #setSerializer} * hasn't been called yet @since 2.0 */ public HttpSerializer serializer() { return this.serializer; } - /** - * Returns the query string parameters passed in the URI. - */ - public Map> getQueryString() { - if (querystring == null) { - try { - querystring = new QueryStringDecoder(request.getUri()).getParameters(); - } catch (IllegalArgumentException e) { - throw new BadRequestException("Bad query string: " + e.getMessage()); - } - } - return querystring; - } - - /** - * Returns the value of the given query string parameter. - *

    - * If this parameter occurs multiple times in the URL, only the last value - * is returned and others are silently ignored. - * @param paramname Name of the query string parameter to get. - * @return The value of the parameter or {@code null} if this parameter - * wasn't passed in the URI. - */ - public String getQueryStringParam(final String paramname) { - final List params = getQueryString().get(paramname); - return params == null ? null : params.get(params.size() - 1); - } - - /** - * Returns the non-empty value of the given required query string parameter. - *

    - * If this parameter occurs multiple times in the URL, only the last value - * is returned and others are silently ignored. - * @param paramname Name of the query string parameter to get. - * @return The value of the parameter. - * @throws BadRequestException if this query string parameter wasn't passed - * or if its last occurrence had an empty value ({@code &a=}). - */ - public String getRequiredQueryStringParam(final String paramname) - throws BadRequestException { - final String value = getQueryStringParam(paramname); - if (value == null || value.isEmpty()) { - throw BadRequestException.missingParameter(paramname); - } - return value; - } - - /** - * Returns whether or not the given query string parameter was passed. - * @param paramname Name of the query string parameter to get. - * @return {@code true} if the parameter - */ - public boolean hasQueryStringParam(final String paramname) { - return getQueryString().get(paramname) != null; - } - - /** - * Returns all the values of the given query string parameter. - *

    - * In case this parameter occurs multiple times in the URL, this method is - * useful to get all the values. - * @param paramname Name of the query string parameter to get. - * @return The values of the parameter or {@code null} if this parameter - * wasn't passed in the URI. - */ - public List getQueryStringParams(final String paramname) { - return getQueryString().get(paramname); - } - - /** - * Returns only the path component of the URI as a string - * This call strips the protocol, host, port and query string parameters - * leaving only the path e.g. "/path/starts/here" - *

    - * Note that for slightly quicker performance you can call request().getUri() - * to get the full path as a string but you'll have to strip query string - * parameters manually. - * @return The path component of the URI - * @throws NullPointerException if the URI is null - * @since 2.0 - */ - public String getQueryPath() { - return new QueryStringDecoder(request.getUri()).getPath(); - } - - /** - * Returns the path component of the URI as an array of strings, split on the - * forward slash - * Similar to the {@link #getQueryPath} call, this returns only the path - * without the protocol, host, port or query string params. E.g. - * "/path/starts/here" will return an array of {"path", "starts", "here"} - *

    - * Note that for maximum speed you may want to parse the query path manually. - * @return An array with 1 or more components, note the first item may be - * an empty string. - * @throws BadRequestException if the URI is empty or does not start with a - * slash - * @throws NullPointerException if the URI is null - * @since 2.0 - */ - public String[] explodePath() { - final String path = this.getQueryPath(); - if (path.isEmpty()) { - throw new BadRequestException("Query path is empty"); - } - if (path.charAt(0) != '/') { - throw new BadRequestException("Query path doesn't start with a slash"); - } - // split may be a tad slower than other methods, but since the URIs are - // usually pretty short and not every request will make this call, we - // probably don't need any premature optimization - return path.substring(1).split("/"); - } - /** * Helper that strips the api and optional version from the URI array since * api calls only care about what comes after. @@ -364,8 +187,6 @@ public String[] explodeAPIPath() { } /** - * Parses the query string to determine the base route for handing a query - * off to an RPC handler. * This method splits the query path component and returns a string suitable * for routing by {@link RpcHandler}. The resulting route is always lower case * and will consist of either an empty string, a deprecated API call or an @@ -383,8 +204,9 @@ public String[] explodeAPIPath() { * max or the version # can't be parsed * @since 2.0 */ + @Override public String getQueryBaseRoute() { - final String[] split = this.explodePath(); + final String[] split = explodePath(); if (split.length < 1) { return ""; } @@ -423,42 +245,6 @@ public String getQueryBaseRoute() { return "api/" + split[2].toLowerCase(); } - /** - * Attempts to parse the character set from the request header. If not set - * defaults to UTF-8 - * @return A Charset object - * @throws UnsupportedCharsetException if the parsed character set is invalid - * @since 2.0 - */ - public Charset getCharset() { - // RFC2616 3.7 - for (String type : this.request.headers().getAll("Content-Type")) { - int idx = type.toUpperCase().indexOf("CHARSET="); - if (idx > 1) { - String charset = type.substring(idx+8); - return Charset.forName(charset); - } - } - return Charset.forName("UTF-8"); - } - - /** @return True if the request has content, false if not @since 2.0 */ - public boolean hasContent() { - return this.request.getContent() != null && - this.request.getContent().readable(); - } - - /** - * Decodes the request content to a string using the appropriate character set - * @return Decoded content or an empty string if the request did not include - * content - * @throws UnsupportedCharsetException if the parsed character set is invalid - * @since 2.0 - */ - public String getContent() { - return this.request.getContent().toString(this.getCharset()); - } - /** * Determines the requested HttpMethod via VERB and QS override. * If the request is a {@code GET} and the user provides a valid override @@ -538,7 +324,7 @@ public void setSerializer() throws InvocationTargetException, // attempt to parse the Content-Type string. We only want the first part, // not the character set. And if the CT is missing, we'll use the default // serializer - String content_type = this.request.headers().get("Content-Type"); + String content_type = request().headers().get("Content-Type"); if (content_type == null || content_type.isEmpty()) { return; } @@ -560,8 +346,9 @@ public void setSerializer() throws InvocationTargetException, * API calls * @param cause The unexpected exception that caused this error. */ + @Override public void internalError(final Exception cause) { - logError("Internal Server Error on " + request.getUri(), cause); + logError("Internal Server Error on " + request().getUri(), cause); if (this.api_version > 0) { // always default to the latest version of the error formatter since we @@ -586,9 +373,11 @@ public void internalError(final Exception cause) { HttpQuery.escapeJson(pretty_exc, buf); buf.append("\"}"); sendReply(HttpResponseStatus.INTERNAL_SERVER_ERROR, buf); - } else if (hasQueryStringParam("png")) { - sendAsPNG(HttpResponseStatus.INTERNAL_SERVER_ERROR, pretty_exc, 30); } else { + String response = ""; + if (pretty_exc != null) { + response = HtmlEscapers.htmlEscaper().escape(pretty_exc); + } sendReply(HttpResponseStatus.INTERNAL_SERVER_ERROR, makePage("Internal Server Error", "Houston, we have a problem", "

    " @@ -596,7 +385,7 @@ public void internalError(final Exception cause) { + "Oops, sorry but your request failed due to a" + " server error.

    " + "Please try again in 30 seconds.
    "
-                         + pretty_exc
+                         + response
                          + "
    ")); } } @@ -611,12 +400,13 @@ public void badRequest(final String explain) { } /** - * Sends an error message to the client with the proeper status code and - * optional details stored in the exception + * Sends an error message to the client. Handles responses from + * deprecated API calls. * @param exception The exception that was thrown */ + @Override public void badRequest(final BadRequestException exception) { - logWarn("Bad Request on " + request.getUri() + ": " + exception.getMessage()); + logWarn("Bad Request on " + request().getUri() + ": " + exception.getMessage()); if (this.api_version > 0) { // always default to the latest version of the error formatter since we // need to return something @@ -635,8 +425,14 @@ public void badRequest(final BadRequestException exception) { buf.append("\"}"); sendReply(HttpResponseStatus.BAD_REQUEST, buf); } else if (hasQueryStringParam("png")) { - sendAsPNG(HttpResponseStatus.BAD_REQUEST, exception.getMessage(), 3600); + final StringBuilder buf = new StringBuilder(10 + + exception.getDetails().length()); + sendReply(HttpResponseStatus.BAD_REQUEST, buf); } else { + String response = ""; + if (exception.getMessage() != null) { + response = HtmlEscapers.htmlEscaper().escape(exception.getMessage()); + } sendReply(HttpResponseStatus.BAD_REQUEST, makePage("Bad Request", "Looks like it's your fault this time", "
    " @@ -644,14 +440,18 @@ public void badRequest(final BadRequestException exception) { + "Sorry but your request was rejected as being" + " invalid.

    " + "The reason provided was:
    " - + exception.getMessage() + + response + "
    ")); } } - /** Sends a 404 error page to the client. */ + /** + * Sends a 404 error page to the client. + * Handles responses from deprecated API calls + */ + @Override public void notFound() { - logWarn("Not Found: " + request.getUri()); + logWarn("Not Found: " + request().getUri()); if (this.api_version > 0) { // always default to the latest version of the error formatter since we // need to return something @@ -665,8 +465,6 @@ public void notFound() { if (hasQueryStringParam("json")) { sendReply(HttpResponseStatus.NOT_FOUND, new StringBuilder("{\"err\":\"Page Not Found\"}")); - } else if (hasQueryStringParam("png")) { - sendAsPNG(HttpResponseStatus.NOT_FOUND, "Page Not Found", 3600); } else { sendReply(HttpResponseStatus.NOT_FOUND, PAGE_NOT_FOUND); } @@ -675,7 +473,7 @@ public void notFound() { /** Redirects the client's browser to the given location. */ public void redirect(final String location) { // set the header AND a meta refresh just in case - response.headers().set("Location", location); + response().headers().set("Location", location); sendReply(HttpResponseStatus.OK, new StringBuilder( " 0) { - response.headers().set(HttpHeaders.Names.AGE, + response().headers().set(HttpHeaders.Names.AGE, (System.currentTimeMillis() - mtime) / 1000); } else { logWarn("Found a file with mtime=" + mtime + ": " + path); } - response.headers().set(HttpHeaders.Names.CACHE_CONTROL, + response().headers().set(HttpHeaders.Names.CACHE_CONTROL, "max-age=" + max_age); - HttpHeaders.setContentLength(response, length); - chan.write(response); + HttpHeaders.setContentLength(response(), length); + channel().write(response()); } final DefaultFileRegion region = new DefaultFileRegion(file.getChannel(), 0, length); - final ChannelFuture future = chan.write(region); + final ChannelFuture future = channel().write(region); future.addListener(new ChannelFutureListener() { public void operationComplete(final ChannelFuture future) { region.releaseExternalResources(); done(); } }); - if (!HttpHeaders.isKeepAlive(request)) { + if (!HttpHeaders.isKeepAlive(request())) { future.addListener(ChannelFutureListener.CLOSE); } } @@ -961,10 +695,11 @@ public void operationComplete(final ChannelFuture future) { /** * Method to call after writing the HTTP response to the wire. */ - private void done() { + @Override + public void done() { final int processing_time = processingTimeMillis(); httplatency.add(processing_time); - logInfo("HTTP " + request.getUri() + " done in " + processing_time + "ms"); + logInfo("HTTP " + request().getUri() + " done in " + processing_time + "ms"); deferred.callback(null); } @@ -975,28 +710,9 @@ private void done() { */ private void sendBuffer(final HttpResponseStatus status, final ChannelBuffer buf) { - if (!chan.isConnected()) { - done(); - return; - } - response.headers().set(HttpHeaders.Names.CONTENT_TYPE, - (api_version < 1 ? guessMimeType(buf) : - serializer.responseContentType())); - - // TODO(tsuna): Server, X-Backend, etc. headers. - // only reset the status if we have the default status, otherwise the user - // already set it - response.setStatus(status); - response.setContent(buf); - final boolean keepalive = HttpHeaders.isKeepAlive(request); - if (keepalive) { - HttpHeaders.setContentLength(response, buf.readableBytes()); - } - final ChannelFuture future = chan.write(response); - if (!keepalive) { - future.addListener(ChannelFutureListener.CLOSE); - } - done(); + final String contentType = (api_version < 1 ? guessMimeType(buf) : + serializer.responseContentType()); + sendBuffer(status, buf, contentType); } /** @@ -1004,7 +720,7 @@ private void sendBuffer(final HttpResponseStatus status, * @param buf The content of the reply to send. */ private String guessMimeType(final ChannelBuffer buf) { - final String mimetype = guessMimeTypeFromUri(request.getUri()); + final String mimetype = guessMimeTypeFromUri(request().getUri()); return mimetype == null ? guessMimeTypeFromContents(buf) : mimetype; } @@ -1243,39 +959,18 @@ public static StringBuilder makePage(final String htmlheader, .append(PAGE_FOOTER); return buf; } - - /** @return Information about the query */ - public String toString() { - return "HttpQuery" - + "(start_time=" + start_time - + ", request=" + request - + ", chan=" + chan - + ", querystring=" + querystring - + ')'; + + @Override + protected Logger logger() { + return LOG; } - - // ---------------- // - // Logging helpers. // - // ---------------- // - - private void logInfo(final String msg) { - LOG.info(chan.toString() + ' ' + msg); - } - - private void logWarn(final String msg) { - LOG.warn(chan.toString() + ' ' + msg); - } - - private void logError(final String msg, final Exception e) { - LOG.error(chan.toString() + ' ' + msg, e); - } - + // -------------------------------------------- // // Boilerplate (shamelessly stolen from Google) // // -------------------------------------------- // private static final String PAGE_HEADER_START = - "" + "" + "" + "" + ""; @@ -1286,7 +981,6 @@ private void logError(final String msg, final Exception e) { + "body{font-family:arial,sans-serif;margin-left:2em}" + "A.l:link{color:#6f6f6f}" + "A.u:link{color:green}" - + ".subg{background-color:#e2f4f7}" + ".fwf{font-family:monospace;white-space:pre-wrap}" + "//--></style>"; @@ -1294,12 +988,10 @@ private void logError(final String msg, final Exception e) { "</head>\n" + "<body text=#000000 bgcolor=#ffffff>" + "<table border=0 cellpadding=2 cellspacing=0 width=100%>" - + "<tr><td rowspan=3 width=1% nowrap><b>" - + "<font color=#c71a32 size=10>T</font>" - + "<font color=#00a189 size=10>S</font>" - + "<font color=#1a65b7 size=10>D</font>" - + "  </b><td> </td></tr>" - + "<tr><td class=subg><font color=#507e9b><b>"; + + "<tr><td rowspan=3 width=1% nowrap>" + + "<img src=s/opentsdb_header.jpg>" + + "<td> </td></tr>" + + "<tr><td><font color=#507e9b><b>"; private static final String PAGE_BODY_MID = "</b></td></tr>" diff --git a/src/tsd/HttpRpc.java b/src/tsd/HttpRpc.java index fc1e3f0bcb..73b3fe783c 100644 --- a/src/tsd/HttpRpc.java +++ b/src/tsd/HttpRpc.java @@ -16,7 +16,9 @@ import net.opentsdb.core.TSDB; -/** Base interface for all HTTP query handlers. */ +/** + * Base interface for all built-in HTTP query handlers. + */ interface HttpRpc { /** @@ -24,6 +26,6 @@ interface HttpRpc { * @param tsdb The TSDB to use. * @param query The HTTP query to execute. */ - void execute(TSDB tsdb, HttpQuery query) throws IOException; + void execute(TSDB tsdb, HttpQuery query) throws BadRequestException, IOException; } diff --git a/src/tsd/HttpRpcPlugin.java b/src/tsd/HttpRpcPlugin.java new file mode 100644 index 0000000000..940393a68f --- /dev/null +++ b/src/tsd/HttpRpcPlugin.java @@ -0,0 +1,113 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2010-2014 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.tsd; + +import java.io.IOException; + +import com.stumbleupon.async.Deferred; + +import net.opentsdb.core.TSDB; +import net.opentsdb.stats.StatsCollector; + +/** + * A plugin that runs along side TSD's built-in HTTP endpoints (like the + * <code>/api</code> endpoints). There can be multiple implementations of + * such plugins per TSD. These plugins run on the same Netty server as + * built-in HTTP endpoints and thus are available on the same port as those + * endpoints. However, these plugins are mounted beneath a special base + * path called <code>/plugin</code>. + * + * <p>Notes on multi-threaded behavior: + * <ul> + * <li>Plugins are created and initialized <strong>once</strong> per instance + * of the TSD. Therefore, these plugins are effectively singletons. + * <li>Plugins will be executed from multiple threads so the {@link #execute} + * and {@link collectStats} methods <strong>must be thread safe</strong> + * with respect to the plugin's internal state and external resources. + * </ul> + * @since 2.2 + */ +public abstract class HttpRpcPlugin { + /** + * Called by TSDB to initialize the plugin. This is called <strong>once</strong> + * (and from a single thread) at the time the plugin in loaded. + * + * <p><b>Note:</b> Implementations should throw exceptions if they can't start + * up properly. The TSD will then shutdown so the operator can fix the + * problem. Please use IllegalArgumentException for configuration issues. + * + * @param tsdb The parent TSDB object + * @throws IllegalArgumentException if required configuration parameters are + * missing + * @throws RuntimeException + */ + public abstract void initialize(TSDB tsdb); + + /** + * Called to gracefully shutdown the plugin. This is called <strong>once</strong> + * (and from a single thread) at the time the owning TSD is shutting down. + * + * @return A deferred object that indicates the completion of the request. + * The {@link Object} has not special meaning and can be {@code null} + * (think of it as {@code Deferred<Void>}). + */ + public abstract Deferred<Object> shutdown(); + + /** + * Should return the version of this plugin in the format: + * MAJOR.MINOR.MAINT, e.g. "2.0.1". The MAJOR version should match the major + * version of OpenTSDB the plugin is meant to work with. + * @return A version string used to log the loaded version + */ + public abstract String version(); + + /** + * Called by the TSD when a request for statistics collection has come in. The + * implementation may provide one or more statistics. If no statistics are + * available for the implementation, simply stub the method. + * + * <p><strong>Note:</strong> Must be thread-safe. + * + * @param collector The collector used for emitting statistics + */ + public abstract void collectStats(StatsCollector collector); + + /** + * The (web) path this plugin should be available at. This value + * <strong>should</strong> start with a <code>/</code>. However, it + * <strong>must not</strong> contain the system's plugin base path or the + * plugin will fail to load. + * + * <p>Here are some examples where + * <code>path --(is available at)--> server path</code> + * <ul> + * <li><code>/myAwesomePlugin --> /plugin/myAwesomePlugin</code> + * <li><code>/myOtherPlugin/operation --> /plugin/myOtherPlugin/operation</code> + * </ul> + * + * @return a slash separated path + */ + public abstract String getPath(); + + /** + * Executes the plugin for the given query received on the path derived from + * {@link #getPath()}. This method will be called by multiple threads + * simultaneously and <b>must be</b> thread-safe. + * + * @param tsdb the owning TSDB instance. + * @param query the parsed query + * @throws IOException + */ + public abstract void execute(TSDB tsdb, HttpRpcPluginQuery query) throws IOException; + +} diff --git a/src/tsd/HttpRpcPluginQuery.java b/src/tsd/HttpRpcPluginQuery.java new file mode 100644 index 0000000000..55464e99af --- /dev/null +++ b/src/tsd/HttpRpcPluginQuery.java @@ -0,0 +1,44 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2010-2014 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.tsd; + +import org.jboss.netty.channel.Channel; +import org.jboss.netty.handler.codec.http.HttpRequest; + +import net.opentsdb.core.TSDB; + +/** + * Query class for {@link HttpRpcPlugin}s. Binds together a request, its + * owning channel, and reponse helpers. + * + * @since 2.2 + */ +public final class HttpRpcPluginQuery extends AbstractHttpQuery { + public HttpRpcPluginQuery(final TSDB tsdb, final HttpRequest request, final Channel chan) { + super(tsdb, request, chan); + } + + /** + * Return the base route with no plugin prefix in it. This is matched with + * values returned by {@link HttpRpcPlugin#getPath()}. + * @return the base route path (no query parameters, etc.) + */ + @Override + public String getQueryBaseRoute() { + final String[] parts = explodePath(); + if (parts.length < 2) { // Must be at least something like: /plugin/blah + throw new BadRequestException("Invalid plugin request path: " + getQueryPath()); + } + return parts[1]; + } +} diff --git a/src/tsd/HttpSerializer.java b/src/tsd/HttpSerializer.java index 8f92cde1dc..f6108466f3 100644 --- a/src/tsd/HttpSerializer.java +++ b/src/tsd/HttpSerializer.java @@ -12,6 +12,8 @@ // see <http://www.gnu.org/licenses/>. package net.opentsdb.tsd; +import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -25,6 +27,7 @@ import ch.qos.logback.classic.spi.ThrowableProxy; import ch.qos.logback.classic.spi.ThrowableProxyUtil; +import com.fasterxml.jackson.core.type.TypeReference; import com.stumbleupon.async.Deferred; import net.opentsdb.core.DataPoints; @@ -41,6 +44,7 @@ import net.opentsdb.tsd.AnnotationRpc.AnnotationBulkDelete; import net.opentsdb.tsd.QueryRpc.LastPointQuery; import net.opentsdb.utils.Config; +import net.opentsdb.utils.JSONException; /** * Abstract base class for Serializers; plugins that handle converting requests @@ -173,6 +177,23 @@ public List<IncomingDataPoint> parsePutV1() { " has not implemented parsePutV1"); } + /** + * Parses one or more data points for storage + * @param <T> The type of incoming data points to parse. + * @param type The type of the class to parse. + * @param typeReference The reference to use for parsing. + * @return an array of data points to process for storage + * @throws BadRequestException if the plugin has not implemented this method + * @since 2.4 + */ + public <T extends IncomingDataPoint> List<T> parsePutV1(final Class<T> type, + final TypeReference<ArrayList<T>> typeReference) { + throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, + "The requested API endpoint has not been implemented", + this.getClass().getCanonicalName() + + " has not implemented parsePutV1"); + } + /** * Parses a suggestion query * @return a hash map of key/value pairs @@ -197,6 +218,18 @@ public HashMap<String, List<String>> parseUidAssignV1() { " has not implemented parseUidAssignV1"); } + /** + * Parses metrics, tagk or tagvs type and name to rename UID + * @return as hash map of type and name + * @throws BadRequestException if the plugin has not implemented this method + */ + public HashMap<String, String> parseUidRenameV1() { + throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, + "The requested API endpoint has not been implemented", + this.getClass().getCanonicalName() + + " has not implemented parseUidRenameV1"); + } + /** * Parses a SearchQuery request * @return The parsed search query @@ -297,7 +330,7 @@ public List<TreeRule> parseTreeRulesV1() { * Parses a tree ID and optional list of TSUIDs to search for collisions or * not matched TSUIDs. * @return A map with "treeId" as an integer and optionally "tsuids" as a - * List<String> + * List<String> * @throws BadRequestException if the plugin has not implemented this method */ public Map<String, Object> parseTreeTSUIDsListV1() { @@ -348,7 +381,7 @@ public AnnotationBulkDelete parseAnnotationBulkDeleteV1() { * @param results A map of results. The map will consist of: * <ul><li>success - (long) the number of successfully parsed datapoints</li> * <li>failed - (long) the number of datapoint parsing failures</li> - * <li>errors - (ArrayList<HashMap<String, Object>>) an optional list of + * <li>errors - (ArrayList<HashMap<String, Object>>) an optional list of * datapoints that had errors. The nested map has these fields: * <ul><li>error - (String) the error that occurred</li> * <li>datapoint - (IncomingDatapoint) the datapoint that generated the error @@ -442,6 +475,19 @@ public ChannelBuffer formatUidAssignV1(final " has not implemented formatUidAssignV1"); } + /** + * Format a response from the Uid Rename RPC + * @param response A map of result and reason for error of the rename + * @return A ChannelBuffer object to pass on to the caller + * @throws BadRequestException if the plugin has not implemented this method + */ + public ChannelBuffer formatUidRenameV1(final Map<String, String> response) { + throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, + "The requested API endpoint has not been implemented", + this.getClass().getCanonicalName() + + " has not implemented formatUidRenameV1"); + } + /** * Format the results from a timeseries data query * @param query The TSQuery object used to fetch the results @@ -458,6 +504,24 @@ public ChannelBuffer formatQueryV1(final TSQuery query, " has not implemented formatQueryV1"); } + /** + * Format the results from a timeseries data query + * @param query The TSQuery object used to fetch the results + * @param results The data fetched from storage + * @param globals An optional list of global annotation objects + * @return A ChannelBuffer object to pass on to the caller + * @throws BadRequestException if the plugin has not implemented this method + * @since 2.2 + */ + public Deferred<ChannelBuffer> formatQueryAsyncV1(final TSQuery query, + final List<DataPoints[]> results, final List<Annotation> globals) + throws IOException { + throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, + "The requested API endpoint has not been implemented", + this.getClass().getCanonicalName() + + " has not implemented formatQueryV1"); + } + /** * Format a list of last data points * @param data_points The results of the query @@ -500,7 +564,7 @@ public ChannelBuffer formatTSMetaV1(final TSMeta meta) { /** * Format a a list of TSMeta objects - * @param meta The list of TSMeta objects to serialize + * @param metas The list of TSMeta objects to serialize * @return A JSON structure * @throws JSONException if serialization failed */ @@ -587,7 +651,7 @@ public ChannelBuffer formatTreeCollisionNotMatchedV1( * @param results The list of results. Main map key is the tsuid. Child map: * "branch" : Parsed branch result, may be null * "meta" : TSMeta object, may be null - * "messages" : An ArrayList<String> of one or more messages + * "messages" : An ArrayList<String> of one or more messages * @return A ChannelBuffer object to pass on to the caller * @throws BadRequestException if the plugin has not implemented this method */ @@ -627,7 +691,7 @@ public ChannelBuffer formatAnnotationsV1(final List<Annotation> notes) { /** * Format the results of a bulk annotation deletion - * @param notes The annotation deletion request to return + * @param request The request to handle. * @return A ChannelBuffer object to pass on to the caller * @throws BadRequestException if the plugin has not implemented this method */ @@ -652,6 +716,62 @@ public ChannelBuffer formatStatsV1(final List<IncomingDataPoint> stats) { " has not implemented formatStatsV1"); } + /** + * Format a list of thread statistics + * @param stats The thread statistics list to format + * @return A ChannelBuffer object to pass on to the caller + * @throws BadRequestException if the plugin has not implemented this method + * @since 2.2 + */ + public ChannelBuffer formatThreadStatsV1(final List<Map<String, Object>> stats) { + throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, + "The requested API endpoint has not been implemented", + this.getClass().getCanonicalName() + + " has not implemented formatThreadStatsV1"); + } + + /** + * format a list of region client statistics + * @param stats The list of region client stats to format + * @return A ChannelBuffer object to pass on to the caller + * @throws BadRequestException if the plugin has not implemented this method + * @since 2.2 + */ + public ChannelBuffer formatRegionStatsV1(final List<Map<String, Object>> stats) { + throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, + "The requested API endpoint has not been implemented", + this.getClass().getCanonicalName() + + " has not implemented formatRegionStatsV1"); + } + + /** + * Format a list of JVM statistics + * @param map The JVM stats list to format + * @return A ChannelBuffer object to pass on to the caller + * @throws BadRequestException if the plugin has not implemented this method + * @since 2.2 + */ + public ChannelBuffer formatJVMStatsV1(final Map<String, Map<String, Object>> map) { + throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, + "The requested API endpoint has not been implemented", + this.getClass().getCanonicalName() + + " has not implemented formatJVMStatsV1"); + } + + /** + * Format the query stats + * @param query_stats Map of query statistics + * @return A ChannelBuffer object to pass on to the caller + * @throws BadRequestException if the plugin has not implemented this method + * @since 2.2 + */ + public ChannelBuffer formatQueryStatsV1(final Map<String, Object> query_stats) { + throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, + "The requested API endpoint has not been implemented", + this.getClass().getCanonicalName() + + " has not implemented formatQueryStatsV1"); + } + /** * Format the response from a search query * @param results The query (hopefully filled with results) to serialize @@ -678,6 +798,20 @@ public ChannelBuffer formatConfigV1(final Config config) { " has not implemented formatConfigV1"); } + /** + * Format the loaded filter configurations + * @param config The filters to serialize + * @return A ChannelBuffer object to pass on to the caller + * @throws BadRequestException if the plugin has not implemented this method + */ + public ChannelBuffer formatFilterConfigV1( + final Map<String, Map<String, String>> config) { + throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, + "The requested API endpoint has not been implemented", + this.getClass().getCanonicalName() + + " has not implemented formatFilterConfigV1"); + } + /** * Formats a 404 error when an endpoint or file wasn't found * <p> diff --git a/src/tsd/LogsRpc.java b/src/tsd/LogsRpc.java index fab9581415..da8afa8cd1 100644 --- a/src/tsd/LogsRpc.java +++ b/src/tsd/LogsRpc.java @@ -12,6 +12,7 @@ // see <http://www.gnu.org/licenses/>. package net.opentsdb.tsd; +import org.jboss.netty.handler.codec.http.HttpMethod; import org.slf4j.LoggerFactory; import com.fasterxml.jackson.core.JsonGenerationException; @@ -35,8 +36,13 @@ final class LogsRpc implements HttpRpc { public void execute(final TSDB tsdb, final HttpQuery query) - throws JsonGenerationException, IOException { + throws BadRequestException, IOException { + + // only accept GET/POST + RpcUtil.allowedMethods(query.method(), HttpMethod.GET.getName(), HttpMethod.POST.getName()); + LogIterator logmsgs = new LogIterator(); + if (query.hasQueryStringParam("json")) { ArrayList<String> logs = new ArrayList<String>(); for (String log : logmsgs) { @@ -93,6 +99,11 @@ public LogIterator() { final Logger root = (Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME); logbuf = (CyclicBufferAppender<ILoggingEvent>) root.getAppender("CYCLIC"); + if (logbuf == null) { + throw new BadRequestException( + "No CyclicBufferAppender found. Please configure logback " + + "to store the latest log entries."); + } } public Iterator<String> iterator() { diff --git a/src/tsd/PipelineFactory.java b/src/tsd/PipelineFactory.java index a9b4a26f71..5e9544d1c4 100644 --- a/src/tsd/PipelineFactory.java +++ b/src/tsd/PipelineFactory.java @@ -15,7 +15,7 @@ import static org.jboss.netty.channel.Channels.pipeline; import java.util.concurrent.ThreadFactory; - +import net.opentsdb.auth.AuthenticationChannelHandler; import org.jboss.netty.buffer.ChannelBuffer; import org.jboss.netty.channel.Channel; import org.jboss.netty.channel.ChannelHandler; @@ -30,7 +30,6 @@ import org.jboss.netty.handler.codec.http.HttpRequestDecoder; import org.jboss.netty.handler.codec.http.HttpResponseEncoder; import org.jboss.netty.handler.timeout.IdleStateHandler; -import org.jboss.netty.util.HashedWheelTimer; import org.jboss.netty.util.Timer; import net.opentsdb.core.TSDB; @@ -38,6 +37,9 @@ /** * Creates a newly configured {@link ChannelPipeline} for a new channel. * This class is supposed to be a singleton. + * NOTE: On creation (as of 2.3) the property given in the config for + * "tsd.core.connections.limit" will be used to limit the number of concurrent + * connections supported by the pipeline. The default is zero. */ public final class PipelineFactory implements ChannelPipelineFactory { @@ -47,9 +49,9 @@ public final class PipelineFactory implements ChannelPipelineFactory { // Those are sharable but maintain some state, so a single instance per // PipelineFactory is needed. - private final ConnectionManager connmgr = new ConnectionManager(); + private final ConnectionManager connmgr; private final DetectHttpOrRpc HTTP_OR_RPC = new DetectHttpOrRpc(); - private final Timer timer = new HashedWheelTimer(new PipelineThreadFactory()); + private final Timer timer; private final ChannelHandler timeoutHandler; /** Stateless handler for RPCs. */ @@ -60,20 +62,52 @@ public final class PipelineFactory implements ChannelPipelineFactory { /** The server side socket timeout. **/ private final int socketTimeout; - + /** * Constructor that initializes the RPC router and loads HTTP formatter - * plugins + * plugins. This constructor creates its own {@link RpcManager}. * @param tsdb The TSDB to use. * @throws RuntimeException if there is an issue loading plugins - * @throws Exception if the HttpQuery handler is unable to load + * @throws RuntimeException if the HttpQuery handler is unable to load * serializers */ public PipelineFactory(final TSDB tsdb) { + this(tsdb, RpcManager.instance(tsdb), + tsdb.getConfig().getInt("tsd.core.connections.limit")); + } + + /** + * Constructor that initializes the RPC router and loads HTTP formatter + * plugins using an already-configured {@link RpcManager}. + * @param tsdb The TSDB to use. + * @param manager instance of a ready-to-use {@link RpcManager}. + * @throws RuntimeException if there is an issue loading plugins + * throws Exception if the HttpQuery handler is unable to load serializers + */ + public PipelineFactory(final TSDB tsdb, final RpcManager manager) { + this(tsdb, RpcManager.instance(tsdb), + tsdb.getConfig().getInt("tsd.core.connections.limit")); + } + + /** + * Constructor that initializes the RPC router and loads HTTP formatter + * plugins using an already-configured {@link RpcManager}. + * @param tsdb The TSDB to use. + * @param manager instance of a ready-to-use {@link RpcManager}. + * @param connections_limit The maximum number of concurrent connections + * supported by the TSD. + * @throws RuntimeException if there is an issue loading plugins + * throws Exception if the HttpQuery handler is unable to load serializers + * @since 2.3 + */ + public PipelineFactory(final TSDB tsdb, final RpcManager manager, + final int connections_limit) { this.tsdb = tsdb; - this.socketTimeout = tsdb.getConfig().getInt("tsd.core.socket.timeout"); - this.timeoutHandler = new IdleStateHandler(this.timer, 0, 0, this.socketTimeout); - this.rpchandler = new RpcHandler(tsdb); + socketTimeout = tsdb.getConfig().getInt("tsd.core.socket.timeout"); + timer = tsdb.getTimer(); + timeoutHandler = new IdleStateHandler(timer, 0, 0, socketTimeout); + rpchandler = new RpcHandler(tsdb, manager); + connmgr = new ConnectionManager(connections_limit); try { HttpQuery.initializeSerializerMaps(tsdb); } catch (RuntimeException e) { @@ -128,6 +162,10 @@ protected Object decode(final ChannelHandlerContext ctx, pipeline.addLast("decoder", DECODER); } + if (tsdb.getAuth() != null) { + pipeline.addLast("authentication", new AuthenticationChannelHandler(tsdb)); + } + pipeline.addLast("timeout", timeoutHandler); pipeline.remove(this); pipeline.addLast("handler", rpchandler); @@ -137,22 +175,5 @@ protected Object decode(final ChannelHandlerContext ctx, } } - - /** - * A class to generate a daemon thread for the idle connection timer. - */ - class PipelineThreadFactory implements ThreadFactory { - @Override - public Thread newThread(final Runnable r) { - final Thread t = new Thread(r, "PipelineFactoryTimer"); - t.setDaemon(true); - return t; - } - - @Override - public String toString() { - return "Pipeline timer thread factory"; - } - } + } - \ No newline at end of file diff --git a/src/tsd/PutDataPointRpc.java b/src/tsd/PutDataPointRpc.java index 4e7f786ce0..039d9f4e30 100644 --- a/src/tsd/PutDataPointRpc.java +++ b/src/tsd/PutDataPointRpc.java @@ -1,5 +1,5 @@ // This file is part of OpenTSDB. -// Copyright (C) 2010-2012 The OpenTSDB Authors. +// Copyright (C) 2010-2015 The OpenTSDB Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU Lesser General Public License as published by @@ -16,64 +16,244 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import com.google.common.base.Strings; import com.stumbleupon.async.Callback; import com.stumbleupon.async.Deferred; +import com.stumbleupon.async.TimeoutException; +import net.opentsdb.auth.AuthState; +import net.opentsdb.auth.Authentication; +import net.opentsdb.auth.Authorization; +import net.opentsdb.auth.Permissions; +import org.apache.zookeeper.KeeperException; +import org.hbase.async.HBaseException; +import org.hbase.async.PleaseThrottleException; import org.jboss.netty.channel.Channel; import org.jboss.netty.handler.codec.http.HttpMethod; import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.jboss.netty.util.Timeout; +import org.jboss.netty.util.TimerTask; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import net.opentsdb.core.Histogram; +import net.opentsdb.core.HistogramPojo; import net.opentsdb.core.IncomingDataPoint; import net.opentsdb.core.TSDB; import net.opentsdb.core.Tags; +import net.opentsdb.rollup.NoSuchRollupForIntervalException; +import net.opentsdb.rollup.RollUpDataPoint; import net.opentsdb.stats.StatsCollector; import net.opentsdb.uid.NoSuchUniqueName; +import net.opentsdb.utils.Config; -/** Implements the "put" telnet-style command. */ -final class PutDataPointRpc implements TelnetRpc, HttpRpc { - private static final Logger LOG = LoggerFactory.getLogger(PutDataPointRpc.class); - private static final AtomicLong requests = new AtomicLong(); - private static final AtomicLong hbase_errors = new AtomicLong(); - private static final AtomicLong invalid_values = new AtomicLong(); - private static final AtomicLong illegal_arguments = new AtomicLong(); - private static final AtomicLong unknown_metrics = new AtomicLong(); - +/** + * This class handles the parsing and writing of data points over any of the + * implemented RPCs. Each put method should track the deferred of the + * {@code tsdb.addPoint()} method so that we know whether or not the write was + * actually successful. Note that if HBase is backed up, then the arguments + * will hang around until the deferred is called back. This could take many + * seconds, during which time the heap will keep growing. + * <p> + * Each execute method also checks the tsd's {@code StorageExceptionHandler} + * object on failure so that we can do something else with the data point such + * as spool it to disk or send it back to an external queue. We do that in the + * error callback here instead of in the TSDB {@code addPoint()} because we + * want to avoid adding yet another callback and chewing up more heap + * unnecessarily. + * <p> + * Note that this class can be subclassed to handle different types of + * data points such as Rollups or Pre-Aggregates + */ +class PutDataPointRpc implements TelnetRpc, HttpRpc { + protected static final Logger LOG = LoggerFactory.getLogger(PutDataPointRpc.class); + protected static final ArrayList<Boolean> EMPTY_DEFERREDS = + new ArrayList<Boolean>(0); + protected static final AtomicLong telnet_requests = new AtomicLong(); + protected static final AtomicLong telnet_requests_unauthorized = new AtomicLong(); + protected static final AtomicLong telnet_requests_forbidden = new AtomicLong(); + protected static final AtomicLong http_requests = new AtomicLong(); + protected static final AtomicLong http_requests_unauthorized = new AtomicLong(); + protected static final AtomicLong http_requests_forbidden = new AtomicLong(); + protected static final AtomicLong raw_dps = new AtomicLong(); + protected static final AtomicLong raw_histograms = new AtomicLong(); + protected static final AtomicLong rollup_dps = new AtomicLong(); + protected static final AtomicLong raw_stored = new AtomicLong(); + protected static final AtomicLong raw_histograms_stored = new AtomicLong(); + protected static final AtomicLong rollup_stored = new AtomicLong(); + protected static final AtomicLong hbase_errors = new AtomicLong(); + protected static final AtomicLong unknown_errors = new AtomicLong(); + protected static final AtomicLong invalid_values = new AtomicLong(); + protected static final AtomicLong illegal_arguments = new AtomicLong(); + protected static final AtomicLong unknown_metrics = new AtomicLong(); + protected static final AtomicLong inflight_exceeded = new AtomicLong(); + protected static final AtomicLong writes_blocked = new AtomicLong(); + protected static final AtomicLong writes_timedout = new AtomicLong(); + protected static final AtomicLong requests_timedout = new AtomicLong(); + + /** Whether or not to send error messages back over telnet */ + private final boolean send_telnet_errors; + + /** The type of data point we're writing. + * @since 2.4 */ + public enum DataPointType { + PUT("put"), + ROLLUP("rollup"), + HISTOGRAM("histogram"); + + private final String name; + DataPointType(final String name) { + this.name = name; + } + + @Override + public String toString() { + return name; + } + } + + /** + * Default Ctor + * @param config The TSDB config to pull from + */ + public PutDataPointRpc(final Config config) { + send_telnet_errors = config.getBoolean("tsd.rpc.telnet.return_errors"); + } + + @Override public Deferred<Object> execute(final TSDB tsdb, final Channel chan, final String[] cmd) { - requests.incrementAndGet(); + telnet_requests.incrementAndGet(); + final DataPointType type; + final String command = cmd[0].toLowerCase(); + + if (command.equals("put")) { + type = DataPointType.PUT; + raw_dps.incrementAndGet(); + } else if (command.equals("rollup")) { + type = DataPointType.ROLLUP; + rollup_dps.incrementAndGet(); + } else if (command.equals("histogram")) { + type = DataPointType.HISTOGRAM; + raw_histograms.incrementAndGet(); + } else { + throw new IllegalArgumentException("Unrecognized command: " + cmd[0]); + } + String errmsg = null; try { - final class PutErrback implements Callback<Exception, Exception> { - public Exception call(final Exception arg) { - if (chan.isConnected()) { - chan.write("put: HBase error: " + arg.getMessage() + '\n'); + + checkAuthorization(tsdb, chan, command); + + /** + * Error callback that handles passing a data point to the storage + * exception handler as well as responding to the client when HBase + * is unable to write the data. + */ + final class PutErrback implements Callback<Object, Exception> { + @Override + public Object call(final Exception arg) { + String errmsg = null; + if (arg instanceof PleaseThrottleException) { + if (send_telnet_errors) { + errmsg = type + ": Please throttle writes: " + arg.getMessage() + '\n'; + } + inflight_exceeded.incrementAndGet(); + } else { + if (send_telnet_errors) { + errmsg = type + ": HBase error: " + arg.getMessage()+ '\n'; + } + if (arg instanceof HBaseException) { + hbase_errors.incrementAndGet(); + } else if (arg instanceof IllegalArgumentException) { + illegal_arguments.incrementAndGet(); + } else { + unknown_errors.incrementAndGet(); + } } - hbase_errors.incrementAndGet(); - return arg; + + // we handle the storage exceptions here so as to avoid creating yet + // another callback object on every data point. + handleStorageException(tsdb, getDataPointFromString(tsdb, cmd), arg); + + if (send_telnet_errors) { + if (chan.isConnected()) { + if (chan.isWritable()) { + chan.write(errmsg); + } else { + writes_blocked.incrementAndGet(); + } + } + } + + return null; } public String toString() { return "report error to channel"; } } - return importDataPoint(tsdb, cmd).addErrback(new PutErrback()); + + /** + * Simply called to increment the success counter and log hearbeats + */ + final class SuccessCB implements Callback<Object, Object> { + @Override + public Object call(final Object obj) { + if (type == DataPointType.PUT) { + raw_stored.incrementAndGet(); + } else if (type == DataPointType.ROLLUP) { + rollup_stored.incrementAndGet(); + } else if (type == DataPointType.HISTOGRAM) { + raw_histograms_stored.incrementAndGet(); + } + return true; + } + } + + // Rollups and histos override this method in their implementation so + // that it will route properly. + return importDataPoint(tsdb, cmd) + .addCallback(new SuccessCB()) + .addErrback(new PutErrback()); } catch (NumberFormatException x) { - errmsg = "put: invalid value: " + x.getMessage() + '\n'; + x.printStackTrace(); + errmsg = type + ": invalid value: " + x.getMessage() + '\n'; invalid_values.incrementAndGet(); + } catch (NoSuchRollupForIntervalException x) { + errmsg = type + ": No such rollup: " + x.getMessage() + '\n'; + illegal_arguments.incrementAndGet(); } catch (IllegalArgumentException x) { - errmsg = "put: illegal argument: " + x.getMessage() + '\n'; + errmsg = type + ": illegal argument: " + x.getMessage() + '\n'; + x.printStackTrace(); illegal_arguments.incrementAndGet(); } catch (NoSuchUniqueName x) { - errmsg = "put: unknown metric: " + x.getMessage() + '\n'; + errmsg = type + ": unknown metric: " + x.getMessage() + '\n'; unknown_metrics.incrementAndGet(); + /*} catch (NoSuchUniqueNameInCache x) { + errmsg = "put: waiting for cache: " + x.getMessage() + '\n'; + handleStorageException(tsdb, getDataPointFromString(cmd), x); */ + } catch (PleaseThrottleException x) { + errmsg = type + ": Throttling exception: " + x.getMessage() + '\n'; + inflight_exceeded.incrementAndGet(); + handleStorageException(tsdb, getDataPointFromString(tsdb, cmd), x); + } catch (TimeoutException tex) { + errmsg = type + ": Request timed out: " + tex.getMessage() + '\n'; + handleStorageException(tsdb, getDataPointFromString(tsdb, cmd), tex); + } catch (RuntimeException rex) { + errmsg = type + ": Unexpected runtime exception: " + rex.getMessage() + '\n'; + throw rex; } - if (errmsg != null) { - LOG.debug(errmsg); - if (chan.isConnected()) { + + if (errmsg != null && chan.isConnected()) { + if (chan.isWritable()) { chan.write(errmsg); + } else { + writes_blocked.incrementAndGet(); } } return Deferred.fromResult(null); @@ -88,114 +268,448 @@ public String toString() { * @throws BadRequestException if the user supplied bad data * @since 2.0 */ + @Override public void execute(final TSDB tsdb, final HttpQuery query) throws IOException { - requests.incrementAndGet(); + http_requests.incrementAndGet(); // only accept POST - if (query.method() != HttpMethod.POST) { - throw new BadRequestException(HttpResponseStatus.METHOD_NOT_ALLOWED, - "Method not allowed", "The HTTP method [" + query.method().getName() + - "] is not permitted for this endpoint"); + RpcUtil.allowedMethods(query.method(), HttpMethod.POST.getName()); + + final List<IncomingDataPoint> dps; + //noinspection TryWithIdenticalCatches + try { + checkAuthorization(tsdb, query); + dps = query.serializer() + .parsePutV1(IncomingDataPoint.class, HttpJsonSerializer.TR_INCOMING); + } catch (BadRequestException e) { + illegal_arguments.incrementAndGet(); + throw e; + } catch (IllegalArgumentException e) { + illegal_arguments.incrementAndGet(); + throw e; } - - final List<IncomingDataPoint> dps = query.serializer().parsePutV1(); + processDataPoint(tsdb, query, dps); + } + + /** + * Handles one or more incoming data point types for the HTTP endpoint + * to put raw, rolled up or aggregated data points + * @param <T> An {@link IncomingDataPoint} class. + * @param tsdb The TSDB to which we belong + * @param query The query to respond to + * @param dps The de-serialized data points + * @throws BadRequestException if the data is invalid in some way + * @since 2.4 + */ + public <T extends IncomingDataPoint> void processDataPoint(final TSDB tsdb, + final HttpQuery query, final List<T> dps) { if (dps.size() < 1) { throw new BadRequestException("No datapoints found in content"); } - + + final HashMap<String, String> query_tags = new HashMap<String, String>(); final boolean show_details = query.hasQueryStringParam("details"); final boolean show_summary = query.hasQueryStringParam("summary"); - final ArrayList<HashMap<String, Object>> details = show_details - ? new ArrayList<HashMap<String, Object>>() : null; - long success = 0; - long total = 0; + final boolean synchronous = query.hasQueryStringParam("sync"); + final int sync_timeout = query.hasQueryStringParam("sync_timeout") ? + Integer.parseInt(query.getQueryStringParam("sync_timeout")) : 0; + // this is used to coordinate timeouts + final AtomicBoolean sending_response = new AtomicBoolean(); + sending_response.set(false); - for (IncomingDataPoint dp : dps) { - total++; - try { - if (dp.getMetric() == null || dp.getMetric().isEmpty()) { + final List<Map<String, Object>> details = show_details + ? new ArrayList<Map<String, Object>>() : null; + int queued = 0; + final List<Deferred<Boolean>> deferreds = synchronous ? + new ArrayList<Deferred<Boolean>>(dps.size()) : null; + + if (tsdb.getConfig().enable_header_tag()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Looking for tag header " + + tsdb.getConfig().get_name_header_tag()); + } + final String header_tag_value = query.getHeaderValue( + tsdb.getConfig().get_name_header_tag()) ; + if (header_tag_value != null) { + if (LOG.isDebugEnabled()) { + LOG.debug(" header found with value:" + header_tag_value); + } + Tags.parse(query_tags, header_tag_value); + } else if (LOG.isDebugEnabled()) { + LOG.debug(" no such header in request"); + } + } + + for (final IncomingDataPoint dp : dps) { + final DataPointType type; + if (dp instanceof RollUpDataPoint) { + type = DataPointType.ROLLUP; + rollup_dps.incrementAndGet(); + } else if (dp instanceof HistogramPojo) { + type = DataPointType.HISTOGRAM; + raw_histograms.incrementAndGet(); + } else { + type = DataPointType.PUT; + raw_dps.incrementAndGet(); + } + + /* + Error back callback to handle storage failures + */ + final class PutErrback implements Callback<Boolean, Exception> { + public Boolean call(final Exception arg) { + if (arg instanceof PleaseThrottleException) { + inflight_exceeded.incrementAndGet(); + } else { + hbase_errors.incrementAndGet(); + } + if (show_details) { - details.add(this.getHttpDetails("Metric name was empty", dp)); + details.add(getHttpDetails("Storage exception: " + + arg.getMessage(), dp)); } - LOG.warn("Metric name was empty: " + dp); - continue; + + // we handle the storage exceptions here so as to avoid creating yet + // another callback object on every data point. + handleStorageException(tsdb, dp, arg); + return false; } - if (dp.getTimestamp() <= 0) { - if (show_details) { - details.add(this.getHttpDetails("Invalid timestamp", dp)); + public String toString() { + return "HTTP Put exception"; + } + } + + final class SuccessCB implements Callback<Boolean, Object> { + @Override + public Boolean call(final Object obj) { + switch (type) { + case PUT: + raw_stored.incrementAndGet(); + break; + case ROLLUP: + rollup_stored.incrementAndGet(); + break; + case HISTOGRAM: + raw_histograms_stored.incrementAndGet(); + break; + default: + // don't care } - LOG.warn("Invalid timestamp: " + dp); - continue; + return true; } - if (dp.getValue() == null || dp.getValue().isEmpty()) { + } + + try { + if (dp == null) { if (show_details) { - details.add(this.getHttpDetails("Empty value", dp)); + details.add(this.getHttpDetails("Unexpected null datapoint encountered in set.", dp)); } - LOG.warn("Empty value: " + dp); + LOG.warn("Datapoint null was encountered in set."); + illegal_arguments.incrementAndGet(); continue; } - if (dp.getTags() == null || dp.getTags().size() < 1) { - if (show_details) { - details.add(this.getHttpDetails("Missing tags", dp)); - } - LOG.warn("Missing tags: " + dp); + + if (!dp.validate(details)) { + illegal_arguments.incrementAndGet(); continue; } - if (Tags.looksLikeInteger(dp.getValue())) { - tsdb.addPoint(dp.getMetric(), dp.getTimestamp(), - Tags.parseLong(dp.getValue()), dp.getTags()); + + // TODO - refactor the add calls someday or move some of this into the + // actual data point class. + final Deferred<Boolean> deferred; + if (type == DataPointType.HISTOGRAM) { + final HistogramPojo pojo = (HistogramPojo) dp; + // validation and/or conversion before storage of histograms by + // decoding then re-encoding. + final Histogram hdp; + if (Strings.isNullOrEmpty(dp.getValue())) { + hdp = pojo.toSimpleHistogram(tsdb); + } else { + hdp = tsdb.histogramManager().decode( + pojo.getId(), pojo.getBytes(), false); + } + deferred = tsdb.addHistogramPoint( + pojo.getMetric(), + pojo.getTimestamp(), + tsdb.histogramManager().encode(hdp.getId(), hdp, true), + pojo.getTags()) + .addCallback(new SuccessCB()) + .addErrback(new PutErrback()); } else { - tsdb.addPoint(dp.getMetric(), dp.getTimestamp(), - Float.parseFloat(dp.getValue()), dp.getTags()); + if (Tags.looksLikeInteger(dp.getValue())) { + switch (type) { + case ROLLUP: + { + final RollUpDataPoint rdp = (RollUpDataPoint)dp; + deferred = tsdb.addAggregatePoint(rdp.getMetric(), + rdp.getTimestamp(), + Tags.parseLong(rdp.getValue()), + dp.getTags(), + rdp.getGroupByAggregator() != null, + rdp.getInterval(), + rdp.getAggregator(), + rdp.getGroupByAggregator()) + .addCallback(new SuccessCB()) + .addErrback(new PutErrback()); + break; + } + default: + deferred = tsdb.addPoint(dp.getMetric(), dp.getTimestamp(), + Tags.parseLong(dp.getValue()), dp.getTags()) + .addCallback(new SuccessCB()) + .addErrback(new PutErrback()); + } + } else { + switch (type) { + case ROLLUP: + { + final RollUpDataPoint rdp = (RollUpDataPoint)dp; + deferred = tsdb.addAggregatePoint(rdp.getMetric(), + rdp.getTimestamp(), + (Tags.fitsInFloat(dp.getValue()) ? + Float.parseFloat(dp.getValue()) : + Double.parseDouble(dp.getValue())), + dp.getTags(), + rdp.getGroupByAggregator() != null, + rdp.getInterval(), + rdp.getAggregator(), + rdp.getGroupByAggregator()) + .addCallback(new SuccessCB()) + .addErrback(new PutErrback()); + break; + } + default: + deferred = tsdb.addPoint(dp.getMetric(), dp.getTimestamp(), + (Tags.fitsInFloat(dp.getValue()) ? + Float.parseFloat(dp.getValue()) : + Double.parseDouble(dp.getValue())), + dp.getTags()) + .addCallback(new SuccessCB()) + .addErrback(new PutErrback()); + } + } + } + ++queued; + if (synchronous) { + deferreds.add(deferred); } - success++; + } catch (NumberFormatException x) { if (show_details) { - details.add(this.getHttpDetails("Unable to parse value to a number", + details.add(getHttpDetails("Unable to parse value to a number", dp)); } LOG.warn("Unable to parse value to a number: " + dp); invalid_values.incrementAndGet(); } catch (IllegalArgumentException iae) { if (show_details) { - details.add(this.getHttpDetails(iae.getMessage(), dp)); + details.add(getHttpDetails(iae.getMessage(), dp)); } LOG.warn(iae.getMessage() + ": " + dp); illegal_arguments.incrementAndGet(); } catch (NoSuchUniqueName nsu) { if (show_details) { - details.add(this.getHttpDetails("Unknown metric", dp)); + details.add(getHttpDetails("Unknown metric", dp)); } LOG.warn("Unknown metric: " + dp); unknown_metrics.incrementAndGet(); + } catch (PleaseThrottleException x) { + handleStorageException(tsdb, dp, x); + if (show_details) { + details.add(getHttpDetails("Please throttle", dp)); + } + inflight_exceeded.incrementAndGet(); + } catch (TimeoutException tex) { + handleStorageException(tsdb, dp, tex); + if (show_details) { + details.add(getHttpDetails("Timeout exception", dp)); + } + requests_timedout.incrementAndGet(); + /*} catch (NoSuchUniqueNameInCache x) { + handleStorageException(tsdb, dp, x); + if (show_details) { + details.add(getHttpDetails("Not cached yet", dp)); + } */ + } catch (RuntimeException e) { + if (show_details) { + details.add(getHttpDetails("Unexpected exception", dp)); + } + LOG.warn("Unexpected exception: " + dp, e); + unknown_errors.incrementAndGet(); + } + } + + /** A timer task that will respond to the user with the number of timeouts + * for synchronous writes. */ + class PutTimeout implements TimerTask { + final int queued; + public PutTimeout(final int queued) { + this.queued = queued; + } + @Override + public void run(final Timeout timeout) throws Exception { + if (sending_response.get()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Put data point call " + query + + " already responded successfully"); + } + return; + } else { + sending_response.set(true); + } + + // figure out how many writes are outstanding + int good_writes = 0; + int failed_writes = 0; + int timeouts = 0; + for (int i = 0; i < deferreds.size(); i++) { + try { + if (deferreds.get(i).join(1)) { + ++good_writes; + } else { + ++failed_writes; + } + } catch (TimeoutException te) { + if (show_details) { + details.add(getHttpDetails("Write timedout", dps.get(i))); + } + ++timeouts; + } + } + writes_timedout.addAndGet(timeouts); + final int failures = dps.size() - queued; + if (!show_summary && !show_details) { + query.sendReply(HttpResponseStatus.BAD_REQUEST, query.serializer().formatErrorV1( + new BadRequestException(HttpResponseStatus.BAD_REQUEST, + "The put call has timedout with " + good_writes + " successful writes, " + + failed_writes + " failed writes and " + timeouts + " timed out writes.", + "Please see the TSD logs or append \"details\" to the put request"))); + } else { + final HashMap<String, Object> summary = new HashMap<String, Object>(); + summary.put("success", good_writes); + summary.put("failed", failures + failed_writes); + summary.put("timeouts", timeouts); + if (show_details) { + summary.put("errors", details); + } + + query.sendReply(HttpResponseStatus.BAD_REQUEST, + query.serializer().formatPutV1(summary)); + } } } - final long failures = total - success; - if (!show_summary && !show_details) { - if (failures > 0) { - throw new BadRequestException(HttpResponseStatus.BAD_REQUEST, - "One or more data points had errors", - "Please see the TSD logs or append \"details\" to the put request"); - } else { - query.sendReply(HttpResponseStatus.NO_CONTENT, "".getBytes()); + // now after everything has been sent we can schedule a timeout if so + // the caller asked for a synchronous write. + final Timeout timeout = sync_timeout > 0 ? + tsdb.getTimer().newTimeout(new PutTimeout(queued), sync_timeout, + TimeUnit.MILLISECONDS) : null; + + /** Serializes the response to the client */ + class GroupCB implements Callback<Object, ArrayList<Boolean>> { + final int queued; + public GroupCB(final int queued) { + this.queued = queued; } - } else { - final HashMap<String, Object> summary = new HashMap<String, Object>(); - summary.put("success", success); - summary.put("failed", failures); - if (show_details) { - summary.put("errors", details); + + @Override + public Object call(final ArrayList<Boolean> results) { + tsdb.response(new Runnable() { + @Override + public void run() { + if (sending_response.get()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Put data point call " + query + " was marked as timedout"); + } + return; + } else { + sending_response.set(true); + if (timeout != null) { + timeout.cancel(); + } + } + int good_writes = 0; + int failed_writes = 0; + for (final boolean result : results) { + if (result) { + ++good_writes; + } else { + ++failed_writes; + } + } + + final int failures = dps.size() - queued; + if (!show_summary && !show_details) { + if (failures + failed_writes > 0) { + query.sendReply(HttpResponseStatus.BAD_REQUEST, + query.serializer().formatErrorV1( + new BadRequestException(HttpResponseStatus.BAD_REQUEST, + "One or more data points had errors", + "Please see the TSD logs or append \"details\" to the put request"))); + } else { + query.sendReply(HttpResponseStatus.NO_CONTENT, "".getBytes()); + } + } else { + final HashMap<String, Object> summary = new HashMap<String, Object>(); + if (sync_timeout > 0) { + summary.put("timeouts", 0); + } + summary.put("success", results.isEmpty() ? queued : good_writes); + summary.put("failed", failures + failed_writes); + if (show_details) { + summary.put("errors", details); + } + + if (failures > 0) { + query.sendReply(HttpResponseStatus.BAD_REQUEST, + query.serializer().formatPutV1(summary)); + } else { + query.sendReply(query.serializer().formatPutV1(summary)); + } + } + } + }); + + return null; } - - if (failures > 0) { - query.sendReply(HttpResponseStatus.BAD_REQUEST, - query.serializer().formatPutV1(summary)); - } else { - query.sendReply(query.serializer().formatPutV1(summary)); + @Override + public String toString() { + return "put data point serialization callback"; + } + } + + /** Catches any unexpected exceptions thrown in the callback chain */ + class ErrCB implements Callback<Object, Exception> { + @Override + public Object call(final Exception e) throws Exception { + if (sending_response.get()) { + if (LOG.isDebugEnabled()) { + LOG.debug("ERROR point call " + query + " was marked as timedout", e); + } + return null; + } else { + sending_response.set(true); + if (timeout != null) { + timeout.cancel(); + } + } + LOG.error("Unexpected exception", e); + throw new RuntimeException("Unexpected exception", e); + } + @Override + public String toString() { + return "put data point error callback"; } } + + if (synchronous) { + Deferred.groupInOrder(deferreds) + .addCallback(new GroupCB(queued)) + .addErrback(new ErrCB()); + } else { + new GroupCB(queued).call(EMPTY_DEFERREDS); + } } /** @@ -203,11 +717,16 @@ public void execute(final TSDB tsdb, final HttpQuery query) * @param collector The collector to use. */ public static void collectStats(final StatsCollector collector) { - collector.record("rpc.received", requests, "type=put"); + collector.record("rpc.forbidden", telnet_requests_forbidden, "type=telnet"); + collector.record("rpc.unauthorized", telnet_requests_unauthorized, "type=telnet"); + collector.record("rpc.forbidden", http_requests_forbidden, "type=http"); + collector.record("rpc.unauthorized", http_requests_unauthorized, "type=http"); + collector.record("rpc.received", http_requests, "type=put"); collector.record("rpc.errors", hbase_errors, "type=hbase_errors"); collector.record("rpc.errors", invalid_values, "type=invalid_values"); collector.record("rpc.errors", illegal_arguments, "type=illegal_arguments"); collector.record("rpc.errors", unknown_metrics, "type=unknown_metrics"); + collector.record("rpc.errors", writes_blocked, "type=socket_writes_blocked"); } /** @@ -220,12 +739,14 @@ public static void collectStats(final StatsCollector collector) { * @throws IllegalArgumentException if any other argument is invalid. * @throws NoSuchUniqueName if the metric isn't registered. */ - private Deferred<Object> importDataPoint(final TSDB tsdb, final String[] words) { + protected Deferred<Object> importDataPoint(final TSDB tsdb, + final String[] words) { words[0] = null; // Ditch the "put". if (words.length < 5) { // Need at least: metric timestamp value tag // ^ 5 and not 4 because words[0] is "put". throw new IllegalArgumentException("not enough arguments" - + " (need least 4, got " + (words.length - 1) + ')'); + + " (need least 4, got " + + (words.length - 1) + ')'); } final String metric = words[1]; if (metric.length() <= 0) { @@ -252,11 +773,45 @@ private Deferred<Object> importDataPoint(final TSDB tsdb, final String[] words) } if (Tags.looksLikeInteger(value)) { return tsdb.addPoint(metric, timestamp, Tags.parseLong(value), tags); - } else { // floating point value + } else if (Tags.fitsInFloat(value)) { // floating point value return tsdb.addPoint(metric, timestamp, Float.parseFloat(value), tags); + } else { + return tsdb.addPoint(metric, timestamp, Double.parseDouble(value), tags); } } - + + /** + * Converts the string array to an IncomingDataPoint. WARNING: This method + * does not perform validation. It should only be used by the Telnet style + * {@code execute} above within the error callback. At that point it means + * the array parsed correctly as per {@code importDataPoint}. + * @param tsdb The TSDB for encoding/decoding. + * @param words The array of strings representing a data point + * @return An incoming data point object. + */ + protected IncomingDataPoint getDataPointFromString(final TSDB tsdb, + final String[] words) { + final IncomingDataPoint dp = new IncomingDataPoint(); + dp.setMetric(words[1]); + + if (words[2].contains(".")) { + dp.setTimestamp(Tags.parseLong(words[2].replace(".", ""))); + } else { + dp.setTimestamp(Tags.parseLong(words[2])); + } + + dp.setValue(words[3]); + + final HashMap<String, String> tags = new HashMap<String, String>(); + for (int i = 4; i < words.length; i++) { + if (!words[i].isEmpty()) { + Tags.parse(tags, words[i]); + } + } + dp.setTags(tags); + return dp; + } + /** * Simple helper to format an error trying to save a data point * @param message The message to return to the user @@ -271,4 +826,67 @@ final private HashMap<String, Object> getHttpDetails(final String message, map.put("datapoint", dp); return map; } + + /** + * Passes a data point off to the storage handler plugin if it has been + * configured. + * @param tsdb The TSDB from which to grab the SEH plugin + * @param dp The data point to process + * @param e The exception that caused this + */ + void handleStorageException(final TSDB tsdb, final IncomingDataPoint dp, + final Exception e) { + final StorageExceptionHandler handler = tsdb.getStorageExceptionHandler(); + if (handler != null) { + handler.handleError(dp, e); + } + } + + private void checkAuthorization(final TSDB tsdb, final HttpQuery query) { + if (tsdb.getConfig().getBoolean("tsd.core.authentication.enable")) { + Authentication authentication = tsdb.getAuth(); + try { + if (authentication.isReady(tsdb, query.channel())) { + AuthState authState = (AuthState) query.channel().getAttachment(); + Authorization authorization = authentication.authorization(); + if ((authorization.hasPermission(authState, Permissions.TELNET_PUT).getStatus() != AuthState.AuthStatus.SUCCESS)) { + http_requests_forbidden.incrementAndGet(); + throw new BadRequestException(HttpResponseStatus.FORBIDDEN, "Forbidden for " + query.getQueryPath()); + } + } else { + http_requests_unauthorized.incrementAndGet(); + throw new BadRequestException(HttpResponseStatus.UNAUTHORIZED, "Unauthorized for " + query.getQueryPath()); + } + } catch (BadRequestException e) { + http_requests_unauthorized.incrementAndGet(); + throw new BadRequestException(HttpResponseStatus.BAD_REQUEST, "Unable to check Authentication for" + query.getQueryPath()); + } + } + // No exceptions thrown, everything is fine. + } + + private void checkAuthorization(final TSDB tsdb, final Channel chan, final String command) { + if (tsdb.getConfig().getBoolean("tsd.core.authentication.enable")) { + Authentication authentication = tsdb.getAuth(); + try { + if (authentication == null) { + throw new IllegalStateException("Authentication is enabled but the Authentication class is NULL"); + } else if (authentication.isReady(tsdb, chan)) { + AuthState authState = (AuthState) chan.getAttachment(); + Authorization authorization = authentication.authorization(); + if ((authorization.hasPermission(authState, Permissions.TELNET_PUT).getStatus() != AuthState.AuthStatus.SUCCESS)) { + telnet_requests_forbidden.incrementAndGet(); + throw new IllegalArgumentException("Unauthorized command: " + command); + } + } else { + telnet_requests_unauthorized.incrementAndGet(); + throw new IllegalArgumentException("Unauthenticated command: " + command); + } + } catch (BadRequestException e) { + telnet_requests_unauthorized.incrementAndGet(); + throw new IllegalArgumentException("Unable to check Authentication for command: " + command); + } + } + // No exceptions thrown, everything is fine. + } } diff --git a/src/tsd/QueryExecutor.java b/src/tsd/QueryExecutor.java new file mode 100644 index 0000000000..b465ba285f --- /dev/null +++ b/src/tsd/QueryExecutor.java @@ -0,0 +1,986 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2010-2016 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.tsd; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.hbase.async.HBaseException; +import org.hbase.async.RpcTimedOutException; +import org.jboss.netty.buffer.ChannelBuffer; +import org.jboss.netty.buffer.ChannelBufferOutputStream; +import org.jboss.netty.buffer.ChannelBuffers; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.jgrapht.experimental.dag.DirectedAcyclicGraph; +import org.jgrapht.experimental.dag.DirectedAcyclicGraph.CycleFoundException; +import org.jgrapht.graph.DefaultEdge; +import org.jgrapht.traverse.TopologicalOrderIterator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.stumbleupon.async.Callback; +import com.stumbleupon.async.Deferred; +import com.stumbleupon.async.DeferredGroupException; + +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.QueryException; +import net.opentsdb.core.TSDB; +import net.opentsdb.core.TSQuery; +import net.opentsdb.core.TSSubQuery; +import net.opentsdb.core.Tags; +import net.opentsdb.query.expression.ExpressionDataPoint; +import net.opentsdb.query.expression.ExpressionIterator; +import net.opentsdb.query.expression.NumericFillPolicy; +import net.opentsdb.query.expression.TimeSyncedIterator; +import net.opentsdb.query.expression.VariableIterator.SetOperator; +import net.opentsdb.query.filter.TagVFilter; +import net.opentsdb.query.pojo.Expression; +import net.opentsdb.query.pojo.Filter; +import net.opentsdb.query.pojo.Metric; +import net.opentsdb.query.pojo.Output; +import net.opentsdb.query.pojo.Query; +import net.opentsdb.query.pojo.Timespan; +import net.opentsdb.stats.QueryStats; +import net.opentsdb.uid.NoSuchUniqueName; +import net.opentsdb.uid.UniqueId.UniqueIdType; +import net.opentsdb.utils.DateTime; +import net.opentsdb.utils.JSON; + +/** + * TEMP class for handling V2 queries with expression support. So far we ONLY + * support expressions and this will be pipelined better. For now it's functioning + * fairly well. + * + * So far this sucker allows for expressions and nested expressions with the + * ability to determine the output. If no output fields are specified, all + * expressions are dumped to the output. If one or more outputs are given then + * only those outputs will be emitted. + * + * TODO + * - handle/add output flags to determine whats emitted + * - allow for queries only, no expressions + * - possibly other set operations + * - time over time queries + * - skip querying for data that isn't going to be emitted + */ +public class QueryExecutor { + private static final Logger LOG = LoggerFactory.getLogger(QueryExecutor.class); + + /** The TSDB to which we belong (and will use for fetching data) */ + private final TSDB tsdb; + + /** The user's query */ + private final Query query; + + /** TEMP A v1 TSQuery that we use for fetching the data from HBase */ + private final TSQuery ts_query; + + /** A map of the sub queries to their Metric ids */ + private final Map<String, TSSubQuery> sub_queries; + + /** A map of the sub query results to their Metric ids */ + private final Map<String, DataPoints[]> sub_query_results; + + /** A map of expression iterators to their IDs */ + private final Map<String, ExpressionIterator> expressions; + + /** A map of Metric fill policies to the metric IDs */ + private final Map<String, NumericFillPolicy> fills; + + /** The HTTP query from the user */ + private HttpQuery http_query; + + /** + * Default Ctor that constructs a TSQuery and TSSubQueries from the new + * Query POJO class. + * @param tsdb The TSDB to which we belong + * @param query The raw query to parse and use for output + * @throws IllegalArgumentException if we were unable to parse the Query into + * a TSQuery. + */ + public QueryExecutor(final TSDB tsdb, final Query query) { + this.tsdb = tsdb; + this.query = query; + + // if metrics is null, this is a bad query + sub_queries = new HashMap<String, TSSubQuery>(query.getMetrics().size()); + sub_query_results = new HashMap<String, DataPoints[]>( + query.getMetrics().size()); + + if (query.getExpressions() != null) { + expressions = new HashMap<String, ExpressionIterator>( + query.getExpressions().size()); + } else { + expressions = null; + } + + final Timespan timespan = query.getTime(); + + // compile the ts_query + ts_query = new TSQuery(); + ts_query.setStart(timespan.getStart()); + ts_query.setTimezone(timespan.getTimezone()); + + if (timespan.getEnd() != null && !timespan.getEnd().isEmpty()) { + ts_query.setEnd(timespan.getEnd()); + } + + fills = new HashMap<String, NumericFillPolicy>(query.getMetrics().size()); + for (final Metric mq : query.getMetrics()) { + if (mq.getFillPolicy() != null) { + fills.put(mq.getId(), mq.getFillPolicy()); + } + final TSSubQuery sub = new TSSubQuery(); + sub_queries.put(mq.getId(), sub); + + sub.setMetric(mq.getMetric()); + + if (timespan.getDownsampler() != null) { + sub.setDownsample(timespan.getDownsampler().getInterval() + "-" + + timespan.getDownsampler().getAggregator()); + } + + // filters + if (mq.getFilter() != null && !mq.getFilter().isEmpty()) { + List<TagVFilter> filters = null; + boolean explicit_tags = false; + if (query.getFilters() == null || query.getFilters().isEmpty()) { + throw new IllegalArgumentException("No filter defined: " + mq.getFilter()); + } + for (final Filter filter : query.getFilters()) { + if (filter.getId().equals(mq.getFilter())) { + // TODO - it'd be more efficient if we could share the filters but + // for now, this is the only way to avoid concurrent modifications. + filters = new ArrayList<TagVFilter>(filter.getTags().size()); + for (final TagVFilter f : filter.getTags()) { + filters.add(f.getCopy()); + } + explicit_tags = filter.getExplicitTags(); + break; + } + } + if (filters != null) { + sub.setFilters(filters); + sub.setExplicitTags(explicit_tags); + } + } + + sub.setRate(timespan.isRate()); + sub.setAggregator( + mq.getAggregator() != null ? mq.getAggregator() : timespan.getAggregator()); + } + + final ArrayList<TSSubQuery> subs = + new ArrayList<TSSubQuery>(sub_queries.values()); + ts_query.setQueries(subs); + + // setup expressions + if (query.getExpressions() != null) { + for (final Expression expression : query.getExpressions()) { + // TODO - flags + + // TODO - get a default from the configs + final SetOperator operator = expression.getJoin() != null ? + expression.getJoin().getOperator() : SetOperator.UNION; + final boolean qts = expression.getJoin() == null ? false : expression.getJoin().getUseQueryTags(); + final boolean ats = expression.getJoin() == null ? true : expression.getJoin().getIncludeAggTags(); + final ExpressionIterator iterator = + new ExpressionIterator(expression.getId(), expression.getExpr(), + operator, qts, ats); + if (expression.getFillPolicy() != null) { + iterator.setFillPolicy(expression.getFillPolicy()); + } + expressions.put(expression.getId(), iterator); + + } + } + + ts_query.validateAndSetQuery(); + } + + /** + * Execute the RPC and serialize the response + * @param query The HTTP query to parse and and return results to + */ + public void execute(final HttpQuery query) { + http_query = query; + final QueryStats query_stats = + new QueryStats(query.getRemoteAddress(), ts_query, query.getHeaders()); + ts_query.setQueryStats(query_stats); + + /** + * Sends the serialized results to the caller. This should be the very + * last callback executed. + */ + class CompleteCB implements Callback<Object, ChannelBuffer> { + @Override + public Object call(final ChannelBuffer cb) throws Exception { + query.sendReply(cb); + return null; + } + } + + /** + * After all of the queries have run and we have data (or not) then we + * need to compile the iterators. + * This class could probably be improved: + * First we iterate over the results AND for each result, iterate over + * the expressions, giving a time synced iterator to each expression that + * needs the result set. + * THEN we iterate over the expressions again and build a DAG to determine + * if any of the expressions require the output of an expression. If so + * then we add the expressions to the proper parent and compile them in + * order. + * After all of that we're ready to start serializing and iterating + * over the results. + */ + class QueriesCB implements Callback<Object, ArrayList<DataPoints[]>> { + public Object call(final ArrayList<DataPoints[]> query_results) + throws Exception { + + for (int i = 0; i < query_results.size(); i++) { + final TSSubQuery sub = ts_query.getQueries().get(i); + + Iterator<Entry<String, TSSubQuery>> it = sub_queries.entrySet().iterator(); + while (it.hasNext()) { + final Entry<String, TSSubQuery> entry = it.next(); + if (entry.getValue().equals(sub)) { + sub_query_results.put(entry.getKey(), query_results.get(i)); + if (expressions != null) { + for (final ExpressionIterator ei : expressions.values()) { + if (ei.getVariableNames().contains(entry.getKey())) { + final TimeSyncedIterator tsi = new TimeSyncedIterator( + entry.getKey(), sub.getFilterTagKs(), + query_results.get(i)); + final NumericFillPolicy fill = fills.get(entry.getKey()); + if (fill != null) { + tsi.setFillPolicy(fill); + } + ei.addResults(entry.getKey(), tsi); + if (LOG.isDebugEnabled()) { + LOG.debug("Added results for " + entry.getKey() + + " to " + ei.getId()); + } + } + } + } + } + } + } + + // handle nested expressions + final DirectedAcyclicGraph<String, DefaultEdge> graph = + new DirectedAcyclicGraph<String, DefaultEdge>(DefaultEdge.class); + + if (expressions != null) { + for (final Entry<String, ExpressionIterator> eii : expressions.entrySet()) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("Expression entry key is %s, value is %s", + eii.getKey(), eii.getValue().toString())); + LOG.debug(String.format("Time to loop through the variable names " + + "for %s", eii.getKey())); + } + + if (!graph.containsVertex(eii.getKey())) { + if (LOG.isDebugEnabled()) { + LOG.debug("Adding vertex " + eii.getKey()); + } + graph.addVertex(eii.getKey()); + } + + for (final String var : eii.getValue().getVariableNames()) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("var is %s", var)); + } + + final ExpressionIterator ei = expressions.get(var); + + if (ei != null) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("The expression iterator for %s is %s", + var, ei.toString())); + } + + // TODO - really ought to calculate this earlier + if (eii.getKey().equals(var)) { + throw new IllegalArgumentException( + "Self referencing expression found: " + eii.getKey()); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Nested expression detected. " + eii.getKey() + + " depends on " + var); + } + + if (!graph.containsVertex(eii.getKey())) { + if (LOG.isDebugEnabled()) { + LOG.debug("Added vertex " + eii.getKey()); + } + graph.addVertex(eii.getKey()); + } else if (LOG.isDebugEnabled()) { + LOG.debug("Already contains vertex " + eii.getKey()); + } + + if (!graph.containsVertex(var)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Added vertex " + var); + } + graph.addVertex(var); + } else if (LOG.isDebugEnabled()) { + LOG.debug("Already contains vertex " + var); + } + + try { + if (LOG.isDebugEnabled()) { + LOG.debug("Added Edge " + eii.getKey() + " - " + var); + } + graph.addDagEdge(eii.getKey(), var); + } catch (CycleFoundException cfe) { + throw new IllegalArgumentException("Circular reference found: " + + eii.getKey(), cfe); + } + } else if (LOG.isDebugEnabled()) { + LOG.debug(String.format("The expression iterator for %s is null", var)); + } + } + } + } + + // compile all of the expressions + final long intersect_start = DateTime.currentTimeMillis(); + + final int expressionLength = expressions == null ? 0 : expressions.size(); + final ExpressionIterator[] compile_stack = + new ExpressionIterator[expressionLength]; + final TopologicalOrderIterator<String, DefaultEdge> it = + new TopologicalOrderIterator<String, DefaultEdge>(graph); + + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("Expressions Size is %d", expressionLength)); + LOG.debug(String.format("Topology Iterator %s", it.toString())); + } + + int i = 0; + while (it.hasNext()) { + String next = it.next(); + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("Expression: %s", next)); + } + ExpressionIterator ei = expressions.get(next); + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("Expression Iterator: %s", ei.toString())); + } + if (ei == null) { + LOG.error(String.format("The expression iterator for %s is null", next)); + } + compile_stack[i] = ei; + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("Added expression %s to compile_stack[%d]", + next, i)); + } + i++; + } + + if (i != expressionLength) { + throw new IOException(String.format(" Internal Error: Fewer " + + "expressions where added to the compile stack than " + + "expressions.size (%d instead of %d)", i, expressionLength)); + } + + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("compile stack length: %d", compile_stack.length)); + } + + for (int x = compile_stack.length - 1; x >= 0; x--) { + if (compile_stack[x] == null) { + throw new NullPointerException(String.format("Item %d in " + + "compile_stack[] is null", x)); + } + // look for and add expressions + for (final String var : compile_stack[x].getVariableNames()) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("Looking for variable %s for %s", var, + compile_stack[x].getId())); + } + ExpressionIterator source = expressions.get(var); + if (source != null) { + compile_stack[x].addResults(var, source.getCopy()); + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("Adding expression %s to %s", + source.getId(), compile_stack[x].getId())); + } + } + } + compile_stack[x].compile(); + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("Successfully compiled %s", + compile_stack[x].getId())); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Finished compilations in " + + (DateTime.currentTimeMillis() - intersect_start) + " ms"); + } + + return serialize() + .addCallback(new CompleteCB()) + .addErrback(new ErrorCB()); + } + } + + /** + * Callback executed after we have resolved the metric, tag names and tag + * values to their respective UIDs. This callback then runs the actual + * queries and fetches their results. + */ + class BuildCB implements Callback<Deferred<Object>, net.opentsdb.core.Query[]> { + @Override + public Deferred<Object> call(final net.opentsdb.core.Query[] queries) { + final ArrayList<Deferred<DataPoints[]>> deferreds = + new ArrayList<Deferred<DataPoints[]>>(queries.length); + + for (final net.opentsdb.core.Query query : queries) { + deferreds.add(query.runAsync()); + } + return Deferred.groupInOrder(deferreds).addCallback(new QueriesCB()) + .addErrback(new ErrorCB()); + } + } + + // TODO - only run the ones that will be involved in an output. Folks WILL + // ask for stuff they don't need.... *sigh* + ts_query.buildQueriesAsync(tsdb) + .addCallback(new BuildCB()) + .addErrback(new ErrorCB()); + } + + /** + * Writes the results to a ChannelBuffer to return to the caller. This will + * iterate over all of the outputs and drop in meta data where appropriate. + * @throws Exception if something went pear shaped + */ + private Deferred<ChannelBuffer> serialize() throws Exception { + final long start = System.currentTimeMillis(); + // buffers and an array list to stored the deferreds + final ChannelBuffer response = ChannelBuffers.dynamicBuffer(); + final OutputStream output_stream = new ChannelBufferOutputStream(response); + + final JsonGenerator json = JSON.getFactory().createGenerator(output_stream); + json.writeStartObject(); + json.writeFieldName("outputs"); + json.writeStartArray(); + + // We want the serializer to execute serially so we need to create a callback + // chain so that when one DPsResolver is finished, it triggers the next to + // start serializing. + final Deferred<Object> cb_chain = new Deferred<Object>(); + + // default to the expressions if there, or fall back to the metrics + final List<Output> outputs; + if (query.getOutputs() == null || query.getOutputs().isEmpty()) { + if (query.getExpressions() != null && !query.getExpressions().isEmpty()) { + outputs = new ArrayList<Output>(query.getExpressions().size()); + for (final Expression exp : query.getExpressions()) { + outputs.add(Output.Builder().setId(exp.getId()).build()); + } + } else if (query.getMetrics() != null && !query.getMetrics().isEmpty()) { + outputs = new ArrayList<Output>(query.getMetrics().size()); + for (final Metric metric : query.getMetrics()) { + outputs.add(Output.Builder().setId(metric.getId()).build()); + } + } else { + throw new IllegalArgumentException( + "How did we get here?? No metrics or expressions??"); + } + } else { + outputs = query.getOutputs(); + } + + for (final Output output : outputs) { + if (expressions != null) { + final ExpressionIterator it = expressions.get(output.getId()); + if (it != null) { + cb_chain.addCallback(new SerializeExpressionIterator(tsdb, json, + output, it, ts_query)); + continue; + } + } + + if (query.getMetrics() != null && !query.getMetrics().isEmpty()) { + final TSSubQuery sub = sub_queries.get(output.getId()); + if (sub != null) { + final TimeSyncedIterator it = new TimeSyncedIterator(output.getId(), + sub.getFilterTagKs(), sub_query_results.get(output.getId())); + cb_chain.addCallback(new SerializeSubIterator(tsdb, json, output, it)); + continue; + } + } else { + LOG.warn("Couldn't find a variable matching: " + output.getId() + + " in query " + query); + } + } + + /** Final callback to close out the JSON array and return our results */ + class FinalCB implements Callback<ChannelBuffer, Object> { + public ChannelBuffer call(final Object obj) + throws Exception { + json.writeEndArray(); + +// ts_query.getQueryStats().setTimeSerialization( +// DateTime.currentTimeMillis() - start); + ts_query.getQueryStats().markSerializationSuccessful(); + + // dump overall stats as an extra object in the array +// if (true) { +// final QueryStats stats = ts_query.getQueryStats(); +// json.writeFieldName("statsSummary"); +// json.writeStartObject(); +// //json.writeStringField("hostname", TSDB.getHostname()); +// //json.writeNumberField("runningQueries", stats.getNumRunningQueries()); +// json.writeNumberField("datapoints", stats.getAggregatedSize()); +// json.writeNumberField("rawDatapoints", stats.getSize()); +// //json.writeNumberField("rowsFetched", stats.getRowsFetched()); +// json.writeNumberField("aggregationTime", stats.getTimeAggregation()); +// json.writeNumberField("serializationTime", stats.getTimeSerialization()); +// json.writeNumberField("storageTime", stats.getTimeStorage()); +// json.writeNumberField("timeTotal", +// ((double)stats.getTimeTotal() / (double)1000000)); +// json.writeEndObject(); +// } + + // dump the original query + if (true) { + json.writeFieldName("query"); + json.writeObject(QueryExecutor.this.query); + } + // IMPORTANT Make sure the close the JSON array and the generator + json.writeEndObject(); + json.close(); + return response; + } + } + + // trigger the callback chain here + cb_chain.callback(null); + return cb_chain.addCallback(new FinalCB()); + } + + /** This has to be attached to callbacks or we may never respond to clients */ + class ErrorCB implements Callback<Object, Exception> { + public Object call(final Exception e) throws Exception { + QueryRpc.query_exceptions.incrementAndGet(); + Throwable ex = e; + try { + LOG.error("Query exception: ", e); + if (e instanceof DeferredGroupException) { + ex = e.getCause(); + while (ex != null && ex instanceof DeferredGroupException) { + ex = ex.getCause(); + } + if (ex == null) { + LOG.error("The deferred group exception didn't have a cause???"); + } + } + if (ex instanceof RpcTimedOutException) { + QueryExecutor.this.http_query.badRequest(new BadRequestException( + HttpResponseStatus.REQUEST_TIMEOUT, ex.getMessage())); + } else if (ex instanceof HBaseException) { + QueryExecutor.this.http_query.badRequest(new BadRequestException( + HttpResponseStatus.FAILED_DEPENDENCY, ex.getMessage())); + } else if (ex instanceof QueryException) { + QueryExecutor.this.http_query.badRequest(new BadRequestException( + ((QueryException)ex).getStatus(), ex.getMessage())); + } else if (ex instanceof BadRequestException) { + QueryExecutor.this.http_query.badRequest((BadRequestException)ex); + } else if (ex instanceof NoSuchUniqueName) { + QueryExecutor.this.http_query.badRequest(new BadRequestException(ex)); + } else { + QueryExecutor.this.http_query.badRequest(new BadRequestException(ex)); + } + + } catch (RuntimeException ex2) { + LOG.error("Exception thrown during exception handling", ex2); + QueryExecutor.this.http_query.sendReply + (HttpResponseStatus.INTERNAL_SERVER_ERROR, ex2.getMessage().getBytes()); + } + return null; + } + } + + /** + * Handles serializing the output of an expression iterator + */ + private class SerializeExpressionIterator + implements Callback<Deferred<Object>, Object> { + final TSDB tsdb; + final JsonGenerator json; + final Output output; + final ExpressionIterator iterator; + final ExpressionDataPoint[] dps; + final TSQuery query; + + // WARNING: Make sure to write an endObject() before triggering this guy + final Deferred<Object> completed; + + /** + * The default ctor to setup the serializer + * @param tsdb The TSDB to use for name resolution + * @param json The JSON generator to write to + * @param output The Output spec associated with this expression + * @param iterator The iterator to run through + * @param query The original TSQuery + */ + public SerializeExpressionIterator(final TSDB tsdb, final JsonGenerator json, + final Output output, final ExpressionIterator iterator, final TSQuery query) { + this.tsdb = tsdb; + this.json = json; + this.output = output; + this.iterator = iterator; + this.query = query; + dps = iterator.values(); + completed = new Deferred<Object>(); + } + + /** Super simple closer that tells the upstream chain we're done with this */ + class MetaCB implements Callback<Object, Object> { + @Override + public Object call(final Object ignored) throws Exception { + completed.callback(null); + return completed; + } + } + + @Override + public Deferred<Object> call(final Object ignored) throws Exception { + //result set opening + json.writeStartObject(); + + json.writeStringField("id", output.getId()); + if (output.getAlias() != null) { + json.writeStringField("alias", output.getAlias()); + } + json.writeFieldName("dps"); + json.writeStartArray(); + + long first_ts = Long.MIN_VALUE; + long last_ts = 0; + long count = 0; + long ts = iterator.nextTimestamp(); + long qs = query.startTime(); + long qe = query.endTime(); + while (iterator.hasNext()) { + iterator.next(ts); + + long timestamp = dps[0].timestamp(); + if (timestamp >= qs && timestamp <= qe) { + json.writeStartArray(); + if (dps.length > 0) { + json.writeNumber(timestamp); + if (first_ts == Long.MIN_VALUE) { + first_ts = timestamp; + } else { + last_ts = timestamp; + } + ++count; + } + for (int i = 0; i < dps.length; i++) { + json.writeNumber(dps[i].toDouble()); + } + + json.writeEndArray(); + } + ts = iterator.nextTimestamp(); + } + json.writeEndArray(); + + // data points meta + json.writeFieldName("dpsMeta"); + json.writeStartObject(); + json.writeNumberField("firstTimestamp", first_ts < 0 ? 0 : first_ts); + json.writeNumberField("lastTimestamp", last_ts); + json.writeNumberField("setCount", count); + json.writeNumberField("series", dps.length); + json.writeEndObject(); + + // resolve meta LAST since we may not even need it + if (dps.length > 0) { + final MetaSerializer meta_serializer = + new MetaSerializer(tsdb, json, iterator.values()); + meta_serializer.call(null).addCallback(new MetaCB()) + .addErrback(QueryExecutor.this.new ErrorCB()); + } else { + // done, not dumping any more info + json.writeEndObject(); + //json.writeEndArray(); + completed.callback(null); + } + + return completed; + } + + } + + /** + * Serializes a raw, non expression result set. + */ + private class SerializeSubIterator implements + Callback<Deferred<Object>, Object> { + final TSDB tsdb; + final JsonGenerator json; + final Output output; + final TimeSyncedIterator iterator; + + // WARNING: Make sure to write an endObject() before triggering this guy + final Deferred<Object> completed; + + public SerializeSubIterator(final TSDB tsdb, final JsonGenerator json, + final Output output, final TimeSyncedIterator iterator) { + this.tsdb = tsdb; + this.json = json; + this.output = output; + this.iterator = iterator; + completed = new Deferred<Object>(); + } + + class MetaCB implements Callback<Object, Object> { + @Override + public Object call(final Object ignored) throws Exception { + completed.callback(null); + return completed; + } + } + + @Override + public Deferred<Object> call(final Object ignored) throws Exception { + //result set opening + json.writeStartObject(); + + json.writeStringField("id", output.getId()); + if (output.getAlias() != null) { + json.writeStringField("alias", output.getAlias()); + } + json.writeFieldName("dps"); + json.writeStartArray(); + + final long first_ts = iterator.nextTimestamp(); + long ts = first_ts; + long last_ts = 0; + long count = 0; + final DataPoint[] dps = iterator.values(); + while (iterator.hasNext()) { + iterator.next(ts); + json.writeStartArray(); + + if (dps.length > 0) { + json.writeNumber(dps[0].timestamp()); + last_ts = dps[0].timestamp(); + ++count; + } + for (int i = 0; i < dps.length; i++) { + json.writeNumber(dps[i].toDouble()); + } + + json.writeEndArray(); + ts = iterator.nextTimestamp(); + } + json.writeEndArray(); + + // data points meta + json.writeFieldName("dpsMeta"); + json.writeStartObject(); + json.writeNumberField("firstTimestamp", first_ts); + json.writeNumberField("lastTimestamp", last_ts); + json.writeNumberField("setCount", count); + json.writeNumberField("series", dps.length); + json.writeEndObject(); + + // resolve meta LAST since we may not even need it + if (dps.length > 0) { + final DataPoints[] odps = iterator.getDataPoints(); + final ExpressionDataPoint[] edps = new ExpressionDataPoint[dps.length]; + for (int i = 0; i < dps.length; i++) { + edps[i] = new ExpressionDataPoint(odps[i]); + } + final MetaSerializer meta_serializer = + new MetaSerializer(tsdb, json, edps); + meta_serializer.call(null).addCallback(new MetaCB()); + } else { + // done, not dumping any more info + json.writeEndObject(); + completed.callback(null); + } + + return completed; + } + + } + + /** + * Handles resolving metrics, tags, aggregated tags and other meta data + * associated with a result set. + */ + private class MetaSerializer implements Callback<Deferred<Object>, Object> { + final TSDB tsdb; + final JsonGenerator json; + final ExpressionDataPoint[] dps; + final List<String> metrics; + final Map<String, String>[] tags; + final List<String>[] agg_tags; + + final Deferred<Object> completed; + + @SuppressWarnings("unchecked") + public MetaSerializer(final TSDB tsdb, final JsonGenerator json, + final ExpressionDataPoint[] dps) { + this.tsdb = tsdb; + this.json = json; + this.dps = dps; + completed = new Deferred<Object>(); + metrics = new ArrayList<String>(); + tags = new Map[dps.length]; + agg_tags = new List[dps.length]; + } + + class MetricsCB implements Callback<Object, ArrayList<String>> { + @Override + public Object call(final ArrayList<String> names) throws Exception { + metrics.addAll(names); + Collections.sort(metrics); + return null; + } + } + + class AggTagsCB implements Callback<Object, ArrayList<String>> { + final int index; + public AggTagsCB(final int index) { + this.index = index; + } + @Override + public Object call(final ArrayList<String> tags) throws Exception { + agg_tags[index] = tags; + return null; + } + } + + class TagsCB implements Callback<Object, Map<String, String>> { + final int index; + public TagsCB(final int index) { + this.index = index; + } + @Override + public Object call(final Map<String, String> tags) throws Exception { + MetaSerializer.this.tags[index] = tags; + return null; + } + } + + class MetaCB implements Callback<Object, ArrayList<Object>> { + @Override + public Object call(final ArrayList<Object> ignored) throws Exception { + json.writeFieldName("meta"); + json.writeStartArray(); + + // first field is the timestamp + json.writeStartObject(); + json.writeNumberField("index", 0); + json.writeFieldName("metrics"); + json.writeStartArray(); + json.writeString("timestamp"); + json.writeEndArray(); + json.writeEndObject(); + + for (int i = 0; i < dps.length; i++) { + json.writeStartObject(); + + json.writeNumberField("index", i + 1); + json.writeFieldName("metrics"); + json.writeObject(metrics); + + json.writeFieldName("commonTags"); + if (tags[i] == null) { + json.writeObject(Collections.emptyMap()); + } else { + json.writeObject(tags[i]); + } + + json.writeFieldName("aggregatedTags"); + if (agg_tags[i] == null) { + json.writeObject(Collections.emptyList()); + } else { + json.writeObject(agg_tags[i]); + } + + // TODO restore when we can calculate size efficiently + //json.writeNumberField("dps", dps[i].size()); + //json.writeNumberField("rawDps", dps[i].rawSize()); + + json.writeEndObject(); + } + + json.writeEndArray(); + + // all done with this series of results + json.writeEndObject(); + completed.callback(null); + return null; + } + } + + @Override + public Deferred<Object> call(final Object ignored) throws Exception { + final List<Deferred<Object>> deferreds = + new ArrayList<Deferred<Object>>(); + + final List<Deferred<String>> metric_deferreds = + new ArrayList<Deferred<String>>(dps[0].metricUIDs().size()); + + for (final byte[] uid : dps[0].metricUIDs()) { + metric_deferreds.add(tsdb.getUidName(UniqueIdType.METRIC, uid)); + } + + deferreds.add(Deferred.group(metric_deferreds) + .addCallback(new MetricsCB())); + + for (int i = 0; i < dps.length; i++) { + if (dps[i].aggregatedTags().size() > 0) { + final List<Deferred<String>> agg_deferreds = + new ArrayList<Deferred<String>>(dps[i].aggregatedTags().size()); + for (final byte[] uid : dps[i].aggregatedTags()) { + agg_deferreds.add(tsdb.getUidName(UniqueIdType.TAGK, uid)); + } + deferreds.add(Deferred.group(agg_deferreds) + .addCallback(new AggTagsCB(i))); + } + + deferreds.add(Tags.getTagsAsync(tsdb, dps[i].tags()) + .addCallback(new TagsCB(i))); + } + + Deferred.groupInOrder(deferreds).addCallback(new MetaCB()) + .addErrback(QueryExecutor.this.new ErrorCB()); + return completed; + } + + } + +} diff --git a/src/tsd/QueryRpc.java b/src/tsd/QueryRpc.java index 3c19897025..d875acd9bc 100644 --- a/src/tsd/QueryRpc.java +++ b/src/tsd/QueryRpc.java @@ -1,5 +1,5 @@ // This file is part of OpenTSDB. -// Copyright (C) 2013 The OpenTSDB Authors. +// Copyright (C) 2013-2017 The OpenTSDB Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU Lesser General Public License as published by @@ -19,8 +19,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.LinkedHashSet; +import java.util.concurrent.atomic.AtomicLong; +import org.hbase.async.HBaseException; +import org.hbase.async.RpcTimedOutException; import org.hbase.async.Bytes.ByteMap; +import org.jboss.netty.buffer.ChannelBuffer; +import org.jboss.netty.channel.Channel; import org.jboss.netty.handler.codec.http.HttpMethod; import org.jboss.netty.handler.codec.http.HttpResponseStatus; import org.slf4j.Logger; @@ -30,19 +37,26 @@ import com.stumbleupon.async.Deferred; import com.stumbleupon.async.DeferredGroupException; +import net.opentsdb.auth.AuthState; import net.opentsdb.core.DataPoints; import net.opentsdb.core.IncomingDataPoint; import net.opentsdb.core.Query; +import net.opentsdb.core.QueryException; import net.opentsdb.core.RateOptions; import net.opentsdb.core.TSDB; import net.opentsdb.core.TSQuery; import net.opentsdb.core.TSSubQuery; import net.opentsdb.core.Tags; import net.opentsdb.meta.Annotation; -import net.opentsdb.meta.TSMeta; import net.opentsdb.meta.TSUIDQuery; +import net.opentsdb.query.expression.ExpressionTree; +import net.opentsdb.query.expression.Expressions; +import net.opentsdb.query.filter.TagVFilter; +import net.opentsdb.stats.QueryStats; +import net.opentsdb.stats.StatsCollector; import net.opentsdb.uid.NoSuchUniqueName; import net.opentsdb.uid.UniqueId; +import net.opentsdb.utils.DateTime; import net.opentsdb.utils.JSON; /** @@ -59,6 +73,13 @@ final class QueryRpc implements HttpRpc { private static final Logger LOG = LoggerFactory.getLogger(QueryRpc.class); + /** Various counters and metrics for reporting query stats */ + static final AtomicLong query_forbidden = new AtomicLong(); + static final AtomicLong query_unauthorized = new AtomicLong(); + static final AtomicLong query_invalid = new AtomicLong(); + static final AtomicLong query_exceptions = new AtomicLong(); + static final AtomicLong query_success = new AtomicLong(); + /** * Implements the /api/query endpoint to fetch data from OpenTSDB. * @param tsdb The TSDB to use for fetching data @@ -66,13 +87,16 @@ final class QueryRpc implements HttpRpc { */ @Override public void execute(final TSDB tsdb, final HttpQuery query) - throws IOException { - - // only accept GET/POST - if (query.method() != HttpMethod.GET && query.method() != HttpMethod.POST) { - throw new BadRequestException(HttpResponseStatus.METHOD_NOT_ALLOWED, - "Method not allowed", "The HTTP method [" + query.method().getName() + - "] is not permitted for this endpoint"); + throws BadRequestException, IOException { + + // only accept GET/POST/DELETE + RpcUtil.allowedMethods(query.method(), HttpMethod.GET.getName(), HttpMethod.DELETE.getName(), HttpMethod.POST.getName()); + + if (query.method() == HttpMethod.DELETE && + !tsdb.getConfig().getBoolean("tsd.http.query.allow_delete")) { + throw new BadRequestException(HttpResponseStatus.BAD_REQUEST, + "Bad request", + "Deleting data is not enabled (tsd.http.query.allow_delete=false)"); } final String[] uri = query.explodeAPIPath(); @@ -80,8 +104,13 @@ public void execute(final TSDB tsdb, final HttpQuery query) if (endpoint.toLowerCase().equals("last")) { handleLastDataPointQuery(tsdb, query); + } else if (endpoint.toLowerCase().equals("gexp")){ + handleQuery(tsdb, query, true); + } else if (endpoint.toLowerCase().equals("exp")) { + handleExpressionQuery(tsdb, query); + return; } else { - handleQuery(tsdb, query); + handleQuery(tsdb, query, false); } } @@ -89,22 +118,35 @@ public void execute(final TSDB tsdb, final HttpQuery query) * Processing for a data point query * @param tsdb The TSDB to which we belong * @param query The HTTP query to parse/respond + * @param allow_expressions Whether or not expressions should be parsed + * (based on the endpoint) */ - private void handleQuery(final TSDB tsdb, final HttpQuery query) { + private void handleQuery(final TSDB tsdb, final HttpQuery query, + final boolean allow_expressions) { + final long start = DateTime.currentTimeMillis(); final TSQuery data_query; + final List<ExpressionTree> expressions; if (query.method() == HttpMethod.POST) { switch (query.apiVersion()) { case 0: case 1: data_query = query.serializer().parseQueryV1(); break; - default: + default: + query_invalid.incrementAndGet(); throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, "Requested API version not implemented", "Version " + query.apiVersion() + " is not implemented"); } + expressions = null; } else { - data_query = this.parseQuery(tsdb, query); + expressions = new ArrayList<ExpressionTree>(); + data_query = parseQuery(tsdb, query, expressions); + } + + if (query.getAPIMethod() == HttpMethod.DELETE && + tsdb.getConfig().getBoolean("tsd.http.query.allow_delete")) { + data_query.setDelete(true); } // validate and then compile the queries @@ -115,71 +157,185 @@ private void handleQuery(final TSDB tsdb, final HttpQuery query) { throw new BadRequestException(HttpResponseStatus.BAD_REQUEST, e.getMessage(), data_query.toString(), e); } + + checkAuthorization(tsdb, query.channel(), data_query); + + // if the user tried this query multiple times from the same IP and src port + // they'll be rejected on subsequent calls + final QueryStats query_stats = + new QueryStats(query.getRemoteAddress(), data_query, + query.getPrintableHeaders()); + data_query.setQueryStats(query_stats); + query.setStats(query_stats); - Query[] tsdbqueries; - try { - tsdbqueries = data_query.buildQueries(tsdb); - } catch(NoSuchUniqueName ex) { - throw new BadRequestException(ex); - } - final int nqueries = tsdbqueries.length; - final ArrayList<DataPoints[]> results = - new ArrayList<DataPoints[]>(nqueries); - final ArrayList<Deferred<DataPoints[]>> deferreds = - new ArrayList<Deferred<DataPoints[]>>(nqueries); - for (int i = 0; i < nqueries; i++) { - deferreds.add(tsdbqueries[i].runAsync()); - } + final int nqueries = data_query.getQueries().size(); + final ArrayList<DataPoints[]> results = new ArrayList<DataPoints[]>(nqueries); + final List<Annotation> globals = new ArrayList<Annotation>(); + + /** This has to be attached to callbacks or we may never respond to clients */ + class ErrorCB implements Callback<Object, Exception> { + public Object call(final Exception e) throws Exception { + Throwable ex = e; + try { + LOG.error("Query exception: ", e); + if (ex instanceof DeferredGroupException) { + ex = e.getCause(); + while (ex != null && ex instanceof DeferredGroupException) { + ex = ex.getCause(); + } + if (ex == null) { + LOG.error("The deferred group exception didn't have a cause???"); + } + } + if (ex instanceof RpcTimedOutException) { + query_stats.markSerialized(HttpResponseStatus.REQUEST_TIMEOUT, ex); + query.badRequest(new BadRequestException( + HttpResponseStatus.REQUEST_TIMEOUT, ex.getMessage())); + query_exceptions.incrementAndGet(); + } else if (ex instanceof HBaseException) { + query_stats.markSerialized(HttpResponseStatus.FAILED_DEPENDENCY, ex); + query.badRequest(new BadRequestException( + HttpResponseStatus.FAILED_DEPENDENCY, ex.getMessage())); + query_exceptions.incrementAndGet(); + } else if (ex instanceof QueryException) { + query_stats.markSerialized(((QueryException)ex).getStatus(), ex); + query.badRequest(new BadRequestException( + ((QueryException)ex).getStatus(), ex.getMessage())); + query_exceptions.incrementAndGet(); + } else if (ex instanceof BadRequestException) { + query_stats.markSerialized(((BadRequestException)ex).getStatus(), ex); + query.badRequest((BadRequestException)ex); + query_invalid.incrementAndGet(); + } else if (ex instanceof NoSuchUniqueName) { + query_stats.markSerialized(HttpResponseStatus.BAD_REQUEST, ex); + query.badRequest(new BadRequestException(ex)); + query_invalid.incrementAndGet(); + } else { + query_stats.markSerialized(HttpResponseStatus.INTERNAL_SERVER_ERROR, ex); + query.badRequest(new BadRequestException(ex)); + query_exceptions.incrementAndGet(); + } + + } catch (RuntimeException ex2) { + LOG.error("Exception thrown during exception handling", ex2); + query_stats.markSerialized(HttpResponseStatus.INTERNAL_SERVER_ERROR, ex2); + query.sendReply(HttpResponseStatus.INTERNAL_SERVER_ERROR, + ex2.getMessage().getBytes()); + query_exceptions.incrementAndGet(); + } + return null; + } + } + /** - * After all of the queries have run, we get the results in the order given - * and add dump the results in an array - */ + * After all of the queries have run, we get the results in the order given + * and add dump the results in an array + */ class QueriesCB implements Callback<Object, ArrayList<DataPoints[]>> { public Object call(final ArrayList<DataPoints[]> query_results) throws Exception { - results.addAll(query_results); + if (allow_expressions) { + // process each of the expressions into a new list, then merge it + // with the original. This avoids possible recursion loops. + final List<DataPoints[]> expression_results = + new ArrayList<DataPoints[]>(expressions.size()); + // let exceptions bubble up + for (final ExpressionTree expression : expressions) { + expression_results.add(expression.evaluate(query_results)); + } + results.addAll(expression_results); + } else { + results.addAll(query_results); + } + + /** Simply returns the buffer once serialization is complete and logs it */ + class SendIt implements Callback<Object, ChannelBuffer> { + public Object call(final ChannelBuffer buffer) throws Exception { + query.sendReply(buffer); + query_success.incrementAndGet(); + return null; + } + } + + switch (query.apiVersion()) { + case 0: + case 1: + query.serializer().formatQueryAsyncV1(data_query, results, + globals).addCallback(new SendIt()).addErrback(new ErrorCB()); + break; + default: + query_invalid.incrementAndGet(); + throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, + "Requested API version not implemented", "Version " + + query.apiVersion() + " is not implemented"); + } return null; } } - // if the user wants global annotations, we need to scan and fetch - // TODO(cl) need to async this at some point. It's not super straight - // forward as we can't just add it to the "deferreds" queue since the types - // are different. - List<Annotation> globals = null; - if (!data_query.getNoAnnotations() && data_query.getGlobalAnnotations()) { - try { - globals = Annotation.getGlobalAnnotations(tsdb, - data_query.startTime() / 1000, data_query.endTime() / 1000) - .joinUninterruptibly(); - } catch (Exception e) { - throw new RuntimeException("Shouldn't be here", e); + /** + * Callback executed after we have resolved the metric, tag names and tag + * values to their respective UIDs. This callback then runs the actual + * queries and fetches their results. + */ + class BuildCB implements Callback<Deferred<Object>, Query[]> { + @Override + public Deferred<Object> call(final Query[] queries) { + final ArrayList<Deferred<DataPoints[]>> deferreds = + new ArrayList<Deferred<DataPoints[]>>(queries.length); + for (final Query query : queries) { + // call different interfaces basing on whether it is a percentile query + if (!query.isHistogramQuery()) { + deferreds.add(query.runAsync()); + } else { + deferreds.add(query.runHistogramAsync()); + } + } + return Deferred.groupInOrder(deferreds).addCallback(new QueriesCB()); } } - - try { - Deferred.groupInOrder(deferreds).addCallback(new QueriesCB()) - .joinUninterruptibly(); - } catch (Exception e) { - throw new RuntimeException("Shouldn't be here", e); - } - switch (query.apiVersion()) { - case 0: - case 1: - query.sendReply(query.serializer().formatQueryV1(data_query, results, - globals)); - break; - default: - throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, - "Requested API version not implemented", "Version " + - query.apiVersion() + " is not implemented"); + /** Handles storing the global annotations after fetching them */ + class GlobalCB implements Callback<Object, List<Annotation>> { + public Object call(final List<Annotation> annotations) throws Exception { + globals.addAll(annotations); + return data_query.buildQueriesAsync(tsdb).addCallback(new BuildCB()); + } + } + + // if we the caller wants to search for global annotations, fire that off + // first then scan for the notes, then pass everything off to the formatter + // when complete + if (!data_query.getNoAnnotations() && data_query.getGlobalAnnotations()) { + Annotation.getGlobalAnnotations(tsdb, + data_query.startTime() / 1000, data_query.endTime() / 1000) + .addCallback(new GlobalCB()).addErrback(new ErrorCB()); + } else { + data_query.buildQueriesAsync(tsdb).addCallback(new BuildCB()) + .addErrback(new ErrorCB()); } } /** - * + * Handles an expression query + * @param tsdb The TSDB to which we belong + * @param query The HTTP query to parse/respond + * @since 2.3 + */ + private void handleExpressionQuery(final TSDB tsdb, final HttpQuery query) { + final net.opentsdb.query.pojo.Query v2_query = + JSON.parseToObject(query.getContent(), net.opentsdb.query.pojo.Query.class); + v2_query.validate(); + + checkAuthorization(tsdb, query.channel(), v2_query); + + final QueryExecutor executor = new QueryExecutor(tsdb, v2_query); + executor.execute(query); + } + + /** + * Processes a last data point query * @param tsdb The TSDB to which we belong * @param query The HTTP query to parse/respond */ @@ -206,12 +362,10 @@ private void handleLastDataPointQuery(final TSDB tsdb, final HttpQuery query) { "Missing sub queries"); } - // list of getLastPoint calls - final ArrayList<Deferred<IncomingDataPoint>> calls = - new ArrayList<Deferred<IncomingDataPoint>>(); - // list of calls to TSUIDQuery for scanning the tsdb-meta table - final ArrayList<Deferred<Object>> tsuid_query_wait = - new ArrayList<Deferred<Object>>(); + // a list of deferreds to wait on + final ArrayList<Deferred<Object>> calls = new ArrayList<Deferred<Object>>(); + // final results for serialization + final List<IncomingDataPoint> results = new ArrayList<IncomingDataPoint>(); /** * Used to catch exceptions @@ -231,7 +385,29 @@ public Object call(final Exception e) throws Exception { } else { throw e; } - } + } + @Override + public String toString() { + return "Error back"; + } + } + + final class FetchCB implements Callback<Deferred<Object>, ArrayList<IncomingDataPoint>> { + @Override + public Deferred<Object> call(final ArrayList<IncomingDataPoint> dps) throws Exception { + synchronized(results) { + for (final IncomingDataPoint dp : dps) { + if (dp != null) { + results.add(dp); + } + } + } + return Deferred.fromResult(null); + } + @Override + public String toString() { + return "Fetched data points CB"; + } } /** @@ -239,78 +415,75 @@ public Object call(final Exception e) throws Exception { * metric and/or tags. If matches were found, it fires off a number of * getLastPoint requests, adding the deferreds to the calls list */ - final class TSUIDQueryCB implements Callback<Object, ByteMap<Long>> { - public Object call(final ByteMap<Long> tsuids) throws Exception { + final class TSUIDQueryCB implements Callback<Deferred<Object>, ByteMap<Long>> { + public Deferred<Object> call(final ByteMap<Long> tsuids) throws Exception { if (tsuids == null || tsuids.isEmpty()) { return null; } - + final ArrayList<Deferred<IncomingDataPoint>> deferreds = + new ArrayList<Deferred<IncomingDataPoint>>(tsuids.size()); for (Map.Entry<byte[], Long> entry : tsuids.entrySet()) { - calls.add(TSUIDQuery.getLastPoint(tsdb, entry.getKey(), + deferreds.add(TSUIDQuery.getLastPoint(tsdb, entry.getKey(), data_query.getResolveNames(), data_query.getBackScan(), entry.getValue())); } - return null; + return Deferred.group(deferreds).addCallbackDeferring(new FetchCB()); } - } - - /** - * Callback used to force the thread to wait for the TSUIDQueries to complete - */ - final class TSUIDQueryWaitCB implements Callback<Object, ArrayList<Object>> { - public Object call(ArrayList<Object> arg0) throws Exception { - return null; + @Override + public String toString() { + return "TSMeta scan CB"; } } - + /** * Used to wait on the list of data point deferreds. Once they're all done * this will return the results to the call via the serializer */ - final class FinalCB implements Callback<Object, ArrayList<IncomingDataPoint>> { - @SuppressWarnings("unchecked") - public Object call(final ArrayList<IncomingDataPoint> data_points) - throws Exception { - if (data_points == null) { - query.sendReply(query.serializer() - .formatLastPointQueryV1(Collections.EMPTY_LIST)); - } else { - query.sendReply(query.serializer() - .formatLastPointQueryV1(data_points)); - } + final class FinalCB implements Callback<Object, ArrayList<Object>> { + public Object call(final ArrayList<Object> done) throws Exception { + query.sendReply(query.serializer().formatLastPointQueryV1(results)); return null; } + @Override + public String toString() { + return "Final CB"; + } } + try { // start executing the queries - for (LastPointSubQuery sub_query : data_query.getQueries()) { + for (final LastPointSubQuery sub_query : data_query.getQueries()) { + final ArrayList<Deferred<IncomingDataPoint>> deferreds = + new ArrayList<Deferred<IncomingDataPoint>>(); // TSUID queries take precedence so if there are any TSUIDs listed, // process the TSUIDs and ignore the metric/tags if (sub_query.getTSUIDs() != null && !sub_query.getTSUIDs().isEmpty()) { - for (String tsuid : sub_query.getTSUIDs()) { - calls.add(TSUIDQuery.getLastPoint(tsdb, UniqueId.stringToUid(tsuid), - data_query.getResolveNames(), data_query.getBackScan(), 0)); + for (final String tsuid : sub_query.getTSUIDs()) { + final TSUIDQuery tsuid_query = new TSUIDQuery(tsdb, + UniqueId.stringToUid(tsuid)); + deferreds.add(tsuid_query.getLastPoint(data_query.getResolveNames(), + data_query.getBackScan())); } } else { - final TSUIDQuery tsuid_query = new TSUIDQuery(tsdb); @SuppressWarnings("unchecked") - final HashMap<String, String> tags = - (HashMap<String, String>) (sub_query.getTags() != null ? - sub_query.getTags() : Collections.EMPTY_MAP); - tsuid_query.setQuery(sub_query.getMetric(), tags); - tsuid_query_wait.add( - tsuid_query.getLastWriteTimes().addCallback(new TSUIDQueryCB())); + final TSUIDQuery tsuid_query = + new TSUIDQuery(tsdb, sub_query.getMetric(), + sub_query.getTags() != null ? + sub_query.getTags() : Collections.EMPTY_MAP); + if (data_query.getBackScan() > 0) { + deferreds.add(tsuid_query.getLastPoint(data_query.getResolveNames(), + data_query.getBackScan())); + } else { + calls.add(tsuid_query.getLastWriteTimes() + .addCallbackDeferring(new TSUIDQueryCB())); + } + } + + if (deferreds.size() > 0) { + calls.add(Deferred.group(deferreds).addCallbackDeferring(new FetchCB())); } } - if (!tsuid_query_wait.isEmpty()) { - // wait on the time series queries first. If you don't, they may try - // to add deferreds to the calls list - Deferred.group(tsuid_query_wait) - .addCallback(new TSUIDQueryWaitCB()) - .addErrback(new ErrBack()) - .joinUninterruptibly(); - } Deferred.group(calls) .addCallback(new FinalCB()) .addErrback(new ErrBack()) @@ -339,8 +512,24 @@ public Object call(final ArrayList<IncomingDataPoint> data_points) * @param query The HTTP Query for parsing * @return A TSQuery if parsing was successful * @throws BadRequestException if parsing was unsuccessful + * @since 2.3 */ - private TSQuery parseQuery(final TSDB tsdb, final HttpQuery query) { + public static TSQuery parseQuery(final TSDB tsdb, final HttpQuery query) { + return parseQuery(tsdb, query, null); + } + + /** + * Parses a query string legacy style query from the URI + * @param tsdb The TSDB we belong to + * @param query The HTTP Query for parsing + * @param expressions A list of parsed expression trees filled from the URI. + * If this is null, it means any expressions in the URI will be skipped. + * @return A TSQuery if parsing was successful + * @throws BadRequestException if parsing was unsuccessful + * @since 2.3 + */ + public static TSQuery parseQuery(final TSDB tsdb, final HttpQuery query, + final List<ExpressionTree> expressions) { final TSQuery data_query = new TSQuery(); data_query.setStart(query.getRequiredQueryStringParam("start")); @@ -366,24 +555,66 @@ private TSQuery parseQuery(final TSDB tsdb, final HttpQuery query) { data_query.setMsResolution(true); } + if (query.hasQueryStringParam("show_query")) { + data_query.setShowQuery(true); + } + + if (query.hasQueryStringParam("show_stats")) { + data_query.setShowStats(true); + } + + if (query.hasQueryStringParam("show_summary")) { + data_query.setShowSummary(true); + } + // handle tsuid queries first if (query.hasQueryStringParam("tsuid")) { final List<String> tsuids = query.getQueryStringParams("tsuid"); for (String q : tsuids) { - this.parseTsuidTypeSubQuery(q, data_query); + parseTsuidTypeSubQuery(q, data_query); } } if (query.hasQueryStringParam("m")) { final List<String> legacy_queries = query.getQueryStringParams("m"); for (String q : legacy_queries) { - this.parseMTypeSubQuery(q, data_query); + parseMTypeSubQuery(q, data_query); + } + } + + // TODO - testing out the graphite style expressions here with the "exp" + // param that could stand for experimental or expression ;) + if (expressions != null) { + if (query.hasQueryStringParam("exp")) { + final List<String> uri_expressions = query.getQueryStringParams("exp"); + final List<String> metric_queries = new ArrayList<String>( + uri_expressions.size()); + // parse the expressions into their trees. If one or more expressions + // are improper then it will toss an exception up + expressions.addAll(Expressions.parseExpressions( + uri_expressions, data_query, metric_queries)); + // iterate over each of the parsed metric queries and store it in the + // TSQuery list so that we fetch the data for them. + for (final String mq: metric_queries) { + parseMTypeSubQuery(mq, data_query); + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Received a request with an expression but at the " + + "wrong endpoint: " + query); } } if (data_query.getQueries() == null || data_query.getQueries().size() < 1) { throw new BadRequestException("Missing sub queries"); } + + // Filter out duplicate queries + Set<TSSubQuery> query_set = new LinkedHashSet<TSSubQuery>(data_query.getQueries()); + data_query.getQueries().clear(); + data_query.getQueries().addAll(query_set); + return data_query; } @@ -396,7 +627,7 @@ private TSQuery parseQuery(final TSDB tsdb, final HttpQuery query) { * @throws BadRequestException if we are unable to parse the query or it is * missing components */ - private void parseMTypeSubQuery(final String query_string, + private static void parseMTypeSubQuery(final String query_string, TSQuery data_query) { if (query_string == null || query_string.isEmpty()) { throw new BadRequestException("The query string was empty"); @@ -417,9 +648,9 @@ private void parseMTypeSubQuery(final String query_string, sub_query.setAggregator(parts[0]); i--; // Move to the last part (the metric name). - HashMap<String, String> tags = new HashMap<String, String>(); - sub_query.setMetric(Tags.parseWithMetric(parts[i], tags)); - sub_query.setTags(tags); + List<TagVFilter> filters = new ArrayList<TagVFilter>(); + sub_query.setMetric(Tags.parseWithMetricAndFilters(parts[i], filters)); + sub_query.setFilters(filters); // parse out the rate and downsampler for (int x = 1; x < parts.length - 1; x++) { @@ -430,6 +661,16 @@ private void parseMTypeSubQuery(final String query_string, } } else if (Character.isDigit(parts[x].charAt(0))) { sub_query.setDownsample(parts[x]); + } else if (parts[x].equalsIgnoreCase("pre-agg")) { + sub_query.setPreAggregate(true); + } else if (parts[x].toLowerCase().startsWith("rollup_")) { + sub_query.setRollupUsage(parts[x]); + } else if (parts[x].toLowerCase().startsWith("percentiles")) { + sub_query.setPercentiles(QueryRpc.parsePercentiles(parts[x])); + } else if (parts[x].toLowerCase().startsWith("show-histogram-buckets")) { + sub_query.setShowHistogramBuckets(true); + } else if (parts[x].toLowerCase().startsWith("explicit_tags")) { + sub_query.setExplicitTags(true); } } @@ -449,7 +690,7 @@ private void parseMTypeSubQuery(final String query_string, * @throws BadRequestException if we are unable to parse the query or it is * missing components */ - private void parseTsuidTypeSubQuery(final String query_string, + private static void parseTsuidTypeSubQuery(final String query_string, TSQuery data_query) { if (query_string == null || query_string.isEmpty()) { throw new BadRequestException("The tsuid query string was empty"); @@ -483,6 +724,10 @@ private void parseTsuidTypeSubQuery(final String query_string, } } else if (Character.isDigit(parts[x].charAt(0))) { sub_query.setDownsample(parts[x]); + } else if (parts[x].toLowerCase().startsWith("percentiles")) { + sub_query.setPercentiles(QueryRpc.parsePercentiles(parts[x])); + } else if (parts[x].toLowerCase().startsWith("show-histogram-buckets")) { + sub_query.setShowHistogramBuckets(true); } } @@ -527,14 +772,15 @@ static final public RateOptions parseRateOptions(final boolean rate, + parts.length + " parts"); } - final boolean counter = "counter".equals(parts[0]); + final boolean counter = parts[0].endsWith("counter"); try { final long max = (parts.length >= 2 && parts[1].length() > 0 ? Long .parseLong(parts[1]) : Long.MAX_VALUE); try { final long reset = (parts.length >= 3 && parts[2].length() > 0 ? Long .parseLong(parts[2]) : RateOptions.DEFAULT_RESET_VALUE); - return new RateOptions(counter, max, reset); + final boolean drop_counter = parts[0].equals("dropcounter"); + return new RateOptions(counter, max, reset, drop_counter); } catch (NumberFormatException e) { throw new BadRequestException( "Reset value of counter was not a number, received '" + parts[2] @@ -592,6 +838,83 @@ private LastPointQuery parseLastPointQuery(final TSDB tsdb, query.setQueries(sub_queries); return query; } + + private void checkAuthorization(final TSDB tsdb, final Channel chan, final net.opentsdb.query.pojo.Query data_query) { + if (tsdb.getConfig().getBoolean("tsd.core.authentication.enable")) { + if (tsdb.getAuth().isReady(tsdb, chan)) { + final AuthState state = tsdb.getAuth().authorization().allowQuery( + (AuthState) chan.getAttachment(), data_query); + handleAuthorization(state); + } + } + } + + private void checkAuthorization(final TSDB tsdb, final Channel chan, final TSQuery data_query) { + if (tsdb.getConfig().getBoolean("tsd.core.authentication.enable")) { + if (tsdb.getAuth().isReady(tsdb, chan)) { + final AuthState state = tsdb.getAuth().authorization().allowQuery( + (AuthState) chan.getAttachment(), data_query); + handleAuthorization(state); + } + } + } + + private void handleAuthorization(AuthState state) { + switch (state.getStatus()) { + case SUCCESS: + // cary on :) + break; + case UNAUTHORIZED: + query_unauthorized.incrementAndGet(); + throw new BadRequestException(HttpResponseStatus.UNAUTHORIZED, + state.getMessage()); + case FORBIDDEN: + query_forbidden.incrementAndGet(); + throw new BadRequestException(HttpResponseStatus.FORBIDDEN, + state.getMessage()); + default: + query_exceptions.incrementAndGet(); + throw new BadRequestException(HttpResponseStatus.INTERNAL_SERVER_ERROR, + state.getMessage()); + } + } + + /** + * Parse the "percentile" section of the query string and returns an list of + * float that contains the percentile calculation paramters + * <p> + * the format of the section: percentile[xx,yy,zz] + * </p> + * <p> + * xx, yy, zz are the floats + * </p> + * @param spec + * @return + */ + public static final List<Float> parsePercentiles(final String spec) { + List<Float> rs = new ArrayList<Float>(); + int start_pos = spec.indexOf('['); + int end_pos = spec.indexOf(']'); + if (start_pos == -1 || end_pos == -1) { + throw new BadRequestException("Malformated percentile query paramater: " + spec); + } + + String [] floats = Tags.splitString(spec.substring(start_pos + 1, end_pos), ','); + for (String s : floats) { + String trimed = s.trim(); + rs.add(Float.valueOf(trimed)); + } + return rs; + } + + /** @param collector Populates the collector with statistics */ + public static void collectStats(final StatsCollector collector) { + collector.record("http.query.unauthorized", query_unauthorized); + collector.record("http.query.forbidden", query_forbidden); + collector.record("http.query.invalid_requests", query_invalid); + collector.record("http.query.exceptions", query_exceptions); + collector.record("http.query.success", query_success); + } public static class LastPointQuery { @@ -698,3 +1021,4 @@ public void setTSUIDs(final List<String> tsuids) { } } } + diff --git a/src/tsd/QueryUi.gwt.xml b/src/tsd/QueryUi.gwt.xml index 0bf3faf297..781cb128d5 100644 --- a/src/tsd/QueryUi.gwt.xml +++ b/src/tsd/QueryUi.gwt.xml @@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8"?> <module rename-to="queryui"> <inherits name="com.google.gwt.user.User"/> - <inherits name="com.google.gwt.user.theme.standard.Standard"/> + <inherits name="com.sensei.themes.opentsdb.Opentsdb"/> <inherits name="com.google.gwt.http.HTTP"/> <inherits name="com.google.gwt.json.JSON"/> <entry-point class="tsd.client.QueryUi"/> diff --git a/src/tsd/RTPublisher.java b/src/tsd/RTPublisher.java index 551267e62d..194a36e353 100644 --- a/src/tsd/RTPublisher.java +++ b/src/tsd/RTPublisher.java @@ -48,7 +48,7 @@ public abstract class RTPublisher { * @param tsdb The parent TSDB object * @throws IllegalArgumentException if required configuration parameters are * missing - * @throws Exception if something else goes wrong + * @throws RuntimeException if something else goes wrong */ public abstract void initialize(final TSDB tsdb); @@ -145,4 +145,22 @@ public abstract Deferred<Object> publishDataPoint(final String metric, */ public abstract Deferred<Object> publishAnnotation(Annotation annotation); + /** + * Called any time a new histogram point is published + * @param metric The name of the metric associated with the data point + * @param timestamp Timestamp as a Unix epoch in seconds or milliseconds + * (depending on the TSD's configuration) + * @param value Encoded raw data blob for the histogram point + * @param tags Tagk/v pairs + * @param tsuid Time series UID for the value + * @return A deferred without special meaning to wait on if necessary. The + * value may be null but a Deferred must be returned. + */ + public Deferred<Object> publishHistogramPoint(final String metric, + final long timestamp, final byte[] value, + final Map<String, String> tags, + final byte[] tsuid) { + throw new UnsupportedOperationException("Not yet implemented"); + } + } diff --git a/src/tsd/RollupDataPointRpc.java b/src/tsd/RollupDataPointRpc.java new file mode 100644 index 0000000000..3798e529c9 --- /dev/null +++ b/src/tsd/RollupDataPointRpc.java @@ -0,0 +1,227 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.tsd; + +import com.stumbleupon.async.Deferred; + +import net.opentsdb.core.IncomingDataPoint; +import net.opentsdb.core.TSDB; +import net.opentsdb.core.Tags; +import net.opentsdb.rollup.RollUpDataPoint; +import net.opentsdb.uid.NoSuchUniqueName; +import net.opentsdb.utils.Config; + +import org.jboss.netty.handler.codec.http.HttpMethod; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; + +/** + * A class that handles overriding parsing calls when writing a rolled up data + * point to storage. + * @since 2.4 + */ +class RollupDataPointRpc extends PutDataPointRpc + implements TelnetRpc, HttpRpc { + + private enum TelnetIndex { + COMMAND, + INTERVAL_AGG, + METRIC, + TIMESTAMP, + VALUE, + TAGS + } + + /** + * Default Ctor + * @param config The TSDB config to pull from + */ + public RollupDataPointRpc(final Config config) { + super(config); + } + + /** + * Handles HTTP RPC put requests + * @param tsdb The TSDB to which we belong + * @param query The HTTP query from the user + * @throws IOException if there is an error parsing the query or formatting + * the output + * @throws BadRequestException if the user supplied bad data + */ + @Override + public void execute(final TSDB tsdb, final HttpQuery query) + throws IOException { + http_requests.incrementAndGet(); + + // only accept POST + if (query.method() != HttpMethod.POST) { + throw new BadRequestException(HttpResponseStatus.METHOD_NOT_ALLOWED, + "Method not allowed", "The HTTP method [" + query.method().getName() + + "] is not permitted for this endpoint"); + } + + final List<RollUpDataPoint> dps = query.serializer() + .parsePutV1(RollUpDataPoint.class, HttpJsonSerializer.TR_ROLLUP); + processDataPoint(tsdb, query, dps); + } + + /** + * Imports a single rolled up data point. + * @param tsdb The TSDB to import the data point into. + * @param words The words describing the data point to import, in + * the following format: + * {@code rollup interval:[aggregator] metric timestamp value ..tags..} + * @return A deferred object that indicates the completion of the request. + * @throws NumberFormatException if the timestamp, value or count is invalid. + * @throws IllegalArgumentException if any other argument is invalid. + * @throws NoSuchUniqueName if the metric isn't registered. + */ + @Override + protected Deferred<Object> importDataPoint(final TSDB tsdb, + final String[] words) { + words[TelnetIndex.COMMAND.ordinal()] = null; // Ditch the "rollup" string. + if (words.length < TelnetIndex.TAGS.ordinal() + 1) { + throw new IllegalArgumentException("not enough arguments" + + " (need least 7, got " + (words.length - 1) + ')'); + } + + final String interval_agg = words[TelnetIndex.INTERVAL_AGG.ordinal()]; + if (interval_agg.isEmpty()) { + throw new IllegalArgumentException("Missing interval or aggregator"); + } + + String interval = null; + String temporal_agg = null; + String spatial_agg = null; + // if the interval_agg has a - in it, then it's an interval. If there's a : + // then it is both. If no dash or colon then it's just a spatial agg. + final String[] interval_parts = interval_agg.split(":"); + final int dash = interval_parts[0].indexOf("-"); + if (dash > -1) { + interval = interval_parts[0].substring(0,dash); + temporal_agg = interval_parts[0].substring(dash + 1); + + } else if (interval_parts.length == 1) { + spatial_agg = interval_parts[0]; + } + if (interval_parts.length > 1) { + spatial_agg = interval_parts[1]; + } + + final String metric = words[TelnetIndex.METRIC.ordinal()]; + if (metric.length() <= 0) { + throw new IllegalArgumentException("empty metric name"); + } + + final long timestamp; + if (words[TelnetIndex.TIMESTAMP.ordinal()].contains(".")) { + timestamp = Tags.parseLong(words[TelnetIndex.TIMESTAMP.ordinal()] + .replace(".", "")); + } else { + timestamp = Tags.parseLong(words[TelnetIndex.TIMESTAMP.ordinal()]); + } + if (timestamp <= 0) { + throw new IllegalArgumentException("invalid timestamp: " + timestamp); + } + + final String value = words[TelnetIndex.VALUE.ordinal()]; + if (value.length() <= 0) { + throw new IllegalArgumentException("empty value"); + } + + final HashMap<String, String> tags = new HashMap<String, String>(); + for (int i = TelnetIndex.TAGS.ordinal(); i < words.length; i++) { + if (!words[i].isEmpty()) { + Tags.parse(tags, words[i]); + } + } + + if (spatial_agg != null && temporal_agg == null) { + temporal_agg = spatial_agg; + } + + if (Tags.looksLikeInteger(value)) { + return tsdb.addAggregatePoint(metric, timestamp, Tags.parseLong(value), + tags, spatial_agg != null ? true : false, interval, temporal_agg, + spatial_agg); + } else if (Tags.fitsInFloat(value)) { // floating point value + return tsdb.addAggregatePoint(metric, timestamp, Float.parseFloat(value), + tags, spatial_agg != null ? true : false, interval, temporal_agg, + spatial_agg); + } else { + return tsdb.addAggregatePoint(metric, timestamp, Double.parseDouble(value), + tags, spatial_agg != null ? true : false, interval, temporal_agg, + spatial_agg); + } + } + + /** + * Converts the string array to an IncomingDataPoint. WARNING: This method + * does not perform validation. It should only be used by the Telnet style + * {@code execute} above within the error callback. At that point it means + * the array parsed correctly as per {@code importDataPoint}. + * @param tsdb The TSDB for encoding/decoding. + * @param words The array of strings representing a data point + * @return An incoming data point object. + */ + @Override + protected IncomingDataPoint getDataPointFromString(final TSDB tsdb, + final String[] words) { + final RollUpDataPoint dp = new RollUpDataPoint(); + + final String interval_agg = words[TelnetIndex.INTERVAL_AGG.ordinal()]; + String interval = null; + String temporal_agg = null; + String spatial_agg = null; + // if the interval_agg has a - in it, then it's an interval. If there's a : + // then it is both. If no dash or colon then it's just a spatial agg. + final String[] interval_parts = interval_agg.split(":"); + final int dash = interval_parts[0].indexOf("-"); + if (dash > -1) { + interval = interval_parts[0].substring(0,dash); + temporal_agg = interval_parts[0].substring(dash + 1); + + } else if (interval_parts.length == 1) { + spatial_agg = interval_parts[0]; + } + if (interval_parts.length > 1) { + spatial_agg = interval_parts[1]; + } + dp.setInterval(interval); + dp.setAggregator(temporal_agg); + dp.setGroupByAggregator(spatial_agg); + + dp.setMetric(words[TelnetIndex.METRIC.ordinal()]); + + if (words[TelnetIndex.TIMESTAMP.ordinal()].contains(".")) { + dp.setTimestamp(Tags.parseLong(words[TelnetIndex.TIMESTAMP.ordinal()] + .replace(".", ""))); + } else { + dp.setTimestamp(Tags.parseLong(words[TelnetIndex.TIMESTAMP.ordinal()])); + } + + dp.setValue(words[TelnetIndex.VALUE.ordinal()]); + + final HashMap<String, String> tags = new HashMap<String, String>(); + for (int i = TelnetIndex.TAGS.ordinal(); i < words.length; i++) { + if (!words[i].isEmpty()) { + Tags.parse(tags, words[i]); + } + } + dp.setTags(tags); + return dp; + } +} diff --git a/src/tsd/RpcHandler.java b/src/tsd/RpcHandler.java index 44365007a0..1424bc4217 100644 --- a/src/tsd/RpcHandler.java +++ b/src/tsd/RpcHandler.java @@ -12,76 +12,86 @@ // see <http://www.gnu.org/licenses/>. package net.opentsdb.tsd; -import java.io.IOException; import java.util.Arrays; -import java.util.HashMap; import java.util.HashSet; import java.util.concurrent.atomic.AtomicLong; +import com.google.common.base.Strings; import com.google.common.net.HttpHeaders; -import com.stumbleupon.async.Callback; import com.stumbleupon.async.Deferred; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import org.jboss.netty.channel.Channel; +import org.jboss.netty.channel.ChannelFuture; +import org.jboss.netty.channel.ChannelFutureListener; import org.jboss.netty.channel.ChannelHandlerContext; import org.jboss.netty.channel.MessageEvent; +import org.jboss.netty.handler.codec.http.DefaultHttpResponse; import org.jboss.netty.handler.codec.http.HttpMethod; import org.jboss.netty.handler.codec.http.HttpRequest; import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.jboss.netty.handler.codec.http.HttpVersion; import org.jboss.netty.handler.timeout.IdleState; import org.jboss.netty.handler.timeout.IdleStateAwareChannelUpstreamHandler; import org.jboss.netty.handler.timeout.IdleStateEvent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import net.opentsdb.BuildData; -import net.opentsdb.core.Aggregators; import net.opentsdb.core.TSDB; import net.opentsdb.stats.StatsCollector; -import net.opentsdb.utils.JSON; /** - * Stateless handler for RPCs (telnet-style or HTTP). + * Stateless handler for all RPCs: telnet-style, built-in or plugin + * HTTP. */ final class RpcHandler extends IdleStateAwareChannelUpstreamHandler { private static final Logger LOG = LoggerFactory.getLogger(RpcHandler.class); - + private static final AtomicLong telnet_rpcs_received = new AtomicLong(); private static final AtomicLong http_rpcs_received = new AtomicLong(); + private static final AtomicLong http_plugin_rpcs_received = new AtomicLong(); private static final AtomicLong exceptions_caught = new AtomicLong(); - /** Commands we can serve on the simple, telnet-style RPC interface. */ - private final HashMap<String, TelnetRpc> telnet_commands; /** RPC executed when there's an unknown telnet-style command. */ private final TelnetRpc unknown_cmd = new Unknown(); - /** Commands we serve on the HTTP interface. */ - private final HashMap<String, HttpRpc> http_commands; /** List of domains to allow access to HTTP. By default this will be empty and * all CORS headers will be ignored. */ private final HashSet<String> cors_domains; /** List of headers allowed for access to HTTP. By default this will contain a * set of known-to-work headers */ private final String cors_headers; + /** RPC plugins. Contains the handlers we dispatch requests to. */ + private final RpcManager rpc_manager; /** The TSDB to use. */ private final TSDB tsdb; - + /** - * Constructor that loads the CORS domain list and configures the route maps - * for telnet and HTTP requests + * Constructor that loads the CORS domain list and prepares for + * handling requests. This constructor creates its own {@link RpcManager}. * @param tsdb The TSDB to use. * @throws IllegalArgumentException if there was an error with the CORS domain * list */ public RpcHandler(final TSDB tsdb) { + this(tsdb, RpcManager.instance(tsdb)); + } + + /** + * Constructor that loads the CORS domain list and prepares for handling + * requests. + * @param tsdb The TSDB to use. + * @param manager instance of a ready-to-use {@link RpcManager}. + * @throws IllegalArgumentException if there was an error with the CORS domain + * list + */ + public RpcHandler(final TSDB tsdb, final RpcManager manager) { this.tsdb = tsdb; + this.rpc_manager = manager; final String cors = tsdb.getConfig().getString("tsd.http.request.cors_domains"); - final String mode = tsdb.getConfig().getString("tsd.mode"); - LOG.info("TSD is in " + mode + " mode"); + LOG.info("TSD is in " + tsdb.getMode() + " mode"); if (cors == null || cors.isEmpty()) { cors_domains = null; @@ -109,64 +119,6 @@ public RpcHandler(final TSDB tsdb) { } else { LOG.info("Loaded CORS headers (" + cors_headers + ")"); } - - telnet_commands = new HashMap<String, TelnetRpc>(); - http_commands = new HashMap<String, HttpRpc>(); - if (mode.equals("rw") || mode.equals("wo")) { - final PutDataPointRpc put = new PutDataPointRpc(); - telnet_commands.put("put", put); - http_commands.put("api/put", put); - } - - if (mode.equals("rw") || mode.equals("ro")) { - http_commands.put("", new HomePage()); - final StaticFileRpc staticfile = new StaticFileRpc(); - http_commands.put("favicon.ico", staticfile); - http_commands.put("s", staticfile); - - final StatsRpc stats = new StatsRpc(); - telnet_commands.put("stats", stats); - http_commands.put("stats", stats); - http_commands.put("api/stats", stats); - - final DropCaches dropcaches = new DropCaches(); - telnet_commands.put("dropcaches", dropcaches); - http_commands.put("dropcaches", dropcaches); - http_commands.put("api/dropcaches", dropcaches); - - final ListAggregators aggregators = new ListAggregators(); - http_commands.put("aggregators", aggregators); - http_commands.put("api/aggregators", aggregators); - - final SuggestRpc suggest_rpc = new SuggestRpc(); - http_commands.put("suggest", suggest_rpc); - http_commands.put("api/suggest", suggest_rpc); - - http_commands.put("logs", new LogsRpc()); - http_commands.put("q", new GraphHandler()); - http_commands.put("api/serializers", new Serializers()); - http_commands.put("api/uid", new UniqueIdRpc()); - http_commands.put("api/query", new QueryRpc()); - http_commands.put("api/tree", new TreeRpc()); - http_commands.put("api/annotation", new AnnotationRpc()); - http_commands.put("api/search", new SearchRpc()); - http_commands.put("api/config", new ShowConfig()); - } - - if (tsdb.getConfig().getString("tsd.no_diediedie").equals("false")) { - final DieDieDie diediedie = new DieDieDie(); - telnet_commands.put("diediedie", diediedie); - http_commands.put("diediedie", diediedie); - } - { - final Version version = new Version(); - telnet_commands.put("version", version); - http_commands.put("version", version); - http_commands.put("api/version", version); - } - - telnet_commands.put("exit", new Exit()); - telnet_commands.put("help", new Help()); } @Override @@ -193,14 +145,14 @@ public void messageReceived(final ChannelHandlerContext ctx, exceptions_caught.incrementAndGet(); } } - + /** * Finds the right handler for a telnet-style RPC and executes it. * @param chan The channel on which the RPC was received. * @param command The split telnet-style command. */ private void handleTelnetRpc(final Channel chan, final String[] command) { - TelnetRpc rpc = telnet_commands.get(command[0]); + TelnetRpc rpc = rpc_manager.lookupTelnetRpc(command[0]); if (rpc == null) { rpc = unknown_cmd; } @@ -209,76 +161,167 @@ private void handleTelnetRpc(final Channel chan, final String[] command) { } /** - * Finds the right handler for an HTTP query and executes it. - * Also handles simple and pre-flight CORS requests if configured, rejecting - * requests that do not match a domain in the list. + * Using the request URI, creates a query instance capable of handling + * the given request. + * @param tsdb the TSDB instance we are running within + * @param request the incoming HTTP request + * @param chan the {@link Channel} the request came in on. + * @return a subclass of {@link AbstractHttpQuery} + * @throws BadRequestException if the request is invalid in a way that + * can be detected early, here. + */ + private AbstractHttpQuery createQueryInstance(final TSDB tsdb, + final HttpRequest request, + final Channel chan) + throws BadRequestException { + final String uri = request.getUri(); + if (Strings.isNullOrEmpty(uri)) { + throw new BadRequestException("Request URI is empty"); + } else if (uri.charAt(0) != '/') { + throw new BadRequestException("Request URI doesn't start with a slash"); + } else if (rpc_manager.isHttpRpcPluginPath(uri)) { + http_plugin_rpcs_received.incrementAndGet(); + return new HttpRpcPluginQuery(tsdb, request, chan); + } else { + http_rpcs_received.incrementAndGet(); + HttpQuery builtinQuery = new HttpQuery(tsdb, request, chan); + return builtinQuery; + } + } + + /** + * Helper method to apply CORS configuration to a request, either a built-in + * RPC or a user plugin. + * @return <code>true</code> if a status reply was sent (in the the case of + * certain HTTP methods); <code>false</code> otherwise. + */ + private boolean applyCorsConfig(final HttpRequest req, final AbstractHttpQuery query) + throws BadRequestException { + final String domain = req.headers().get("Origin"); + + // catch CORS requests and add the header or refuse them if the domain + // list has been configured + if (query.method() == HttpMethod.OPTIONS || + (cors_domains != null && domain != null && !domain.isEmpty())) { + if (cors_domains == null || domain == null || domain.isEmpty()) { + throw new BadRequestException(HttpResponseStatus.METHOD_NOT_ALLOWED, + "Method not allowed", "The HTTP method [" + + query.method().getName() + "] is not permitted"); + } + + if (cors_domains.contains("*") || + cors_domains.contains(domain.toUpperCase())) { + + // when a domain has matched successfully, we need to add the header + query.response().headers().add(HttpHeaders.ACCESS_CONTROL_ALLOW_ORIGIN, + domain); + query.response().headers().add(HttpHeaders.ACCESS_CONTROL_ALLOW_METHODS, + "GET, POST, PUT, DELETE"); + query.response().headers().add(HttpHeaders.ACCESS_CONTROL_ALLOW_HEADERS, + cors_headers); + + // if the method requested was for OPTIONS then we'll return an OK + // here and no further processing is needed. + if (query.method() == HttpMethod.OPTIONS) { + query.sendStatusOnly(HttpResponseStatus.OK); + return true; + } + } else { + // You'd think that they would want the server to return a 403 if + // the Origin wasn't in the CORS domain list, but they want a 200 + // without the allow origin header. We'll return an error in the + // body though. + throw new BadRequestException(HttpResponseStatus.OK, + "CORS domain not allowed", "The domain [" + domain + + "] is not permitted access"); + } + } + return false; + } + + /** + * Finds the right handler for an HTTP query (either built-in or user plugin) + * and executes it. Also handles simple and pre-flight CORS requests if + * configured, rejecting requests that do not match a domain in the list. * @param chan The channel on which the query was received. * @param req The parsed HTTP request. */ private void handleHttpQuery(final TSDB tsdb, final Channel chan, final HttpRequest req) { - http_rpcs_received.incrementAndGet(); - final HttpQuery query = new HttpQuery(tsdb, req, chan); - if (!tsdb.getConfig().enable_chunked_requests() && req.isChunked()) { - logError(query, "Received an unsupported chunked request: " - + query.request()); - query.badRequest("Chunked request not supported."); - return; - } + // quick bail if not GET/POST/OPTIONS/PUT/DELETE, no other methods are allowed anywhere try { - final String route = query.getQueryBaseRoute(); - query.setSerializer(); - - final String domain = req.headers().get("Origin"); + RpcUtil.allowedMethods(req.getMethod(), HttpMethod.GET.getName(), HttpMethod.POST.getName(), + HttpMethod.OPTIONS.getName(), HttpMethod.PUT.getName(), HttpMethod.DELETE.getName()); + } catch (BadRequestException bre) { + sendStatusAndClose(chan, HttpResponseStatus.METHOD_NOT_ALLOWED); + } - // catch CORS requests and add the header or refuse them if the domain - // list has been configured - if (query.method() == HttpMethod.OPTIONS || - (cors_domains != null && domain != null && !domain.isEmpty())) { - if (cors_domains == null || domain == null || domain.isEmpty()) { - throw new BadRequestException(HttpResponseStatus.METHOD_NOT_ALLOWED, - "Method not allowed", "The HTTP method [" + - query.method().getName() + "] is not permitted"); + AbstractHttpQuery abstractQuery = null; + try { + abstractQuery = createQueryInstance(tsdb, req, chan); + if (!tsdb.getConfig().enable_chunked_requests() && req.isChunked()) { + logError(abstractQuery, "Received an unsupported chunked request: " + + abstractQuery.request()); + abstractQuery.badRequest(new BadRequestException("Chunked request not supported.")); + return; + } + // NOTE: Some methods in HttpQuery have side-effects (getQueryBaseRoute and + // setSerializer for instance) so invocation order is important here. + final String route = abstractQuery.getQueryBaseRoute(); + if (abstractQuery.getClass().isAssignableFrom(HttpRpcPluginQuery.class)) { + if (applyCorsConfig(req, abstractQuery)) { + return; } - - if (cors_domains.contains("*") || - cors_domains.contains(domain.toUpperCase())) { - - // when a domain has matched successfully, we need to add the header - query.response().headers().add(HttpHeaders.ACCESS_CONTROL_ALLOW_ORIGIN, - domain); - query.response().headers().add(HttpHeaders.ACCESS_CONTROL_ALLOW_METHODS, - "GET, POST, PUT, DELETE"); - query.response().headers().add(HttpHeaders.ACCESS_CONTROL_ALLOW_HEADERS, - cors_headers); - - // if the method requested was for OPTIONS then we'll return an OK - // here and no further processing is needed. - if (query.method() == HttpMethod.OPTIONS) { - query.sendStatusOnly(HttpResponseStatus.OK); - return; - } + final HttpRpcPluginQuery pluginQuery = (HttpRpcPluginQuery) abstractQuery; + final HttpRpcPlugin rpc = rpc_manager.lookupHttpRpcPlugin(route); + if (rpc != null) { + rpc.execute(tsdb, pluginQuery); + } else { + pluginQuery.notFound(); + } + } else if (abstractQuery.getClass().isAssignableFrom(HttpQuery.class)) { + final HttpQuery builtinQuery = (HttpQuery) abstractQuery; + builtinQuery.setSerializer(); + if (applyCorsConfig(req, abstractQuery)) { + return; + } + final HttpRpc rpc = rpc_manager.lookupHttpRpc(route); + if (rpc != null) { + rpc.execute(tsdb, builtinQuery); } else { - // You'd think that they would want the server to return a 403 if - // the Origin wasn't in the CORS domain list, but they want a 200 - // without the allow origin header. We'll return an error in the - // body though. - throw new BadRequestException(HttpResponseStatus.OK, - "CORS domain not allowed", "The domain [" + domain + - "] is not permitted access"); + builtinQuery.notFound(); } - } - - final HttpRpc rpc = http_commands.get(route); - if (rpc != null) { - rpc.execute(tsdb, query); } else { - query.notFound(); + throw new IllegalStateException("Unknown instance of AbstractHttpQuery: " + + abstractQuery.getClass().getName()); } } catch (BadRequestException ex) { - query.badRequest(ex); + if (abstractQuery == null) { + LOG.warn("{} Unable to create query for {}. Reason: {}", chan, req, ex); + sendStatusAndClose(chan, HttpResponseStatus.BAD_REQUEST); + } else { + abstractQuery.badRequest(ex); + } } catch (Exception ex) { - query.internalError(ex); exceptions_caught.incrementAndGet(); + if (abstractQuery == null) { + LOG.warn("{} Unexpected error handling HTTP request {}. Reason: {} ", chan, req, ex); + sendStatusAndClose(chan, HttpResponseStatus.INTERNAL_SERVER_ERROR); + } else { + abstractQuery.internalError(ex); + } + } + } + + /** + * Helper method for sending a status-only HTTP response. This is used in cases where + * {@link #createQueryInstance(TSDB, HttpRequest, Channel)} failed to determine a query + * and we still want to return an error status to the client. + */ + private void sendStatusAndClose(final Channel chan, final HttpResponseStatus status) { + if (chan.isConnected()) { + final DefaultHttpResponse response = new DefaultHttpResponse(HttpVersion.HTTP_1_1, status); + final ChannelFuture future = chan.write(response); + future.addListener(ChannelFutureListener.CLOSE); } } @@ -289,182 +332,12 @@ private void handleHttpQuery(final TSDB tsdb, final Channel chan, final HttpRequ public static void collectStats(final StatsCollector collector) { collector.record("rpc.received", telnet_rpcs_received, "type=telnet"); collector.record("rpc.received", http_rpcs_received, "type=http"); + collector.record("rpc.received", http_plugin_rpcs_received, "type=http_plugin"); collector.record("rpc.exceptions", exceptions_caught); HttpQuery.collectStats(collector); GraphHandler.collectStats(collector); PutDataPointRpc.collectStats(collector); - } - - // ---------------------------- // - // Individual command handlers. // - // ---------------------------- // - - /** The "diediedie" command and "/diediedie" endpoint. */ - private final class DieDieDie implements TelnetRpc, HttpRpc { - public Deferred<Object> execute(final TSDB tsdb, final Channel chan, - final String[] cmd) { - logWarn(chan, "shutdown requested"); - chan.write("Cleaning up and exiting now.\n"); - return doShutdown(tsdb, chan); - } - - public void execute(final TSDB tsdb, final HttpQuery query) { - logWarn(query, "shutdown requested"); - query.sendReply(HttpQuery.makePage("TSD Exiting", "You killed me", - "Cleaning up and exiting now.")); - doShutdown(tsdb, query.channel()); - } - - private Deferred<Object> doShutdown(final TSDB tsdb, final Channel chan) { - ((GraphHandler) http_commands.get("q")).shutdown(); - ConnectionManager.closeAllConnections(); - // Netty gets stuck in an infinite loop if we shut it down from within a - // NIO thread. So do this from a newly created thread. - final class ShutdownNetty extends Thread { - ShutdownNetty() { - super("ShutdownNetty"); - } - public void run() { - chan.getFactory().releaseExternalResources(); - } - } - new ShutdownNetty().start(); // Stop accepting new connections. - - // Log any error that might occur during shutdown. - final class ShutdownTSDB implements Callback<Exception, Exception> { - public Exception call(final Exception arg) { - LOG.error("Unexpected exception while shutting down", arg); - return arg; - } - public String toString() { - return "shutdown callback"; - } - } - return tsdb.shutdown().addErrback(new ShutdownTSDB()); - } - } - - /** The "exit" command. */ - private static final class Exit implements TelnetRpc { - public Deferred<Object> execute(final TSDB tsdb, final Channel chan, - final String[] cmd) { - chan.disconnect(); - return Deferred.fromResult(null); - } - } - - /** The "help" command. */ - private final class Help implements TelnetRpc { - public Deferred<Object> execute(final TSDB tsdb, final Channel chan, - final String[] cmd) { - final StringBuilder buf = new StringBuilder(); - buf.append("available commands: "); - // TODO(tsuna): Maybe sort them? - for (final String command : telnet_commands.keySet()) { - buf.append(command).append(' '); - } - buf.append('\n'); - chan.write(buf.toString()); - return Deferred.fromResult(null); - } - } - - /** The home page ("GET /"). */ - private static final class HomePage implements HttpRpc { - public void execute(final TSDB tsdb, final HttpQuery query) - throws IOException { - final StringBuilder buf = new StringBuilder(2048); - buf.append("<div id=queryuimain></div>" - + "<noscript>You must have JavaScript enabled.</noscript>" - + "<iframe src=javascript:'' id=__gwt_historyFrame tabIndex=-1" - + " style=position:absolute;width:0;height:0;border:0>" - + "</iframe>"); - query.sendReply(HttpQuery.makePage( - "<script type=text/javascript language=javascript" - + " src=/s/queryui.nocache.js></script>", - "TSD", "Time Series Database", buf.toString())); - } - } - - /** The "/aggregators" endpoint. */ - private static final class ListAggregators implements HttpRpc { - public void execute(final TSDB tsdb, final HttpQuery query) - throws IOException { - - // only accept GET/POST - if (query.method() != HttpMethod.GET && query.method() != HttpMethod.POST) { - throw new BadRequestException(HttpResponseStatus.METHOD_NOT_ALLOWED, - "Method not allowed", "The HTTP method [" + query.method().getName() + - "] is not permitted for this endpoint"); - } - - if (query.apiVersion() > 0) { - query.sendReply( - query.serializer().formatAggregatorsV1(Aggregators.set())); - } else { - query.sendReply(JSON.serializeToBytes(Aggregators.set())); - } - } - } - - /** For unknown commands. */ - private static final class Unknown implements TelnetRpc { - public Deferred<Object> execute(final TSDB tsdb, final Channel chan, - final String[] cmd) { - logWarn(chan, "unknown command : " + Arrays.toString(cmd)); - chan.write("unknown command: " + cmd[0] + ". Try `help'.\n"); - return Deferred.fromResult(null); - } - } - - /** The "version" command. */ - private static final class Version implements TelnetRpc, HttpRpc { - public Deferred<Object> execute(final TSDB tsdb, final Channel chan, - final String[] cmd) { - if (chan.isConnected()) { - chan.write(BuildData.revisionString() + '\n' - + BuildData.buildString() + '\n'); - } - return Deferred.fromResult(null); - } - - public void execute(final TSDB tsdb, final HttpQuery query) throws - IOException { - - // only accept GET/POST - if (query.method() != HttpMethod.GET && query.method() != HttpMethod.POST) { - throw new BadRequestException(HttpResponseStatus.METHOD_NOT_ALLOWED, - "Method not allowed", "The HTTP method [" + query.method().getName() + - "] is not permitted for this endpoint"); - } - - final HashMap<String, String> version = new HashMap<String, String>(); - version.put("version", BuildData.version); - version.put("short_revision", BuildData.short_revision); - version.put("full_revision", BuildData.full_revision); - version.put("timestamp", Long.toString(BuildData.timestamp)); - version.put("repo_status", BuildData.repo_status.toString()); - version.put("user", BuildData.user); - version.put("host", BuildData.host); - version.put("repo", BuildData.repo); - - if (query.apiVersion() > 0) { - query.sendReply(query.serializer().formatVersionV1(version)); - } else { - final boolean json = query.request().getUri().endsWith("json"); - if (json) { - query.sendReply(JSON.serializeToBytes(version)); - } else { - final String revision = BuildData.revisionString(); - final String build = BuildData.buildString(); - StringBuilder buf; - buf = new StringBuilder(2 // For the \n's - + revision.length() + build.length()); - buf.append(revision).append('\n').append(build).append('\n'); - query.sendReply(buf); - } - } - } + QueryRpc.collectStats(collector); } /** @@ -489,92 +362,19 @@ static String getDirectoryFromSystemProp(final String prop) { } return dir; } - - /** The "dropcaches" command. */ - private static final class DropCaches implements TelnetRpc, HttpRpc { + + // ---------------------------- // + // Individual command handlers. // + // ---------------------------- // + + /** For unknown commands. */ + private static final class Unknown implements TelnetRpc { public Deferred<Object> execute(final TSDB tsdb, final Channel chan, final String[] cmd) { - dropCaches(tsdb, chan); - chan.write("Caches dropped.\n"); + logWarn(chan, "unknown command : " + Arrays.toString(cmd)); + chan.write("unknown command: " + cmd[0] + ". Try `help'.\n"); return Deferred.fromResult(null); } - - public void execute(final TSDB tsdb, final HttpQuery query) - throws IOException { - dropCaches(tsdb, query.channel()); - - // only accept GET/POST - if (query.method() != HttpMethod.GET && query.method() != HttpMethod.POST) { - throw new BadRequestException(HttpResponseStatus.METHOD_NOT_ALLOWED, - "Method not allowed", "The HTTP method [" + query.method().getName() + - "] is not permitted for this endpoint"); - } - - if (query.apiVersion() > 0) { - final HashMap<String, String> response = new HashMap<String, String>(); - response.put("status", "200"); - response.put("message", "Caches dropped"); - query.sendReply(query.serializer().formatDropCachesV1(response)); - } else { // deprecated API - query.sendReply("Caches dropped.\n"); - } - } - - /** Drops in memory caches. */ - private void dropCaches(final TSDB tsdb, final Channel chan) { - LOG.warn(chan + " Dropping all in-memory caches."); - tsdb.dropCaches(); - } - } - - /** The /api/formatters endpoint - * @since 2.0 */ - private static final class Serializers implements HttpRpc { - public void execute(final TSDB tsdb, final HttpQuery query) - throws IOException { - // only accept GET/POST - if (query.method() != HttpMethod.GET && query.method() != HttpMethod.POST) { - throw new BadRequestException(HttpResponseStatus.METHOD_NOT_ALLOWED, - "Method not allowed", "The HTTP method [" + query.method().getName() + - "] is not permitted for this endpoint"); - } - - switch (query.apiVersion()) { - case 0: - case 1: - query.sendReply(query.serializer().formatSerializersV1()); - break; - default: - throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, - "Requested API version not implemented", "Version " + - query.apiVersion() + " is not implemented"); - } - } - } - - private static final class ShowConfig implements HttpRpc { - - @Override - public void execute(TSDB tsdb, HttpQuery query) throws IOException { - // only accept GET/POST - if (query.method() != HttpMethod.GET && query.method() != HttpMethod.POST) { - throw new BadRequestException(HttpResponseStatus.METHOD_NOT_ALLOWED, - "Method not allowed", "The HTTP method [" + query.method().getName() + - "] is not permitted for this endpoint"); - } - - switch (query.apiVersion()) { - case 0: - case 1: - query.sendReply(query.serializer().formatConfigV1(tsdb.getConfig())); - break; - default: - throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, - "Requested API version not implemented", "Version " + - query.apiVersion() + " is not implemented"); - } - } - } @Override @@ -586,45 +386,23 @@ public void channelIdle(ChannelHandlerContext ctx, IdleStateEvent e) { LOG.info("Closed idle socket: " + channel_info); } } - + // ---------------- // // Logging helpers. // // ---------------- // - //private static void logInfo(final HttpQuery query, final String msg) { - // LOG.info(query.channel().toString() + ' ' + msg); - //} - - private static void logWarn(final HttpQuery query, final String msg) { + private static void logWarn(final AbstractHttpQuery query, final String msg) { LOG.warn(query.channel().toString() + ' ' + msg); } - //private void logWarn(final HttpQuery query, final String msg, - // final Exception e) { - // LOG.warn(query.channel().toString() + ' ' + msg, e); - //} - - private void logError(final HttpQuery query, final String msg) { + private void logError(final AbstractHttpQuery query, final String msg) { LOG.error(query.channel().toString() + ' ' + msg); } - //private static void logError(final HttpQuery query, final String msg, - // final Exception e) { - // LOG.error(query.channel().toString() + ' ' + msg, e); - //} - - //private void logInfo(final Channel chan, final String msg) { - // LOG.info(chan.toString() + ' ' + msg); - //} - private static void logWarn(final Channel chan, final String msg) { LOG.warn(chan.toString() + ' ' + msg); } - //private void logWarn(final Channel chan, final String msg, final Exception e) { - // LOG.warn(chan.toString() + ' ' + msg, e); - //} - private void logError(final Channel chan, final String msg) { LOG.error(chan.toString() + ' ' + msg); } diff --git a/src/tsd/RpcManager.java b/src/tsd/RpcManager.java new file mode 100644 index 0000000000..32ccfff40b --- /dev/null +++ b/src/tsd/RpcManager.java @@ -0,0 +1,829 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2010-2014 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.tsd; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Pattern; +import com.google.common.collect.Table; +import net.opentsdb.core.TSDB.TableAvailability; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.Atomics; +import com.stumbleupon.async.Callback; +import com.stumbleupon.async.Deferred; + +import org.jboss.netty.channel.Channel; +import org.jboss.netty.handler.codec.http.HttpMethod; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import net.opentsdb.tools.BuildData; +import net.opentsdb.core.Aggregators; +import net.opentsdb.core.TSDB; +import net.opentsdb.core.TSDB.OperationMode; +import net.opentsdb.query.filter.TagVFilter; +import net.opentsdb.stats.StatsCollector; +import net.opentsdb.utils.Config; +import net.opentsdb.utils.JSON; +import net.opentsdb.utils.PluginLoader; + +/** + * Manager for the lifecycle of <code>HttpRpc</code>s, <code>TelnetRpc</code>s, + * <code>RpcPlugin</code>s, and <code>HttpRpcPlugin</code>. This is a + * singleton. Its lifecycle must be managed by the "container". If you are + * launching via {@code TSDMain} then shutdown (and non-lazy initialization) + * is taken care of. Outside of the use of {@code TSDMain}, you are responsible + * for shutdown, at least. + * + * <p> Here's an example of how to correctly handle shutdown manually: + * + * <pre> + * // Startup our TSDB instance... + * TSDB tsdb_instance = ...; + * + * // ... later, during shtudown .. + * + * if (RpcManager.isInitialized()) { + * // Check that its actually been initialized. We don't want to + * // create a new instance only to shutdown! + * RpcManager.instance(tsdb_instance).shutdown().join(); + * } + * </pre> + * + * @since 2.2 + */ +public final class RpcManager { + private static final Logger LOG = LoggerFactory.getLogger(RpcManager.class); + + /** This is base path where {@link HttpRpcPlugin}s are rooted. It's used + * to match incoming requests. */ + @VisibleForTesting + protected static final String PLUGIN_BASE_WEBPATH = "plugin"; + + /** Splitter for web paths. Removes empty strings to handle trailing or + * leading slashes. For instance, all of <code>/plugin/mytest</code>, + * <code>plugin/mytest/</code>, and <code>plugin/mytest</code> will be + * split to <code>[plugin, mytest]</code>. */ + private static final Splitter WEBPATH_SPLITTER = Splitter.on('/') + .trimResults() + .omitEmptyStrings(); + + /** Matches paths declared by {@link HttpRpcPlugin}s that are rooted in + * the system's plugins path. */ + private static final Pattern HAS_PLUGIN_BASE_WEBPATH = Pattern.compile( + "^/?" + PLUGIN_BASE_WEBPATH + "/?.*", + Pattern.CASE_INSENSITIVE | Pattern.DOTALL); + + /** Reference to our singleton instance. Set in {@link #initialize}. */ + private static final AtomicReference<RpcManager> INSTANCE = Atomics.newReference(); + + /** Commands we can serve on the simple, telnet-style RPC interface. */ + private ImmutableMap<String, TelnetRpc> telnet_commands; + /** Commands we serve on the HTTP interface. */ + private ImmutableMap<String, HttpRpc> http_commands; + /** HTTP commands from user plugins. */ + private ImmutableMap<String, HttpRpcPlugin> http_plugin_commands; + /** List of activated RPC plugins */ + private ImmutableList<RpcPlugin> rpc_plugins; + /** Status command—we keep a reference so we can explicitly shut it down. */ + private Status status; + + /** The TSDB that owns us. */ + private TSDB tsdb; + + /** + * Constructor used by singleton factory method. + * @param tsdb the owning TSDB instance. + */ + private RpcManager(final TSDB tsdb) { + this.tsdb = tsdb; + } + + /** + * Get or create the singleton instance of the manager, loading all the + * plugins enabled in the given TSDB's {@link Config}. + * @return the shared instance of {@link RpcManager}. It's okay to + * hold this reference once obtained. + */ + public static synchronized RpcManager instance(final TSDB tsdb) { + final RpcManager existing = INSTANCE.get(); + if (existing != null) { + return existing; + } + + final RpcManager manager = new RpcManager(tsdb); + + // Load any plugins that are enabled via Config. Fail if any plugin cannot be loaded. + + final ImmutableList.Builder<RpcPlugin> rpcBuilder = ImmutableList.builder(); + if (tsdb.getConfig().hasProperty("tsd.rpc.plugins")) { + final String[] plugins = tsdb.getConfig().getString("tsd.rpc.plugins").split(","); + manager.initializeRpcPlugins(plugins, rpcBuilder); + } + manager.rpc_plugins = rpcBuilder.build(); + + final ImmutableMap.Builder<String, TelnetRpc> telnetBuilder = ImmutableMap.builder(); + final ImmutableMap.Builder<String, HttpRpc> httpBuilder = ImmutableMap.builder(); + manager.initializeBuiltinRpcs(tsdb.getMode(), telnetBuilder, httpBuilder); + manager.telnet_commands = telnetBuilder.build(); + manager.http_commands = httpBuilder.build(); + + final ImmutableMap.Builder<String, HttpRpcPlugin> httpPluginsBuilder = ImmutableMap.builder(); + if (tsdb.getConfig().hasProperty("tsd.http.rpc.plugins")) { + final String[] plugins = tsdb.getConfig().getString("tsd.http.rpc.plugins").split(","); + manager.initializeHttpRpcPlugins(tsdb.getMode(), plugins, httpPluginsBuilder); + } + manager.http_plugin_commands = httpPluginsBuilder.build(); + + INSTANCE.set(manager); + return manager; + } + + /** + * @return {@code true} if the shared instance has been initialized; + * {@code false} otherwise. + */ + public static synchronized boolean isInitialized() { + return INSTANCE.get() != null; + } + + /** + * @return list of loaded {@link RpcPlugin}s. Possibly empty but + * never {@code null}. + */ + @VisibleForTesting + protected ImmutableList<RpcPlugin> getRpcPlugins() { + return rpc_plugins; + } + + /** + * Lookup a {@link TelnetRpc} based on given command name. Note that this + * lookup is case sensitive in that the {@code command} passed in must + * match a registered RPC command exactly. + * @param command a telnet API command name. + * @return the {@link TelnetRpc} for the given {@code command} or {@code null} + * if not found. + */ + TelnetRpc lookupTelnetRpc(final String command) { + return telnet_commands.get(command); + } + + /** + * Lookup a built-in {@link HttpRpc} based on the given {@code queryBaseRoute}. + * The lookup is based on exact match of the input parameter and the registered + * {@link HttpRpc}s. + * @param queryBaseRoute the HTTP query's base route, with no trailing or + * leading slashes. For example: {@code api/query} + * @return the {@link HttpRpc} for the given {@code queryBaseRoute} or + * {@code null} if not found. + */ + HttpRpc lookupHttpRpc(final String queryBaseRoute) { + return http_commands.get(queryBaseRoute); + } + + /** + * Lookup a user-supplied {@link HttpRpcPlugin} for the given + * {@code queryBaseRoute}. The lookup is based on exact match of the input + * parameter and the registered {@link HttpRpcPlugin}s. + * @param queryBaseRoute the value of {@link HttpRpcPlugin#getPath()} with no + * trailing or leading slashes. + * @return the {@link HttpRpcPlugin} for the given {@code queryBaseRoute} or + * {@code null} if not found. + */ + HttpRpcPlugin lookupHttpRpcPlugin(final String queryBaseRoute) { + return http_plugin_commands.get(queryBaseRoute); + } + + /** + * @param uri HTTP request URI, with or without query parameters. + * @return {@code true} if the URI represents a request for a + * {@link HttpRpcPlugin}; {@code false} otherwise. Note that this + * method returning true <strong>says nothing</strong> about + * whether or not there is a {@link HttpRpcPlugin} registered + * at the given URI, only that it's a valid RPC plugin request. + */ + boolean isHttpRpcPluginPath(final String uri) { + if (Strings.isNullOrEmpty(uri) || uri.length() <= PLUGIN_BASE_WEBPATH.length()) { + return false; + } else { + // Don't consider the query portion, if any. + int qmark = uri.indexOf('?'); + String path = uri; + if (qmark != -1) { + path = uri.substring(0, qmark); + } + + final List<String> parts = WEBPATH_SPLITTER.splitToList(path); + return (parts.size() > 1 && parts.get(0).equals(PLUGIN_BASE_WEBPATH)); + } + } + + /** + * Load and init instances of {@link TelnetRpc}s and {@link HttpRpc}s. + * These are not generally configurable via TSDB config. + * @param telnet a map of telnet command names to {@link TelnetRpc} + * instances. + * @param mode is this TSD in read/write ("rw") or read-only ("ro") + * mode? + * @param http a map of API endpoints to {@link HttpRpc} instances. + */ + private void initializeBuiltinRpcs(final OperationMode mode, + final ImmutableMap.Builder<String, TelnetRpc> telnet, + final ImmutableMap.Builder<String, HttpRpc> http) { + + final Boolean enableApi = tsdb.getConfig().getString("tsd.core.enable_api").equals("true"); + final Boolean enableUi = tsdb.getConfig().getString("tsd.core.enable_ui").equals("true"); + final Boolean enableDieDieDie = tsdb.getConfig().getString("tsd.no_diediedie").equals("false"); + + LOG.info("Mode: {}, HTTP UI Enabled: {}, HTTP API Enabled: {}", mode, enableUi, enableApi); + + // defaults common to every mode + final StatsRpc stats = new StatsRpc(); + final ListAggregators aggregators = new ListAggregators(); + final DropCachesRpc dropcaches = new DropCachesRpc(); + final Version version = new Version(); + status = new Status(); + + telnet.put("stats", stats); + telnet.put("dropcaches", dropcaches); + telnet.put("version", version); + telnet.put("status", status); + telnet.put("exit", new Exit()); + telnet.put("help", new Help()); + + if (enableUi) { + http.put("aggregators", aggregators); + http.put("logs", new LogsRpc()); + http.put("stats", stats); + http.put("version", version); + } + + if (enableApi) { + http.put("api/aggregators", aggregators); + http.put("api/config", new ShowConfig()); + http.put("api/dropcaches", dropcaches); + http.put("api/stats", stats); + http.put("api/version", version); + http.put("api/status", status); + } + + final PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + final RollupDataPointRpc rollups = new RollupDataPointRpc(tsdb.getConfig()); + final HistogramDataPointRpc histos = new HistogramDataPointRpc(tsdb.getConfig()); + final SuggestRpc suggest_rpc = new SuggestRpc(); + final AnnotationRpc annotation_rpc = new AnnotationRpc(); + final StaticFileRpc staticfile = new StaticFileRpc(); + + switch(mode) { + case WRITEONLY: + telnet.put("put", put); + telnet.put("rollup", rollups); + telnet.put("histogram", histos); + + if (enableApi) { + http.put("api/annotation", annotation_rpc); + http.put("api/annotations", annotation_rpc); + http.put("api/put", put); + http.put("api/rollup", rollups); + http.put("api/histogram", histos); + http.put("api/tree", new TreeRpc()); + http.put("api/uid", new UniqueIdRpc(mode)); + } + break; + case READONLY: + if (enableUi) { + http.put("", new HomePage()); + http.put("s", staticfile); + http.put("favicon.ico", staticfile); + http.put("suggest", suggest_rpc); + http.put("q", new GraphHandler()); + } + + if (enableApi) { + http.put("api/query", new QueryRpc()); + http.put("api/search", new SearchRpc()); + http.put("api/suggest", suggest_rpc); + http.put("api/uid", new UniqueIdRpc(mode)); + } + + break; + case READWRITE: + telnet.put("put", put); + telnet.put("rollup", rollups); + telnet.put("histogram", histos); + + if (enableUi) { + http.put("", new HomePage()); + http.put("s", staticfile); + http.put("favicon.ico", staticfile); + http.put("suggest", suggest_rpc); + http.put("q", new GraphHandler()); + } + + if (enableApi) { + http.put("api/query", new QueryRpc()); + http.put("api/search", new SearchRpc()); + http.put("api/annotation", annotation_rpc); + http.put("api/annotations", annotation_rpc); + http.put("api/suggest", suggest_rpc); + http.put("api/put", put); + http.put("api/rollup", rollups); + http.put("api/histogram", histos); + http.put("api/tree", new TreeRpc()); + http.put("api/uid", new UniqueIdRpc(mode)); + } + } + + if (enableDieDieDie) { + final DieDieDie diediedie = new DieDieDie(); + telnet.put("diediedie", diediedie); + if (enableUi) { + http.put("diediedie", diediedie); + } + } + } + + /** + * Load and init the {@link HttpRpcPlugin}s provided as an array of + * {@code pluginClassNames}. + * @param mode is this TSD in read/write ("rw") or read-only ("ro") + * mode? + * @param pluginClassNames fully-qualified class names that are + * instances of {@link HttpRpcPlugin}s + * @param http a map of canonicalized paths + * (obtained via {@link #canonicalizePluginPath(String)}) + * to {@link HttpRpcPlugin} instance. + */ + @VisibleForTesting + protected void initializeHttpRpcPlugins(final OperationMode mode, + final String[] pluginClassNames, + final ImmutableMap.Builder<String, HttpRpcPlugin> http) { + + for (final String plugin : pluginClassNames) { + final HttpRpcPlugin rpc = createAndInitialize(plugin, HttpRpcPlugin.class); + validateHttpRpcPluginPath(rpc.getPath()); + final String path = rpc.getPath().trim(); + final String canonicalized_path = canonicalizePluginPath(path); + http.put(canonicalized_path, rpc); + LOG.info("Mounted HttpRpcPlugin [{}] at path \"{}\"", rpc.getClass().getName(), canonicalized_path); + } + } + + /** + * Ensure that the given path for an {@link HttpRpcPlugin} is valid. This + * method simply returns for valid inputs; throws and exception otherwise. + * @param path a request path, no query parameters, etc. + * @throws IllegalArgumentException on invalid paths. + */ + @VisibleForTesting + protected void validateHttpRpcPluginPath(final String path) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(path), + "Invalid HttpRpcPlugin path. Path is null or empty."); + final String testPath = path.trim(); + Preconditions.checkArgument(!HAS_PLUGIN_BASE_WEBPATH.matcher(path).matches(), + "Invalid HttpRpcPlugin path %s. Path contains system's plugin base path.", + testPath); + + URI uri = URI.create(testPath); + Preconditions.checkArgument(!Strings.isNullOrEmpty(uri.getPath()), + "Invalid HttpRpcPlugin path %s. Parsed path is null or empty.", testPath); + Preconditions.checkArgument(!uri.getPath().equals("/"), + "Invalid HttpRpcPlugin path %s. Path is equal to root.", testPath); + Preconditions.checkArgument(Strings.isNullOrEmpty(uri.getQuery()), + "Invalid HttpRpcPlugin path %s. Path contains query parameters.", testPath); + } + + /** + * @param origPath a request path, no query parameters, etc. + * @return a canonical representation of the input, with trailing and leading + * slashes removed. + * @throws IllegalArgumentException if the given path is a root. + */ + @VisibleForTesting + protected String canonicalizePluginPath(final String origPath) { + Preconditions.checkArgument(!(Strings.isNullOrEmpty(origPath) || origPath.equals("/")), + "Path %s is a root.", origPath); + String new_path = origPath; + if (new_path.startsWith("/")) { + new_path = new_path.substring(1); + } + if (new_path.endsWith("/")) { + new_path = new_path.substring(0, new_path.length()-1); + } + return new_path; + } + + /** + * Load and init the {@link RpcPlugin}s provided as an array of + * {@code pluginClassNames}. + * @param pluginClassNames fully-qualified class names that are + * instances of {@link RpcPlugin}s + * @param rpcs a list of loaded and initialized plugins + */ + private void initializeRpcPlugins(final String[] pluginClassNames, + final ImmutableList.Builder<RpcPlugin> rpcs) { + for (final String plugin : pluginClassNames) { + final RpcPlugin rpc = createAndInitialize(plugin, RpcPlugin.class); + rpcs.add(rpc); + } + } + + /** + * Helper method to load and initialize a given plugin class. This uses reflection + * because plugins share no common interfaces. (They could though!) + * @param pluginClassName the class name of the plugin to load + * @param pluginClass class of the plugin + * @return loaded an initialized instance of {@code pluginClass} + */ + @VisibleForTesting + protected <T> T createAndInitialize(final String pluginClassName, final Class<T> pluginClass) { + final T instance = PluginLoader.loadSpecificPlugin(pluginClassName, pluginClass); + Preconditions.checkState(instance != null, + "Unable to locate %s using name '%s", pluginClass, pluginClassName); + try { + final Method initMeth = instance.getClass().getMethod("initialize", TSDB.class); + initMeth.invoke(instance, tsdb); + final Method versionMeth = instance.getClass().getMethod("version"); + String version = (String) versionMeth.invoke(instance); + LOG.info("Successfully initialized plugin [{}] version: {}", + instance.getClass().getCanonicalName(), + version); + return instance; + } catch (Exception e) { + throw new RuntimeException("Failed to initialize " + instance.getClass(), e); + } + } + + /** + * Called to gracefully shutdown the plugin. Implementations should close + * any IO they have open + * @return A deferred object that indicates the completion of the request. + * The {@link Object} has not special meaning and can be {@code null} + * (think of it as {@code Deferred<Void>}). + */ + public Deferred<ArrayList<Object>> shutdown() { + status.shutdown(); + + // Clear shared instance. + INSTANCE.set(null); + + final Collection<Deferred<Object>> deferreds = Lists.newArrayList(); + + if (http_plugin_commands != null) { + for (final Map.Entry<String, HttpRpcPlugin> entry : http_plugin_commands.entrySet()) { + deferreds.add(entry.getValue().shutdown()); + } + } + + if (rpc_plugins != null) { + for (final RpcPlugin rpc : rpc_plugins) { + deferreds.add(rpc.shutdown()); + } + } + + return Deferred.groupInOrder(deferreds); + } + + /** + * Collect stats on the shared instance of {@link RpcManager}. + */ + static void collectStats(final StatsCollector collector) { + final RpcManager manager = INSTANCE.get(); + if (manager != null) { + if (manager.rpc_plugins != null) { + try { + collector.addExtraTag("plugin", "rpc"); + for (final RpcPlugin rpc : manager.rpc_plugins) { + rpc.collectStats(collector); + } + } finally { + collector.clearExtraTag("plugin"); + } + } + + if (manager.http_plugin_commands != null) { + try { + collector.addExtraTag("plugin", "httprpc"); + for (final Map.Entry<String, HttpRpcPlugin> entry + : manager.http_plugin_commands.entrySet()) { + entry.getValue().collectStats(collector); + } + } finally { + collector.clearExtraTag("plugin"); + } + } + } + } + + // ---------------------------- // + // Individual command handlers. // + // ---------------------------- // + + /** The "diediedie" command and "/diediedie" endpoint. */ + private final class DieDieDie implements TelnetRpc, HttpRpc { + public Deferred<Object> execute(final TSDB tsdb, final Channel chan, + final String[] cmd) { + LOG.warn("{} {}", chan, "shutdown requested"); + chan.write("Cleaning up and exiting now.\n"); + return doShutdown(tsdb, chan); + } + + public void execute(final TSDB tsdb, final HttpQuery query) { + LOG.warn("{} {}", query, "shutdown requested"); + query.sendReply(HttpQuery.makePage("TSD Exiting", "You killed me", + "Cleaning up and exiting now.")); + doShutdown(tsdb, query.channel()); + } + + private Deferred<Object> doShutdown(final TSDB tsdb, final Channel chan) { + ((GraphHandler) http_commands.get("q")).shutdown(); + ConnectionManager.closeAllConnections(); + // Netty gets stuck in an infinite loop if we shut it down from within a + // NIO thread. So do this from a newly created thread. + final class ShutdownNetty extends Thread { + ShutdownNetty() { + super("ShutdownNetty"); + } + public void run() { + chan.getFactory().releaseExternalResources(); + } + } + new ShutdownNetty().start(); // Stop accepting new connections. + + // Log any error that might occur during shutdown. + final class ShutdownTSDB implements Callback<Exception, Exception> { + public Exception call(final Exception arg) { + LOG.error("Unexpected exception while shutting down", arg); + return arg; + } + public String toString() { + return "shutdown callback"; + } + } + return tsdb.shutdown().addErrback(new ShutdownTSDB()); + } + } + + /** The "exit" command. */ + private static final class Exit implements TelnetRpc { + public Deferred<Object> execute(final TSDB tsdb, final Channel chan, + final String[] cmd) { + chan.disconnect(); + return Deferred.fromResult(null); + } + } + + /** The "help" command. */ + private final class Help implements TelnetRpc { + public Deferred<Object> execute(final TSDB tsdb, final Channel chan, + final String[] cmd) { + final StringBuilder buf = new StringBuilder(); + buf.append("available commands: "); + // TODO(tsuna): Maybe sort them? + for (final String command : telnet_commands.keySet()) { + buf.append(command).append(' '); + } + buf.append('\n'); + chan.write(buf.toString()); + return Deferred.fromResult(null); + } + } + + /** The home page ("GET /"). */ + private static final class HomePage implements HttpRpc { + public void execute(final TSDB tsdb, final HttpQuery query) + throws IOException { + final StringBuilder buf = new StringBuilder(2048); + buf.append("<div id=queryuimain></div>" + + "<noscript>You must have JavaScript enabled.</noscript>" + + "<iframe src=javascript:'' id=__gwt_historyFrame tabIndex=-1" + + " style=position:absolute;width:0;height:0;border:0>" + + "</iframe>"); + query.sendReply(HttpQuery.makePage( + "<script type=text/javascript language=javascript" + + " src=s/queryui.nocache.js></script>", + "OpenTSDB", "", buf.toString())); + } + } + + /** The "/aggregators" endpoint. */ + private static final class ListAggregators implements HttpRpc { + public void execute(final TSDB tsdb, final HttpQuery query) + throws IOException { + + // only accept GET + RpcUtil.allowedMethods(query.method(), HttpMethod.GET.getName()); + + if (query.apiVersion() > 0) { + query.sendReply( + query.serializer().formatAggregatorsV1(Aggregators.set())); + } else { + query.sendReply(JSON.serializeToBytes(Aggregators.set())); + } + } + } + + /** The "status" command. */ + static final class Status implements TelnetRpc, HttpRpc { + String status = "startup"; + + /** Called by RpcManager when it is shutdown. */ + public void shutdown() { + status = "shutting-down"; + } + + /** Update status, return Deferred that fires when status is updated. */ + private Deferred<Object> updateStatus(final TSDB tsdb) { + // Once we're in shutdown mode the status never changes. + if (status == "shutting-down") { + return Deferred.fromResult(null); + } + + Deferred<TableAvailability> availability = tsdb.checkNecessaryTablesAvailability(); + + final class AvailabilityToStatusCB implements Callback<Object,TableAvailability> { + @Override + public Object call(final TableAvailability availability) { + // If we're in startup mode, lack of availability may just be due to + // starting up, so don't consider that an error state. + if ((status == "startup") && (availability == TableAvailability.NONE)) { + return null; + } + + if (availability == TableAvailability.FULL) { + status = "ok"; + } else if (availability == TableAvailability.PARTIAL) { + status = "partial"; + } else { + status = "error"; + } + return null; + } + } + return availability.addCallback(new AvailabilityToStatusCB()); + } + + public Deferred<Object> execute(final TSDB tsdb, final Channel chan, + final String[] cmd) { + final class WriteStatusCB implements Callback<Object,Object> { + @Override + public Object call(final Object o) { + if (chan.isConnected()) { + chan.write(status + '\n'); + } + return null; + } + } + + return updateStatus(tsdb).addCallback(new WriteStatusCB()); + } + + public void execute(final TSDB tsdb, final HttpQuery query) throws + IOException { + // only accept GET + RpcUtil.allowedMethods(query.method(), HttpMethod.GET.getName()); + + final class WriteStatusCB implements Callback<Object,Object> { + @Override + public Object call(final Object o) { + final HashMap<String, String> result = new HashMap<String, String>(); + result.put("status", status); + query.sendReply(JSON.serializeToBytes(result)); + return null; + } + } + + updateStatus(tsdb).addCallback(new WriteStatusCB()); + } + } + + /** The "version" command. */ + private static final class Version implements TelnetRpc, HttpRpc { + public Deferred<Object> execute(final TSDB tsdb, final Channel chan, + final String[] cmd) { + if (chan.isConnected()) { + chan.write(BuildData.revisionString() + '\n' + + BuildData.buildString() + '\n'); + } + return Deferred.fromResult(null); + } + + public void execute(final TSDB tsdb, final HttpQuery query) throws + IOException { + + // only accept GET + RpcUtil.allowedMethods(query.method(), HttpMethod.GET.getName()); + + final HashMap<String, String> version = new HashMap<String, String>(); + version.put("version", BuildData.version); + version.put("short_revision", BuildData.short_revision); + version.put("full_revision", BuildData.full_revision); + version.put("timestamp", Long.toString(BuildData.timestamp)); + version.put("repo_status", BuildData.repo_status.toString()); + version.put("user", BuildData.user); + version.put("host", BuildData.host); + version.put("repo", BuildData.repo); + version.put("branch", BuildData.branch); + + if (query.apiVersion() > 0) { + query.sendReply(query.serializer().formatVersionV1(version)); + } else { + final boolean json = query.request().getUri().endsWith("json"); + if (json) { + query.sendReply(JSON.serializeToBytes(version)); + } else { + final String revision = BuildData.revisionString(); + final String build = BuildData.buildString(); + StringBuilder buf; + buf = new StringBuilder(2 // For the \n's + + revision.length() + build.length()); + buf.append(revision).append('\n').append(build).append('\n'); + query.sendReply(buf); + } + } + } + } + + /** The /api/formatters endpoint + * @since 2.0 */ + private static final class Serializers implements HttpRpc { + public void execute(final TSDB tsdb, final HttpQuery query) + throws IOException { + // only accept GET / POST + RpcUtil.allowedMethods(query.method(), HttpMethod.GET.getName(), HttpMethod.POST.getName()); + + switch (query.apiVersion()) { + case 0: + case 1: + query.sendReply(query.serializer().formatSerializersV1()); + break; + default: + throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, + "Requested API version not implemented", "Version " + + query.apiVersion() + " is not implemented"); + } + } + } + + private static final class ShowConfig implements HttpRpc { + @Override + public void execute(TSDB tsdb, HttpQuery query) throws IOException { + // only accept GET/POST + RpcUtil.allowedMethods(query.method(), HttpMethod.GET.getName(), HttpMethod.POST.getName()); + + final String[] uri = query.explodeAPIPath(); + final String endpoint = uri.length > 1 ? uri[1].toLowerCase() : ""; + + if (endpoint.equals("filters")) { + switch (query.apiVersion()) { + case 0: + case 1: + query.sendReply(query.serializer().formatFilterConfigV1( + TagVFilter.loadedFilters())); + break; + default: + throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, + "Requested API version not implemented", "Version " + + query.apiVersion() + " is not implemented"); + } + } else { + switch (query.apiVersion()) { + case 0: + case 1: + query.sendReply(query.serializer().formatConfigV1(tsdb.getConfig())); + break; + default: + throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, + "Requested API version not implemented", "Version " + + query.apiVersion() + " is not implemented"); + } + } + } + } + +} diff --git a/src/tsd/RpcPlugin.java b/src/tsd/RpcPlugin.java index 8b053945cf..fd97ea9759 100644 --- a/src/tsd/RpcPlugin.java +++ b/src/tsd/RpcPlugin.java @@ -45,7 +45,7 @@ public abstract class RpcPlugin { * @param tsdb The parent TSDB object * @throws IllegalArgumentException if required configuration parameters are * missing - * @throws Exception if something else goes wrong + * @throws RuntimeException if something else goes wrong */ public abstract void initialize(final TSDB tsdb); diff --git a/src/tsd/RpcUtil.java b/src/tsd/RpcUtil.java new file mode 100644 index 0000000000..07660e208f --- /dev/null +++ b/src/tsd/RpcUtil.java @@ -0,0 +1,37 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2010-2014 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.tsd; + +import org.jboss.netty.handler.codec.http.HttpMethod; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +public class RpcUtil { + + private static final Logger LOG = LoggerFactory.getLogger(RpcUtil.class); + + public static void allowedMethods(HttpMethod requestMethod, String... allowedMethods) { + for(String method : allowedMethods) { + LOG.debug(String.format("Trying Method: %s", method)); + if (requestMethod.getName() == method) { + LOG.debug(String.format("Method Allowed: %s", method)); + return; + } + } + throw new BadRequestException(HttpResponseStatus.METHOD_NOT_ALLOWED, + "Method not allowed", "The HTTP method [" + requestMethod.getName() + "] is not permitted for this endpoint"); + } +} diff --git a/src/tsd/SearchRpc.java b/src/tsd/SearchRpc.java index 3b40425bff..4b15856b03 100644 --- a/src/tsd/SearchRpc.java +++ b/src/tsd/SearchRpc.java @@ -13,13 +13,19 @@ package net.opentsdb.tsd; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import org.jboss.netty.handler.codec.http.HttpMethod; import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import com.stumbleupon.async.Callback; +import com.stumbleupon.async.Deferred; +import com.stumbleupon.async.DeferredGroupException; + import net.opentsdb.core.RowKey; import net.opentsdb.core.TSDB; import net.opentsdb.core.Tags; @@ -29,6 +35,8 @@ import net.opentsdb.uid.NoSuchUniqueId; import net.opentsdb.uid.NoSuchUniqueName; import net.opentsdb.uid.UniqueId; +import net.opentsdb.uid.UniqueId.UniqueIdType; +import net.opentsdb.utils.Exceptions; import net.opentsdb.utils.Pair; /** @@ -47,12 +55,9 @@ final class SearchRpc implements HttpRpc { */ @Override public void execute(TSDB tsdb, HttpQuery query) { - final HttpMethod method = query.getAPIMethod(); - if (method != HttpMethod.GET && method != HttpMethod.POST) { - throw new BadRequestException("Unsupported method: " + method.getName()); - } - + RpcUtil.allowedMethods(method, HttpMethod.GET.getName(), HttpMethod.POST.getName()); + // the uri will be /api/vX/search/<type> or /api/search/<type> final String[] uri = query.explodeAPIPath(); final String endpoint = uri.length > 1 ? uri[1] : ""; @@ -109,6 +114,15 @@ private final SearchQuery parseQueryString(final HttpQuery query, } catch (IllegalArgumentException e) { throw new BadRequestException("Unable to parse query", e); } + if (query.hasQueryStringParam("limit")) { + final String limit = query.getQueryStringParam("limit"); + try { + search_query.setLimit(Integer.parseInt(limit)); + } catch (NumberFormatException e) { + throw new BadRequestException( + "Unable to convert 'limit' to a valid number"); + } + } return search_query; } @@ -135,6 +149,11 @@ private final SearchQuery parseQueryString(final HttpQuery query, } } + if (query.hasQueryStringParam("use_meta")) { + search_query.setUseMeta(Boolean.parseBoolean( + query.getQueryStringParam("use_meta"))); + } + return search_query; } @@ -157,44 +176,120 @@ private void processLookup(final TSDB tsdb, final HttpQuery query, "Missing metric and tags. Please supply at least one value."); } final long start = System.currentTimeMillis(); - try { - final List<byte[]> tsuids = - new TimeSeriesLookup(tsdb, search_query).lookup(); - - search_query.setTotalResults(tsuids.size()); - // TODO maybe track in nanoseconds so we can get a floating point. But most - // lookups will probably take a fair amount of time. - search_query.setTime(System.currentTimeMillis() - start); + + class MetricCB implements Callback<Object, String> { + final Map<String, Object> series; + MetricCB(final Map<String, Object> series) { + this.series = series; + } - final List<Object> results = new ArrayList<Object>(tsuids.size()); + @Override + public Object call(final String name) throws Exception { + series.put("metric", name); + return null; + } + } + + class TagsCB implements Callback<Object, HashMap<String, String>> { + final Map<String, Object> series; + TagsCB(final Map<String, Object> series) { + this.series = series; + } - Map<String, Object> series; - List<byte[]> tag_ids; + @Override + public Object call(final HashMap<String, String> names) throws Exception { + series.put("tags", names); + return null; + } + } + + class Serialize implements Callback<Object, ArrayList<Object>> { + final List<Object> results; + Serialize(final List<Object> results) { + this.results = results; + } - // TODO - honor limit and pagination - for (final byte[] tsuid : tsuids) { - series = new HashMap<String, Object>((tsuid.length / 2) + 1); - try { + @Override + public Object call(final ArrayList<Object> ignored) throws Exception { + search_query.setResults(results); + search_query.setTime(System.currentTimeMillis() - start); + query.sendReply(query.serializer().formatSearchResultsV1(search_query)); + return null; + } + } + + class LookupCB implements Callback<Deferred<Object>, List<byte[]>> { + @Override + public Deferred<Object> call(final List<byte[]> tsuids) throws Exception { + final List<Object> results = new ArrayList<Object>(tsuids.size()); + search_query.setTotalResults(tsuids.size()); + + final ArrayList<Deferred<Object>> deferreds = + new ArrayList<Deferred<Object>>(tsuids.size()); + + for (final byte[] tsuid : tsuids) { + // has to be concurrent if the uid table is split across servers + final Map<String, Object> series = + new ConcurrentHashMap<String, Object>(3); + results.add(series); + series.put("tsuid", UniqueId.uidToString(tsuid)); - series.put("metric", RowKey.metricNameAsync(tsdb, tsuid) - .joinUninterruptibly()); - tag_ids = UniqueId.getTagPairsFromTSUID(tsuid); - series.put("tags", Tags.resolveIdsAsync(tsdb, tag_ids) - .joinUninterruptibly()); - } catch (NoSuchUniqueId nsui) { - throw new BadRequestException(HttpResponseStatus.NOT_FOUND, - "Unable to resolve one or more UIDs", nsui); - } catch (Exception e) { - throw new RuntimeException("Shouldn't be here", e); + byte[] metric_uid = Arrays.copyOfRange(tsuid, 0, TSDB.metrics_width()); + deferreds.add(tsdb.getUidName(UniqueIdType.METRIC, metric_uid) + .addCallback(new MetricCB(series))); + + final List<byte[]> tag_ids = UniqueId.getTagPairsFromTSUID(tsuid); + deferreds.add(Tags.resolveIdsAsync(tsdb, tag_ids) + .addCallback(new TagsCB(series))); } - results.add(series); + + return Deferred.group(deferreds).addCallback(new Serialize(results)); } - - search_query.setResults(results); - query.sendReply(query.serializer().formatSearchResultsV1(search_query)); - } catch (NoSuchUniqueName nsun) { - throw new BadRequestException(HttpResponseStatus.NOT_FOUND, - "Unable to resolve one or more names", nsun); } + + class ErrCB implements Callback<Object, Exception> { + @Override + public Object call(final Exception e) throws Exception { + if (e instanceof NoSuchUniqueId) { + query.sendReply(HttpResponseStatus.NOT_FOUND, + query.serializer().formatErrorV1( + new BadRequestException(HttpResponseStatus.NOT_FOUND, + "Unable to resolve one or more TSUIDs", (NoSuchUniqueId)e))); + } else if (e instanceof NoSuchUniqueName) { + query.sendReply(HttpResponseStatus.NOT_FOUND, + query.serializer().formatErrorV1( + new BadRequestException(HttpResponseStatus.NOT_FOUND, + "Unable to resolve one or more UIDs", (NoSuchUniqueName)e))); + } else if (e instanceof DeferredGroupException) { + final Throwable ex = Exceptions.getCause((DeferredGroupException)e); + if (ex instanceof NoSuchUniqueId) { + query.sendReply(HttpResponseStatus.NOT_FOUND, + query.serializer().formatErrorV1( + new BadRequestException(HttpResponseStatus.NOT_FOUND, + "Unable to resolve one or more TSUIDs", (NoSuchUniqueId)ex))); + } else if (ex instanceof NoSuchUniqueName) { + query.sendReply(HttpResponseStatus.NOT_FOUND, + query.serializer().formatErrorV1( + new BadRequestException(HttpResponseStatus.NOT_FOUND, + "Unable to resolve one or more UIDs", (NoSuchUniqueName)ex))); + } else { + query.sendReply(HttpResponseStatus.INTERNAL_SERVER_ERROR, + query.serializer().formatErrorV1( + new BadRequestException(HttpResponseStatus.INTERNAL_SERVER_ERROR, + "Unexpected exception", ex))); + } + } else { + query.sendReply(HttpResponseStatus.INTERNAL_SERVER_ERROR, + query.serializer().formatErrorV1( + new BadRequestException(HttpResponseStatus.INTERNAL_SERVER_ERROR, + "Unexpected exception", e))); + } + return null; + } + } + + new TimeSeriesLookup(tsdb, search_query).lookupAsync() + .addCallback(new LookupCB()) + .addErrback(new ErrCB()); } } diff --git a/src/tsd/StaticFileRpc.java b/src/tsd/StaticFileRpc.java index f3f8c552ef..28f4220bf9 100644 --- a/src/tsd/StaticFileRpc.java +++ b/src/tsd/StaticFileRpc.java @@ -15,6 +15,7 @@ import java.io.IOException; import net.opentsdb.core.TSDB; +import org.jboss.netty.handler.codec.http.HttpMethod; /** Implements the "/s" endpoint to serve static files. */ final class StaticFileRpc implements HttpRpc { @@ -26,13 +27,18 @@ public StaticFileRpc() { } public void execute(final TSDB tsdb, final HttpQuery query) - throws IOException { + throws BadRequestException, IOException { + + // only accept GET + RpcUtil.allowedMethods(query.method(), HttpMethod.GET.getName()); + final String uri = query.request().getUri(); if ("/favicon.ico".equals(uri)) { query.sendFile(tsdb.getConfig().getDirectoryName("tsd.http.staticroot") + "/favicon.ico", 31536000 /*=1yr*/); return; } + if (uri.length() < 3) { // Must be at least 3 because of the "/s/". throw new BadRequestException("URI too short <code>" + uri + "</code>"); } @@ -41,6 +47,7 @@ public void execute(final TSDB tsdb, final HttpQuery query) if (uri.indexOf("..", 3) > 0) { throw new BadRequestException("Malformed URI <code>" + uri + "</code>"); } + final int questionmark = uri.indexOf('?', 3); final int pathend = questionmark > 0 ? questionmark : uri.length(); query.sendFile(tsdb.getConfig().getDirectoryName("tsd.http.staticroot") diff --git a/src/tsd/StatsRpc.java b/src/tsd/StatsRpc.java index 28bd037c26..7dd67c2503 100644 --- a/src/tsd/StatsRpc.java +++ b/src/tsd/StatsRpc.java @@ -12,18 +12,31 @@ // see <http://www.gnu.org/licenses/>. package net.opentsdb.tsd; +import java.io.IOException; +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryPoolMXBean; +import java.lang.management.OperatingSystemMXBean; +import java.lang.management.RuntimeMXBean; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Set; import net.opentsdb.core.IncomingDataPoint; import net.opentsdb.core.TSDB; +import net.opentsdb.stats.QueryStats; import net.opentsdb.stats.StatsCollector; import net.opentsdb.utils.JSON; +import org.hbase.async.RegionClientStats; import org.jboss.netty.channel.Channel; import org.jboss.netty.handler.codec.http.HttpMethod; import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.stumbleupon.async.Deferred; @@ -36,7 +49,8 @@ * @since 2.0 */ public final class StatsRpc implements TelnetRpc, HttpRpc { - + private static final Logger LOG = LoggerFactory.getLogger(StatsRpc.class); + /** * Telnet RPC responder that returns the stats in ASCII style * @param tsdb The TSDB to use for fetching stats @@ -54,16 +68,35 @@ public Deferred<Object> execute(final TSDB tsdb, final Channel chan, } /** - * HTTP resposne handler + * HTTP response handler * @param tsdb The TSDB to which we belong * @param query The query to parse and respond to */ - public void execute(final TSDB tsdb, final HttpQuery query) { - // only accept GET/POST - if (query.method() != HttpMethod.GET && query.method() != HttpMethod.POST) { - throw new BadRequestException(HttpResponseStatus.METHOD_NOT_ALLOWED, - "Method not allowed", "The HTTP method [" + query.method().getName() + - "] is not permitted for this endpoint"); + public void execute(final TSDB tsdb, final HttpQuery query) throws BadRequestException, IOException { + + RpcUtil.allowedMethods(query.method(), HttpMethod.GET.getName(), HttpMethod.POST.getName()); + + try { + final String[] uri = query.explodeAPIPath(); + final String endpoint = uri.length > 1 ? uri[1].toLowerCase() : ""; + + // Handle /threads and /regions. + if ("threads".equals(endpoint)) { + printThreadStats(query); + return; + } else if ("jvm".equals(endpoint)) { + printJVMStats(tsdb, query); + return; + } else if ("query".equals(endpoint)) { + printQueryStats(query); + return; + } else if ("region_clients".equals(endpoint)) { + printRegionClientStats(tsdb, query); + return; + } + } catch (IllegalArgumentException e) { + // this is thrown if the url doesn't start with /api. To maintain backwards + // compatibility with the /stats endpoint we can catch and continue here. } final boolean canonical = tsdb.getConfig().getBoolean("tsd.stats.canonical"); @@ -89,6 +122,7 @@ public void execute(final TSDB tsdb, final HttpQuery query) { canonical); ConnectionManager.collectStats(collector); RpcHandler.collectStats(collector); + RpcManager.collectStats(collector); tsdb.collectStats(collector); query.sendReply(query.serializer().formatStatsV1(dps)); } @@ -103,9 +137,206 @@ private void doCollectStats(final TSDB tsdb, final StatsCollector collector, collector.addHostTag(canonical); ConnectionManager.collectStats(collector); RpcHandler.collectStats(collector); + RpcManager.collectStats(collector); + collectThreadStats(collector); tsdb.collectStats(collector); } + /** + * Display stats for each region client + * @param tsdb The TSDB to use for fetching stats + * @param query The query to respond to + */ + private void printRegionClientStats(final TSDB tsdb, final HttpQuery query) { + final List<RegionClientStats> region_stats = tsdb.getClient().regionStats(); + final List<Map<String, Object>> stats = + new ArrayList<Map<String, Object>>(region_stats.size()); + for (final RegionClientStats rcs : region_stats) { + final Map<String, Object> stat_map = new HashMap<String, Object>(8); + stat_map.put("rpcsSent", rcs.rpcsSent()); + stat_map.put("rpcsInFlight", rcs.inflightRPCs()); + stat_map.put("pendingRPCs", rcs.pendingRPCs()); + stat_map.put("pendingBatchedRPCs", rcs.pendingBatchedRPCs()); + stat_map.put("dead", rcs.isDead()); + stat_map.put("rpcid", rcs.rpcID()); + stat_map.put("endpoint", rcs.remoteEndpoint()); + stat_map.put("rpcsTimedout", rcs.rpcsTimedout()); + stat_map.put("rpcResponsesTimedout", rcs.rpcResponsesTimedout()); + stat_map.put("rpcResponsesUnknown", rcs.rpcResponsesUnknown()); + stat_map.put("inflightBreached", rcs.inflightBreached()); + stat_map.put("pendingBreached", rcs.pendingBreached()); + stat_map.put("writesBlocked", rcs.writesBlocked()); + + stats.add(stat_map); + } + query.sendReply(query.serializer().formatRegionStatsV1(stats)); + } + + + /** + * Grabs a snapshot of all JVM thread states and formats it in a manner to + * be displayed via API. + * @param query The query to respond to + */ + private void printThreadStats(final HttpQuery query) { + final Set<Thread> threads = Thread.getAllStackTraces().keySet(); + final List<Map<String, Object>> output = + new ArrayList<Map<String, Object>>(threads.size()); + for (final Thread thread : threads) { + final Map<String, Object> status = new HashMap<String, Object>(); + status.put("threadID", thread.getId()); + status.put("name", thread.getName()); + status.put("state", thread.getState().toString()); + status.put("interrupted", thread.isInterrupted()); + status.put("priority", thread.getPriority()); + + final List<String> stack = + new ArrayList<String>(thread.getStackTrace().length); + for (final StackTraceElement element: thread.getStackTrace()) { + stack.add(element.toString()); + } + status.put("stack", stack); + output.add(status); + } + query.sendReply(query.serializer().formatThreadStatsV1(output)); + } + + /** + * Yield (chiefly memory-related) stats about this OpenTSDB instance's JVM. + * @param tsdb The TSDB from which to fetch stats. + * @param query The query to which to respond. + */ + private void printJVMStats(final TSDB tsdb, final HttpQuery query) { + final Map<String, Map<String, Object>> map = + new HashMap<String, Map<String, Object>>(); + + final RuntimeMXBean runtime_bean = ManagementFactory.getRuntimeMXBean(); + final Map<String, Object> runtime = new HashMap<String, Object>(); + map.put("runtime", runtime); + + runtime.put("startTime", runtime_bean.getStartTime()); + runtime.put("uptime", runtime_bean.getUptime()); + runtime.put("vmName", runtime_bean.getVmName()); + runtime.put("vmVendor", runtime_bean.getVmVendor()); + runtime.put("vmVersion", runtime_bean.getVmVersion()); + + final MemoryMXBean mem_bean = ManagementFactory.getMemoryMXBean(); + final Map<String, Object> memory = new HashMap<String, Object>(); + map.put("memory", memory); + + memory.put("heapMemoryUsage", mem_bean.getHeapMemoryUsage()); + memory.put("nonHeapMemoryUsage", mem_bean.getNonHeapMemoryUsage()); + memory.put("objectsPendingFinalization", + mem_bean.getObjectPendingFinalizationCount()); + + final List<GarbageCollectorMXBean> gc_beans = + ManagementFactory.getGarbageCollectorMXBeans(); + final Map<String, Object> gc = new HashMap<String, Object>(); + map.put("gc", gc); + + for (final GarbageCollectorMXBean gc_bean : gc_beans) { + final Map<String, Object> stats = new HashMap<String, Object>(); + final String name = formatStatName(gc_bean.getName()); + if (name == null) { + LOG.warn("Null name for bean: " + gc_bean); + continue; + } + + gc.put(name, stats); + stats.put("collectionCount", gc_bean.getCollectionCount()); + stats.put("collectionTime", gc_bean.getCollectionTime()); + } + + final List<MemoryPoolMXBean> pool_beans = + ManagementFactory.getMemoryPoolMXBeans(); + final Map<String, Object> pools = new HashMap<String, Object>(); + map.put("pools", pools); + + for (final MemoryPoolMXBean pool_bean : pool_beans) { + final Map<String, Object> stats = new HashMap<String, Object>(); + final String name = formatStatName(pool_bean.getName()); + if (name == null) { + LOG.warn("Null name for bean: " + pool_bean); + continue; + } + pools.put(name, stats); + + stats.put("collectionUsage", pool_bean.getCollectionUsage()); + stats.put("usage", pool_bean.getUsage()); + stats.put("peakUsage", pool_bean.getPeakUsage()); + stats.put("type", pool_bean.getType()); + } + + final OperatingSystemMXBean os_bean = + ManagementFactory.getOperatingSystemMXBean(); + final Map<String, Object> os = new HashMap<String, Object>(); + map.put("os", os); + + os.put("systemLoadAverage", os_bean.getSystemLoadAverage()); + + query.sendReply(query.serializer().formatJVMStatsV1(map)); + } + + /** + * Runs through the live threads and counts captures a coune of their + * states for dumping in the stats page. + * @param collector The collector to write to + */ + private void collectThreadStats(final StatsCollector collector) { + final Set<Thread> threads = Thread.getAllStackTraces().keySet(); + final Map<String, Integer> states = new HashMap<String, Integer>(6); + states.put("new", 0); + states.put("runnable", 0); + states.put("blocked", 0); + states.put("waiting", 0); + states.put("timed_waiting", 0); + states.put("terminated", 0); + for (final Thread thread : threads) { + int state_count = states.get(thread.getState().toString().toLowerCase()); + state_count++; + states.put(thread.getState().toString().toLowerCase(), state_count); + } + for (final Map.Entry<String, Integer> entry : states.entrySet()) { + collector.record("jvm.thread.states", entry.getValue(), "state=" + + entry.getKey()); + } + collector.record("jvm.thread.count", threads.size()); + } + + /** + * Little helper to convert the first character to lowercase and remove any + * spaces + * @param stat The name to cleanup + * @return a clean name or null if the original string was null or empty + */ + private static String formatStatName(final String stat) { + if (stat == null || stat.isEmpty()) { + return stat; + } + String name = stat.replace(" ", ""); + return name.substring(0, 1).toLowerCase() + name.substring(1); + } + + /** + * Print the detailed query stats to the caller using the proper serializer + * @param query The query to answer to + * @throws BadRequestException if the API version hasn't been implemented + * yet + */ + private void printQueryStats(final HttpQuery query) { + switch (query.apiVersion()) { + case 0: + case 1: + query.sendReply(query.serializer().formatQueryStatsV1( + QueryStats.getRunningAndCompleteStats())); + break; + default: + throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, + "Requested API version not implemented", "Version " + + query.apiVersion() + " is not implemented"); + } + } + /** * Implements the StatsCollector with ASCII style output. Builds a string * buffer response to send to the caller diff --git a/src/tsd/StorageExceptionHandler.java b/src/tsd/StorageExceptionHandler.java new file mode 100644 index 0000000000..7c7a2ee42a --- /dev/null +++ b/src/tsd/StorageExceptionHandler.java @@ -0,0 +1,80 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.tsd; + +import com.stumbleupon.async.Deferred; + +import net.opentsdb.core.IncomingDataPoint; +import net.opentsdb.core.TSDB; +import net.opentsdb.stats.StatsCollector; + +/** + * This is a plugin for handling data points that fail the write to the + * underlying data store for various reasons. For example, HBase may lose a + * region server and a very busy TSD may queue up RPCs in a region, hit the + * high watermark, and start rejecting any data points that would hit the + * region that's offline. In the error callback for each data point, we can + * call into this object and have it queued to disk, send it to an external + * queue or even push it to another TSD. + * @since 2.2 + */ +public abstract class StorageExceptionHandler { + + /** + * Called by TSDB to initialize the plugin + * Implementations are responsible for setting up any IO they need as well + * as starting any required background threads. + * <b>Note:</b> Implementations should throw exceptions if they can't start + * up properly. The TSD will then shutdown so the operator can fix the + * problem. Please use IllegalArgumentException for configuration issues. + * @param tsdb The parent TSDB object + * @throws IllegalArgumentException if required configuration parameters are + * missing + * @throws RuntimeException if something else goes wrong + */ + public abstract void initialize(final TSDB tsdb); + + /** + * Called to gracefully shutdown the plugin. Implementations should close + * any IO they have open + * @return A deferred object that indicates the completion of the request. + * The {@link Object} has not special meaning and can be {@code null} + * (think of it as {@code Deferred<Void>}). + */ + public abstract Deferred<Object> shutdown(); + + /** + * Should return the version of this plugin in the format: + * MAJOR.MINOR.MAINT, e.g. 2.0.1. The MAJOR version should match the major + * version of OpenTSDB the plugin is meant to work with. + * @return A version string used to log the loaded version + */ + public abstract String version(); + + /** + * Called by the TSD when a request for statistics collection has come in. The + * implementation may provide one or more statistics. If no statistics are + * available for the implementation, simply stub the method. + * @param collector The collector used for emitting statistics + */ + public abstract void collectStats(final StatsCollector collector); + + /** + * Receives a data point from the storage attempt along with the exception + * that was associated with the failure. + * @param dp The data point to store + * @param exception The exception associated with the data point + */ + public abstract void handleError(final IncomingDataPoint dp, + final Exception exception); +} diff --git a/src/tsd/SuggestRpc.java b/src/tsd/SuggestRpc.java index 7c8601ddf8..6e4c677919 100644 --- a/src/tsd/SuggestRpc.java +++ b/src/tsd/SuggestRpc.java @@ -39,14 +39,10 @@ final class SuggestRpc implements HttpRpc { */ public void execute(final TSDB tsdb, final HttpQuery query) throws IOException { - + // only accept GET/POST - if (query.method() != HttpMethod.GET && query.method() != HttpMethod.POST) { - throw new BadRequestException(HttpResponseStatus.METHOD_NOT_ALLOWED, - "Method not allowed", "The HTTP method [" + query.method().getName() + - "] is not permitted for this endpoint"); - } - + RpcUtil.allowedMethods(query.method(), HttpMethod.GET.getName(), HttpMethod.POST.getName()); + final String type; final String q; final String max; diff --git a/src/tsd/TreeRpc.java b/src/tsd/TreeRpc.java index 380c4eb308..3cdfe436fd 100644 --- a/src/tsd/TreeRpc.java +++ b/src/tsd/TreeRpc.java @@ -52,6 +52,9 @@ final class TreeRpc implements HttpRpc { */ @Override public void execute(TSDB tsdb, HttpQuery query) throws IOException { + // only accept GET/POST + RpcUtil.allowedMethods(query.method(), HttpMethod.GET.getName(), HttpMethod.POST.getName()); + // the uri will be /api/vX/tree/? or /api/tree/? final String[] uri = query.explodeAPIPath(); final String endpoint = uri.length > 1 ? uri[1] : ""; @@ -208,11 +211,9 @@ private void handleTree(TSDB tsdb, HttpQuery query) { * @throws BadRequestException if the request was invalid. */ private void handleBranch(TSDB tsdb, HttpQuery query) { - if (query.getAPIMethod() != HttpMethod.GET) { - throw new BadRequestException(HttpResponseStatus.BAD_REQUEST, - "Unsupported HTTP request method"); - } - + + RpcUtil.allowedMethods(query.getAPIMethod(), HttpMethod.GET.getName(), HttpMethod.POST.getName()); + try { final int tree_id = parseTreeId(query, false); final String branch_hex = diff --git a/src/tsd/UniqueIdRpc.java b/src/tsd/UniqueIdRpc.java index a3b3c62642..34ffbc3379 100644 --- a/src/tsd/UniqueIdRpc.java +++ b/src/tsd/UniqueIdRpc.java @@ -22,6 +22,7 @@ import java.util.TreeMap; import org.hbase.async.Bytes; +import org.hbase.async.Bytes.ByteMap; import org.hbase.async.PutRequest; import org.jboss.netty.handler.codec.http.HttpMethod; import org.jboss.netty.handler.codec.http.HttpResponseStatus; @@ -46,6 +47,12 @@ */ final class UniqueIdRpc implements HttpRpc { + private final TSDB.OperationMode mode; + + public UniqueIdRpc(TSDB.OperationMode mode) { + this.mode = mode; + } + @Override public void execute(TSDB tsdb, HttpQuery query) throws IOException { @@ -62,6 +69,9 @@ public void execute(TSDB tsdb, HttpQuery query) throws IOException { } else if (endpoint.toLowerCase().equals("tsmeta")) { this.handleTSMeta(tsdb, query); return; + } else if (endpoint.toLowerCase().equals("rename")) { + this.handleRename(tsdb, query); + return; } else { throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, "Other UID endpoints have not been implemented yet"); @@ -83,13 +93,14 @@ public void execute(TSDB tsdb, HttpQuery query) throws IOException { * @param query The query for this request */ private void handleAssign(final TSDB tsdb, final HttpQuery query) { - // only accept GET And POST - if (query.method() != HttpMethod.GET && query.method() != HttpMethod.POST) { - throw new BadRequestException(HttpResponseStatus.METHOD_NOT_ALLOWED, - "Method not allowed", "The HTTP method [" + query.method().getName() + - "] is not permitted for this endpoint"); + if (!mode.isWrite()) { + throw new BadRequestException(HttpResponseStatus.NOT_FOUND, "Operation not allowed", + "This operation is not allowed in ro mode."); } - + + // only accept GET/POST + RpcUtil.allowedMethods(query.method(), HttpMethod.GET.getName(), HttpMethod.POST.getName()); + final HashMap<String, List<String>> source; if (query.method() == HttpMethod.POST) { source = query.serializer().parseUidAssignV1(); @@ -156,9 +167,15 @@ private void handleAssign(final TSDB tsdb, final HttpQuery query) { private void handleUIDMeta(final TSDB tsdb, final HttpQuery query) { final HttpMethod method = query.getAPIMethod(); + RpcUtil.allowedMethods(method, HttpMethod.GET.getName(), HttpMethod.POST.getName(), HttpMethod.PUT.getName(), HttpMethod.DELETE.getName()); + // GET if (method == HttpMethod.GET) { - + if (!mode.isRead()) { + throw new BadRequestException(HttpResponseStatus.NOT_FOUND, "Operation not allowed", + "This operation is not allowed in wo mode."); + } + final String uid = query.getRequiredQueryStringParam("uid"); final UniqueIdType type = UniqueId.stringToUniqueIdType( query.getRequiredQueryStringParam("type")); @@ -174,7 +191,11 @@ private void handleUIDMeta(final TSDB tsdb, final HttpQuery query) { } // POST } else if (method == HttpMethod.POST || method == HttpMethod.PUT) { - + if (!mode.isWrite()) { + throw new BadRequestException(HttpResponseStatus.NOT_FOUND, "Operation not allowed", + "This operation is not allowed in ro mode."); + } + final UIDMeta meta; if (query.hasContent()) { meta = query.serializer().parseUidMetaV1(); @@ -220,7 +241,11 @@ public Deferred<UIDMeta> call(Boolean success) throws Exception { } // DELETE } else if (method == HttpMethod.DELETE) { - + if (!mode.isWrite()) { + throw new BadRequestException(HttpResponseStatus.NOT_FOUND, "Operation not allowed", + "This operation is not allowed in ro mode."); + } + final UIDMeta meta; if (query.hasContent()) { meta = query.serializer().parseUidMetaV1(); @@ -255,9 +280,15 @@ public Deferred<UIDMeta> call(Boolean success) throws Exception { private void handleTSMeta(final TSDB tsdb, final HttpQuery query) { final HttpMethod method = query.getAPIMethod(); + RpcUtil.allowedMethods(method, HttpMethod.GET.getName(), HttpMethod.POST.getName(), HttpMethod.DELETE.getName(), HttpMethod.PUT.getName()); + // GET if (method == HttpMethod.GET) { - + if (!mode.isRead()) { + throw new BadRequestException(HttpResponseStatus.NOT_FOUND, "Operation not allowed", + "This operation is not allowed in wo mode."); + } + String tsuid = null; if (query.hasQueryStringParam("tsuid")) { tsuid = query.getQueryStringParam("tsuid"); @@ -291,11 +322,10 @@ private void handleTSMeta(final TSDB tsdb, final HttpQuery query) { } catch (IllegalArgumentException e) { throw new BadRequestException(e); } - final TSUIDQuery tsuid_query = new TSUIDQuery(tsdb); + final TSUIDQuery tsuid_query = new TSUIDQuery(tsdb, metric, tags); try { - tsuid_query.setQuery(metric, tags); final List<TSMeta> tsmetas = tsuid_query.getTSMetas() - .joinUninterruptibly(); + .joinUninterruptibly(); query.sendReply(query.serializer().formatTSMetaListV1(tsmetas)); } catch (NoSuchUniqueName e) { throw new BadRequestException(HttpResponseStatus.NOT_FOUND, @@ -310,6 +340,10 @@ private void handleTSMeta(final TSDB tsdb, final HttpQuery query) { } // POST / PUT } else if (method == HttpMethod.POST || method == HttpMethod.PUT) { + if (!mode.isWrite()) { + throw new BadRequestException(HttpResponseStatus.NOT_FOUND, "Operation not allowed", + "This operation is not allowed in ro mode."); + } final TSMeta meta; if (query.hasContent()) { @@ -428,7 +462,11 @@ public Boolean call(Boolean exists) throws Exception { } // DELETE } else if (method == HttpMethod.DELETE) { - + if (!mode.isWrite()) { + throw new BadRequestException(HttpResponseStatus.NOT_FOUND, "Operation not allowed", + "This operation is not allowed in ro mode."); + } + final TSMeta meta; if (query.hasContent()) { meta = query.serializer().parseTSMetaV1(); @@ -442,10 +480,6 @@ public Boolean call(Boolean exists) throws Exception { throw new BadRequestException("Unable to delete TSMeta information", e); } query.sendStatusOnly(HttpResponseStatus.NO_CONTENT); - } else { - throw new BadRequestException(HttpResponseStatus.METHOD_NOT_ALLOWED, - "Method not allowed", "The HTTP method [" + method.getName() + - "] is not permitted for this endpoint"); } } @@ -478,6 +512,70 @@ private UIDMeta parseUIDMetaQS(final HttpQuery query) { return meta; } + /** + * Rename UID to a new name of the given metric, tagk or tagv names + * <p> + * This handler supports GET and POST whereby the GET command can parse query + * strings with the {@code type} and {@code name} as their parameters. + * <p> + * @param tsdb The TSDB from the RPC router + * @param query The query for this request + */ + private void handleRename(final TSDB tsdb, final HttpQuery query) { + if (!mode.isWrite()) { + throw new BadRequestException(HttpResponseStatus.NOT_FOUND, "Operation not allowed", + "This operation is not allowed in ro mode."); + } + // only accept GET and POST + RpcUtil.allowedMethods(query.method(), HttpMethod.GET.getName(), HttpMethod.POST.getName()); + + final HashMap<String, String> source; + if (query.method() == HttpMethod.POST) { + source = query.serializer().parseUidRenameV1(); + } else { + source = new HashMap<String, String>(3); + final String[] types = {"metric", "tagk", "tagv", "name"}; + for (int i = 0; i < types.length; i++) { + final String value = query.getQueryStringParam(types[i]); + if (value!= null && !value.isEmpty()) { + source.put(types[i], value); + } + } + } + String type = null; + String oldname = null; + String newname = null; + for (Map.Entry<String, String> entry : source.entrySet()) { + if (entry.getKey().equals("name")) { + newname = entry.getValue(); + } else { + type = entry.getKey(); + oldname = entry.getValue(); + } + } + + // we need a type/value and new name + if (type == null || oldname == null || newname == null) { + throw new BadRequestException("Missing necessary values to rename UID"); + } + + HashMap<String, String> response = new HashMap<String, String>(2); + try { + tsdb.renameUid(type, oldname, newname); + response.put("result", "true"); + } catch (IllegalArgumentException e) { + response.put("result", "false"); + response.put("error", e.getMessage()); + } + + if (!response.containsKey("error")) { + query.sendReply(query.serializer().formatUidRenameV1(response)); + } else { + query.sendReply(HttpResponseStatus.BAD_REQUEST, + query.serializer().formatUidRenameV1(response)); + } + } + /** * Used with verb overrides to parse out values from a query string * @param query The query to parse @@ -486,8 +584,13 @@ private UIDMeta parseUIDMetaQS(final HttpQuery query) { * be parsed */ private TSMeta parseTSMetaQS(final HttpQuery query) { - final String tsuid = query.getRequiredQueryStringParam("tsuid"); - final TSMeta meta = new TSMeta(tsuid); + final String tsuid = query.getQueryStringParam("tsuid"); + final TSMeta meta; + if (tsuid != null && !tsuid.isEmpty()) { + meta = new TSMeta(tsuid); + } else { + meta = new TSMeta(); + } final String display_name = query.getQueryStringParam("display_name"); if (display_name != null) { @@ -550,6 +653,7 @@ private TSMeta parseTSMetaQS(final HttpQuery query) { * @param data_query The query we're building * @throws BadRequestException if we are unable to parse the query or it is * missing components + * @todo - make this asynchronous */ private String getTSUIDForMetric(final String query_string, TSDB tsdb) { if (query_string == null || query_string.isEmpty()) { @@ -566,16 +670,23 @@ private String getTSUIDForMetric(final String query_string, TSDB tsdb) { } catch (IllegalArgumentException e) { throw new BadRequestException(e); } - final TreeMap<String, String> sortedTags = new TreeMap<String, String>(tags); + + // sort the UIDs on tagk values + final ByteMap<byte[]> tag_uids = new ByteMap<byte[]>(); + for (final Entry<String, String> pair : tags.entrySet()) { + tag_uids.put(tsdb.getUID(UniqueIdType.TAGK, pair.getKey()), + tsdb.getUID(UniqueIdType.TAGV, pair.getValue())); + } + // Byte Buffer to generate TSUID, pre allocated to the size of the TSUID final ByteArrayOutputStream buf = new ByteArrayOutputStream( - TSDB.metrics_width() + sortedTags.size() * + TSDB.metrics_width() + tag_uids.size() * (TSDB.tagk_width() + TSDB.tagv_width())); try { - buf.write(tsdb.getUID(UniqueIdType.METRIC, metric)); - for (Entry<String, String> e: sortedTags.entrySet()) { - buf.write(tsdb.getUID(UniqueIdType.TAGK, e.getKey()), 0, 3); - buf.write(tsdb.getUID(UniqueIdType.TAGV, e.getValue()), 0, 3); + buf.write(tsdb.getUID(UniqueIdType.METRIC, metric)); + for (final Entry<byte[], byte[]> uids: tag_uids.entrySet()) { + buf.write(uids.getKey()); + buf.write(uids.getValue()); } } catch (IOException e) { throw new BadRequestException(e); diff --git a/src/tsd/client/MetricForm.java b/src/tsd/client/MetricForm.java index e409d3e0fa..ac3bc5456c 100644 --- a/src/tsd/client/MetricForm.java +++ b/src/tsd/client/MetricForm.java @@ -14,6 +14,9 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.List; + import com.google.gwt.event.dom.client.BlurEvent; import com.google.gwt.event.dom.client.BlurHandler; import com.google.gwt.event.dom.client.ClickEvent; @@ -50,6 +53,7 @@ public static interface MetricChangeHandler extends EventHandler { private final CheckBox downsample = new CheckBox("Downsample"); private final ListBox downsampler = new ListBox(); private final ValidatedTextBox interval = new ValidatedTextBox(); + private final ListBox fill_policy = new ListBox(); private final CheckBox rate = new CheckBox("Rate"); private final CheckBox rate_counter = new CheckBox("Rate Ctr"); private final TextBox counter_max = new TextBox(); @@ -66,6 +70,7 @@ public MetricForm(final EventsHandler handler) { downsampler.addChangeHandler(handler); interval.addBlurHandler(handler); interval.addKeyPressHandler(handler); + fill_policy.addChangeHandler(handler); rate.addClickHandler(handler); rate_counter.addClickHandler(handler); counter_max.addBlurHandler(handler); @@ -116,17 +121,85 @@ private String parseWithMetric(final String metric) { clearTags(); return metric.substring(0, len - 2); } + final int num_tags_before = getNumTags(); + + final List<Filter> filters = new ArrayList<Filter>(); + final int close = metric.indexOf('}'); + int i = 0; + if (close != metric.length() - 1) { // "foo{...}{tagk=filter}" + final int filter_bracket = metric.lastIndexOf('{'); + for (final String filter : metric.substring(filter_bracket + 1, + metric.length() - 1).split(",")) { + if (filter.isEmpty()) { + break; + } + final String[] kv = filter.split("="); + if (kv.length != 2 || kv[0].isEmpty() || kv[1].isEmpty()) { + continue; // Invalid tag. + } + final Filter f = new Filter(); + f.tagk = kv[0]; + f.tagv = kv[1]; + f.is_groupby = false; + filters.add(f); + i++; + } + } + + i = 0; + for (final String tag : metric.substring(curly + 1, close).split(",")) { + if (tag.isEmpty() && close != metric.length() - 1){ + break; + } + final String[] kv = tag.split("="); + if (kv.length != 2 || kv[0].isEmpty() || kv[1].isEmpty()) { + continue; // Invalid tag. + } + final Filter f = new Filter(); + f.tagk = kv[0]; + f.tagv = kv[1]; + f.is_groupby = true; + filters.add(f); + i++; + } + if (!filters.isEmpty()) { + Collections.sort(filters); + } + + i = 0; + for (int x = filters.size() - 1; x >= 0; x--) { + final Filter filter = filters.get(x); + if (i < num_tags_before) { + setTag(i++, filter.tagk, filter.tagv, filter.is_groupby); + } else { + addTag(filter.tagk, filter.tagv, filter.is_groupby); + } + } + + if (i < num_tags_before) { + setTag(i, "", "", true); + } else { + addTag(); + } + // Remove extra tags. + for (i++; i < num_tags_before; i++) { + tagtable.removeRow(i + 1); + } + // Return the "foo" part of "foo{a=b,...,x=y}" + return metric.substring(0, curly); + + /* // substring the tags out of "foo{a=b,...,x=y}" and parse them. int i = 0; // Tag index. final int num_tags_before = getNumTags(); for (final String tag : metric.substring(curly + 1, len - 1).split(",")) { final String[] kv = tag.split("="); if (kv.length != 2 || kv[0].isEmpty() || kv[1].isEmpty()) { - setTag(i, "", ""); + setTag(i, "", "", true); continue; // Invalid tag. } if (i < num_tags_before) { - setTag(i, kv[0], kv[1]); + setTag(i, kv[0], kv[1], true); } else { addTag(kv[0], kv[1]); } @@ -134,7 +207,7 @@ private String parseWithMetric(final String metric) { } // Leave an empty line at the end. if (i < num_tags_before) { - setTag(i, "", ""); + setTag(i, "", "", true); } else { addTag(); } @@ -143,7 +216,7 @@ private String parseWithMetric(final String metric) { tagtable.removeRow(i + 1); } // Return the "foo" part of "foo{a=b,...,x=y}" - return metric.substring(0, curly); + return metric.substring(0, curly); */ } public void updateFromQueryString(final String m, final String o) { @@ -179,18 +252,38 @@ public void updateFromQueryString(final String m, final String o) { // downsampling function & interval. if (i > 0) { - final int dash = parts[1].indexOf('-', 1); // 1st char can't be `-'. - if (dash < 0) { + // First dash should have been given. + final int first_dash = parts[1].indexOf('-', 1); // 1st char can't be `-'. + if (first_dash < 0) { disableDownsample(); return; // Invalid downsampling specifier. } + + // Second dash (and subsequent fill policy) are optional. + final int second_dash = parts[1].indexOf('-', first_dash + 1); + downsample.setValue(true, false); downsampler.setEnabled(true); - setSelectedItem(downsampler, parts[1].substring(dash + 1)); + fill_policy.setEnabled(true); + if (-1 == second_dash) { + // No fill policy given. + setSelectedItem(downsampler, parts[1].substring(first_dash + 1)); + + // So use a default. + // TODO: don't assume this exists. + setSelectedItem(fill_policy, "lerp"); + } else { + // User specified fill policy. + setSelectedItem(downsampler, parts[1].substring(first_dash + 1, + second_dash)); + + // So use what was given. + setSelectedItem(fill_policy, parts[1].substring(second_dash + 1)); + } interval.setEnabled(true); - interval.setText(parts[1].substring(0, dash)); + interval.setText(parts[1].substring(0, first_dash)); } else { disableDownsample(); } @@ -202,6 +295,7 @@ private void disableDownsample() { downsample.setValue(false, false); interval.setEnabled(false); downsampler.setEnabled(false); + fill_policy.setEnabled(false); } public CheckBox x1y2() { @@ -264,6 +358,7 @@ private void assembleUi() { final HorizontalPanel hbox = new HorizontalPanel(); hbox.add(downsampler); hbox.add(interval); + hbox.add(fill_policy); vbox.add(hbox); } add(vbox); @@ -281,9 +376,18 @@ public void setAggregators(final ArrayList<String> aggs) { aggregators.addItem((String)agg); downsampler.addItem((String)agg); } + // TODO: don't assume we will get these. setSelectedItem(aggregators, "sum"); setSelectedItem(downsampler, "avg"); } + + public void setFillPolicies(final List<String> policies) { + for (final String policy : policies) { + fill_policy.addItem(policy); + } + // TODO: don't assume we will get this. + setSelectedItem(fill_policy, "lerp"); + } public boolean buildQueryString(final StringBuilder url) { final String metric = getMetric(); @@ -294,14 +398,20 @@ public boolean buildQueryString(final StringBuilder url) { url.append(selectedValue(aggregators)); if (downsample.getValue()) { url.append(':').append(interval.getValue()) - .append('-').append(selectedValue(downsampler)); + .append('-').append(selectedValue(downsampler)) + .append('-').append(selectedValue(fill_policy)); } if (rate.getValue()) { url.append(":rate"); if (rate_counter.getValue()) { - url.append('{').append("counter"); + url.append('{');//.append("counter"); final String max = counter_max.getValue().trim(); final String reset = counter_reset_value.getValue().trim(); + if (max.isEmpty() && (reset.equals("0") || reset.isEmpty())) { + url.append("dropcounter"); + } else { + url.append("counter"); + } if (max.length() > 0 && reset.length() > 0) { url.append(',').append(max).append(',').append(reset); } else if (max.length() > 0 && reset.length() == 0) { @@ -313,24 +423,32 @@ public boolean buildQueryString(final StringBuilder url) { } } url.append(':').append(metric); - { - final int ntags = getNumTags(); - url.append('{'); - for (int tag = 0; tag < ntags; tag++) { - final String tagname = getTagName(tag); - final String tagvalue = getTagValue(tag); - if (tagname.isEmpty() || tagvalue.isEmpty()) { - continue; + List<Filter> filters = getFilters(true); + url.append('{'); + if (!filters.isEmpty()) { + for (int i = 0; i < filters.size(); i++) { + if (i > 0) { + url.append(","); } - url.append(tagname).append('=').append(tagvalue) - .append(','); + url.append(filters.get(i).tagk) + .append("=") + .append(filters.get(i).tagv); } - final int last = url.length() - 1; - if (url.charAt(last) == '{') { // There was no tag. - url.setLength(last); // So remove the `{'. - } else { // Need to replace the last `,' with a `}'. - url.setCharAt(url.length() - 1, '}'); + } + url.append('}'); + // now the non-group bys + filters = getFilters(false); + if (!filters.isEmpty()) { + url.append('{'); + for (int i = 0; i < filters.size(); i++) { + if (i > 0) { + url.append(","); + } + url.append(filters.get(i).tagk) + .append("=") + .append(filters.get(i).tagv); } + url.append('}'); } url.append("&o="); if (x1y2.getValue()) { @@ -338,7 +456,34 @@ public boolean buildQueryString(final StringBuilder url) { } return true; } - + + /** + * Helper method to extract the tags from the row set and sort them before + * sending to the API so that we avoid a bug wherein the sort order changes + * on reload. + * @param group_by Whether or not to fetch group by or non-group by filters. + * @return A non-null list of filters. May be empty. + */ + private List<Filter> getFilters(final boolean group_by) { + final int ntags = getNumTags(); + final List<Filter> filters = new ArrayList<Filter>(ntags); + for (int tag = 0; tag < ntags; tag++) { + final Filter filter = new Filter(); + filter.tagk = getTagName(tag); + filter.tagv = getTagValue(tag); + filter.is_groupby = isTagGroupby(tag); + if (filter.tagk.isEmpty() || filter.tagv.isEmpty()) { + continue; + } + if (filter.is_groupby == group_by) { + filters.add(filter); + } + } + // sort on the tagk + Collections.sort(filters); + return filters; + } + private int getNumTags() { return tagtable.getRowCount() - 1; } @@ -350,6 +495,10 @@ private String getTagName(final int i) { private String getTagValue(final int i) { return ((SuggestBox) tagtable.getWidget(i + 1, 2)).getValue(); } + + private boolean isTagGroupby(final int i) { + return ((CheckBox) tagtable.getWidget(i + 1, 3)).getValue(); + } private void setTagName(final int i, final String value) { ((SuggestBox) tagtable.getWidget(i + 1, 1)).setValue(value); @@ -359,6 +508,10 @@ private void setTagValue(final int i, final String value) { ((SuggestBox) tagtable.getWidget(i + 1, 2)).setValue(value); } + private void isTagGroupby(final int i, final boolean groupby) { + ((CheckBox) tagtable.getWidget(i + 1, 3)).setValue(groupby); + } + /** * Changes the name/value of an existing tag. * @param i The index of the tag to change. @@ -366,27 +519,34 @@ private void setTagValue(final int i, final String value) { * @param value The new value of the tag. * Requires: {@code i < getNumTags()}. */ - private void setTag(final int i, final String name, final String value) { + private void setTag(final int i, final String name, final String value, + final boolean groupby) { setTagName(i, name); setTagValue(i, value); + isTagGroupby(i, groupby); } private void addTag() { - addTag(null, null); + addTag(null, null, true); } private void addTag(final String default_tagname) { - addTag(default_tagname, null); + addTag(default_tagname, null, true); } private void addTag(final String default_tagname, - final String default_value) { + final String default_value, + final boolean is_groupby) { final int row = tagtable.getRowCount(); - + final ValidatedTextBox tagname = new ValidatedTextBox(); final SuggestBox suggesttagk = RemoteOracle.newSuggestBox("tagk", tagname); final ValidatedTextBox tagvalue = new ValidatedTextBox(); final SuggestBox suggesttagv = RemoteOracle.newSuggestBox("tagv", tagvalue); + final CheckBox groupby = new CheckBox(); + groupby.setValue(is_groupby); + groupby.setTitle("Group by"); + groupby.addClickHandler(events_handler); tagname.setValidationRegexp(TSDB_ID_RE); tagvalue.setValidationRegexp(TSDB_TAGVALUE_RE); tagname.setWidth("100%"); @@ -400,6 +560,7 @@ private void addTag(final String default_tagname, tagtable.setWidget(row, 1, suggesttagk); tagtable.setWidget(row, 2, suggesttagv); + tagtable.setWidget(row, 3, groupby); if (row > 2) { final Button remove = new Button("x"); remove.addClickHandler(removetag); @@ -417,7 +578,7 @@ private void addTag(final String default_tagname, } private void clearTags() { - setTag(0, "", ""); + setTag(0, "", "", true); for (int i = getNumTags() - 1; i > 1; i++) { tagtable.removeRow(i + 1); } @@ -453,9 +614,10 @@ public void onBlur(final BlurEvent event) { for (int tag = 1; tag < ntags; tag++) { final String tagname = getTagName(tag); final String tagvalue = getTagValue(tag); - setTag(tag - 1, tagname, tagvalue); + // todo - groupby + setTag(tag - 1, tagname, tagvalue, isTagGroupby(tag)); } - setTag(ntags - 1, "", ""); + setTag(ntags - 1, "", "", true); } // Try to remove empty lines from the tag table (but never remove the // first line or last line, even if they're empty). Walk the table @@ -496,6 +658,7 @@ public void onClick(final ClickEvent event) { private void setupDownsampleWidgets() { downsampler.setEnabled(false); + fill_policy.setEnabled(false); interval.setEnabled(false); interval.setMaxLength(5); interval.setVisibleLength(5); @@ -505,6 +668,7 @@ private void setupDownsampleWidgets() { public void onClick(final ClickEvent event) { final boolean checked = ((CheckBox) event.getSource()).getValue(); downsampler.setEnabled(checked); + fill_policy.setEnabled(checked); interval.setEnabled(checked); if (checked) { downsampler.setFocus(true); @@ -567,7 +731,7 @@ static final public LocalRateOptions parseRateOptions(boolean rate, String spec) try { LocalRateOptions options = new LocalRateOptions(); - options.is_counter = "counter".equals(parts[0]); + options.is_counter = parts[0].endsWith("counter"); options.counter_max = (parts.length >= 2 && parts[1].length() > 0 ? Long .parseLong(parts[1]) : Long.MAX_VALUE); options.reset_value = (parts.length >= 3 && parts[2].length() > 0 ? Long @@ -578,6 +742,25 @@ static final public LocalRateOptions parseRateOptions(boolean rate, String spec) } } + private static class Filter implements Comparable<Filter> { + String tagk; + String tagv; + boolean is_groupby; + + @Override + public int compareTo(final Filter filter) { + if (filter == this) { + return 0; + } + int tagkv_order = tagk.compareTo(filter.tagk) == 0 ? + tagv.compareTo(filter.tagv) : tagk.compareTo(filter.tagk); + + int groupby_order = is_groupby == filter.is_groupby ? 0 : (is_groupby ? 1 : -1); + + return tagkv_order == 0 ? groupby_order : tagkv_order; + } + } + // ------------------- // // Focusable interface // // ------------------- // diff --git a/src/tsd/client/QueryUi.java b/src/tsd/client/QueryUi.java index 25de12556c..440b1419bd 100644 --- a/src/tsd/client/QueryUi.java +++ b/src/tsd/client/QueryUi.java @@ -18,9 +18,18 @@ * virtually no exposure to the technology except through the tutorial. --tsuna */ +import java.net.URLEncoder; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import net.opentsdb.graph.Plot; import com.google.gwt.core.client.EntryPoint; import com.google.gwt.dom.client.Style; @@ -80,6 +89,7 @@ import com.google.gwt.user.client.ui.Image; import com.google.gwt.user.client.ui.InlineLabel; import com.google.gwt.user.client.ui.Label; +import com.google.gwt.user.client.ui.ListBox; import com.google.gwt.user.client.ui.RadioButton; import com.google.gwt.user.client.ui.RootPanel; import com.google.gwt.user.client.ui.TextBox; @@ -91,11 +101,23 @@ * Manages the entire UI, forms to query the TSDB and other misc panels. */ public class QueryUi implements EntryPoint, HistoryListener { + + /** Map of available gnuplot data styles. */ + public static Map<String, Integer> stylesMap = new HashMap<String, Integer>(); + static { + Map<String, Integer> map = new HashMap<String, Integer>(); + map.put("linespoint", 0); + map.put("points", 1); + map.put("circles", 3); + map.put("dots", 4); + stylesMap = Collections.unmodifiableMap(map); + } + // Some URLs we use to fetch data from the TSD. - private static final String AGGREGATORS_URL = "/aggregators"; - private static final String LOGS_URL = "/logs?json"; - private static final String STATS_URL = "/stats?json"; - private static final String VERSION_URL = "/version?json"; + private static final String AGGREGATORS_URL = "aggregators"; + private static final String LOGS_URL = "logs?json"; + private static final String STATS_URL = "stats?json"; + private static final String VERSION_URL = "version?json"; private static final DateTimeFormat FULLDATE = DateTimeFormat.getFormat("yyyy/MM/dd-HH:mm:ss"); @@ -117,6 +139,7 @@ public class QueryUi implements EntryPoint, HistoryListener { private final ValidatedTextBox yformat = new ValidatedTextBox(); private final ValidatedTextBox y2format = new ValidatedTextBox(); private final ValidatedTextBox wxh = new ValidatedTextBox(); + private final CheckBox global_annotations = new CheckBox("Global annotations"); private String keypos = ""; // Position of the key on the graph. private final CheckBox horizontalkey = new CheckBox("Horizontal layout"); @@ -125,6 +148,12 @@ public class QueryUi implements EntryPoint, HistoryListener { // Styling options. private final CheckBox smooth = new CheckBox(); + private final ListBox styles = new ListBox(); + private String timezone = ""; + + // Annotations handling flags. + private boolean hide_annotations = false; + private boolean show_global_annotations = false; /** * Handles every change to the query form and gets a new graph. @@ -176,7 +205,14 @@ protected <H extends EventHandler> void onEvent(final DomEvent<H> event) { /** List of known aggregation functions. Fetched once from the server. */ private final ArrayList<String> aggregators = new ArrayList<String>(); - + + /** + * List of known downsampling fill policies. + * TODO: fetch from server. + */ + private final List<String> fill_policies = Arrays.asList("none", "nan", + "zero", "null"); + private final DecoratedTabPanel metrics = new DecoratedTabPanel(); /** Panel to place generated graphs and a box for zoom highlighting. */ @@ -256,10 +292,13 @@ public void onValueChange(final ValueChangeEvent<Date> event) { y2format.addKeyPressHandler(refreshgraph); wxh.addBlurHandler(refreshgraph); wxh.addKeyPressHandler(refreshgraph); + global_annotations.addBlurHandler(refreshgraph); + global_annotations.addKeyPressHandler(refreshgraph); horizontalkey.addClickHandler(refreshgraph); keybox.addClickHandler(refreshgraph); nokey.addClickHandler(refreshgraph); smooth.addClickHandler(refreshgraph); + styles.addChangeHandler(refreshgraph); yrange.setValidationRegexp("^(" // Nothing or + "|\\[([-+.0-9eE]+|\\*)?" // "[start @@ -347,6 +386,11 @@ public void onValueChange(final ValueChangeEvent<Boolean> event) { hbox.add(wxh); table.setWidget(0, 3, hbox); } + { + final HorizontalPanel hbox = new HorizontalPanel(); + hbox.add(global_annotations); + table.setWidget(0, 4, hbox); + } { addMetricForm("metric 1", 0); metrics.selectTab(0); @@ -373,6 +417,7 @@ public void onBeforeSelection(final BeforeSelectionEvent<Integer> event) { optpanel.add(makeStylePanel(), "Style"); optpanel.selectTab(0); table.setWidget(1, 3, optpanel); + table.getFlexCellFormatter().setColSpan(1, 3, 2); final DecoratorPanel decorator = new DecoratorPanel(); decorator.setWidget(table); @@ -455,9 +500,14 @@ public void onHistoryChanged(String historyToken) { /** Additional styling options. */ private Grid makeStylePanel() { + for (Entry<String, Integer> item : stylesMap.entrySet()) { + styles.insertItem(item.getKey(), item.getValue()); + } final Grid grid = new Grid(5, 3); grid.setText(0, 1, "Smooth"); grid.setWidget(0, 2, smooth); + grid.setText(1, 1, "Style"); + grid.setWidget(1, 2, styles); return grid; } @@ -491,6 +541,7 @@ private MetricForm addMetricForm(final String label, final int item) { metric.x1y2().addClickHandler(updatey2range); metric.setMetricChangeHandler(metric_change_handler); metric.setAggregators(aggregators); + metric.setFillPolicies(fill_policies); metrics.insert(metric, label, item); return metric; } @@ -673,12 +724,12 @@ private void addLabels(final StringBuilder url) { private void addFormats(final StringBuilder url) { final String yformat = this.yformat.getText(); if (!yformat.isEmpty()) { - url.append("&yformat=").append(yformat); + url.append("&yformat=").append(URL.encode(yformat)); } if (y2format.isEnabled()) { final String y2format = this.y2format.getText(); if (!y2format.isEmpty()) { - url.append("&y2format=").append(y2format); + url.append("&y2format=").append(URL.encode(y2format)); } } } @@ -743,14 +794,25 @@ private static QueryString getQueryString(final String qs) { } private void refreshFromQueryString() { - final QueryString qs = getQueryString(History.getToken()); + final QueryString qs = getQueryString(URL.decode(History.getToken())); maybeSetTextbox(qs, "start", start_datebox.getTextBox()); maybeSetTextbox(qs, "end", end_datebox.getTextBox()); setTextbox(qs, "wxh", wxh); + global_annotations.setValue(qs.containsKey("global_annotations")); autoreload.setValue(qs.containsKey("autoreload"), true); maybeSetTextbox(qs, "autoreload", autoreoload_interval); + show_global_annotations = qs.containsKey("global_annotations") ? true : false; + hide_annotations = qs.containsKey("no_annotations") ? true : false; + + //get the tz param value + final ArrayList<String> tzvalues = qs.get("tz"); + if (tzvalues == null) + timezone = ""; + else + timezone = tzvalues.get(0); + final ArrayList<String> newmetrics = qs.get("m"); if (newmetrics == null) { // Clear all metric forms. final int toremove = metrics.getWidgetCount() - 1; @@ -812,6 +874,9 @@ private void refreshFromQueryString() { } nokey.setValue(qs.containsKey("nokey")); smooth.setValue(qs.containsKey("smooth")); + if (stylesMap.containsKey(qs.getFirst("style"))) { + styles.setSelectedIndex(stylesMap.get(qs.getFirst("style"))); + } } private void refreshGraph() { @@ -829,7 +894,7 @@ private void refreshGraph() { } } final StringBuilder url = new StringBuilder(); - url.append("/q?start="); + url.append("q?start="); final String start_text = start_datebox.getTextBox().getText(); if (start_text.endsWith(" ago") || start_text.endsWith("-ago")) { url.append(start_text); @@ -854,6 +919,13 @@ private void refreshGraph() { // a special parameter that the server will delete from the query. url.append("&ignore=" + nrequests++); } + if (global_annotations.getValue()) { + url.append("&global_annotations"); + } + + if(timezone.length() > 1) + url.append("&tz=").append(timezone); + if (!addAllMetrics(url)) { return; } @@ -879,6 +951,15 @@ private void refreshGraph() { if (smooth.getValue()) { url.append("&smooth=csplines"); } + url.append("&style=").append(styles.getValue(styles.getSelectedIndex())); + + if (hide_annotations) { + url.append("&no_annotations=true"); + } + if (show_global_annotations) { + url.append("&global_annotations=true"); + } + final String unencodedUri = url.toString(); final String uri = URL.encode(unencodedUri); if (uri.equals(lastgraphuri)) { @@ -904,12 +985,12 @@ public void got(final JSONValue json) { } else { clearError(); - String history = unencodedUri.substring(3) // Remove "/q?". + String history = unencodedUri.substring(2) // Remove "q?". .replaceFirst("ignore=[^&]*&", ""); // Unnecessary cruft. if (autoreload.getValue()) { history += "&autoreload=" + autoreoload_interval.getText(); } - if (!history.equals(History.getToken())) { + if (!history.equals(URL.decode(History.getToken()))) { History.newItem(history, false); } diff --git a/src/tsd/client/RemoteOracle.java b/src/tsd/client/RemoteOracle.java index 971b42a489..2eb18ecae9 100644 --- a/src/tsd/client/RemoteOracle.java +++ b/src/tsd/client/RemoteOracle.java @@ -40,7 +40,7 @@ */ final class RemoteOracle extends SuggestOracle { - private static final String SUGGEST_URL = "/suggest?type="; // + type&q=foo + private static final String SUGGEST_URL = "suggest?type="; // + type&q=foo /** * Maps an oracle type to its suggestion cache. diff --git a/src/tsd/static/favicon.ico b/src/tsd/static/favicon.ico old mode 100644 new mode 100755 index 954d3c335e..b2d9ef2245 Binary files a/src/tsd/static/favicon.ico and b/src/tsd/static/favicon.ico differ diff --git a/src/tsd/static/opentsdb_header.jpg b/src/tsd/static/opentsdb_header.jpg new file mode 100755 index 0000000000..ecca08a88f Binary files /dev/null and b/src/tsd/static/opentsdb_header.jpg differ diff --git a/src/uid/FailedToAssignUniqueIdException.java b/src/uid/FailedToAssignUniqueIdException.java new file mode 100644 index 0000000000..fb79e6355c --- /dev/null +++ b/src/uid/FailedToAssignUniqueIdException.java @@ -0,0 +1,82 @@ +package net.opentsdb.uid; + +/** + * Thrown when we failed to assign an ID to a string such as a metric, tagk + * or tag v. + * @see UniqueId + */ +public final class FailedToAssignUniqueIdException extends RuntimeException { + /** The 'kind' of the table. */ + private final String kind; + /** The name of the object attempting to be assigned */ + private final String name; + /** How many attempts were made to assign the ID */ + private final int attempts; + + /** + * CTor + * @param kind The kind of object that couldn't be assigned + * @param name The name of the object that couldn't be assigned + * @param attempts How many attempts were made to assign + */ + public FailedToAssignUniqueIdException(final String kind, final String name, + final int attempts) { + super("Failed to assign random ID for kind='" + kind + "' name='" + + name + "' after " + attempts + " attempts"); + this.kind = kind; + this.name = name; + this.attempts = attempts; + } + + /** + * CTor + * @param kind The kind of object that couldn't be assigned + * @param name The name of the object that couldn't be assigned + * @param attempts How many attempts were made to assign + * @param msg A message to append + * @since 2.3 + */ + public FailedToAssignUniqueIdException(final String kind, final String name, + final int attempts, final String msg) { + super("Failed to assign ID for kind='" + kind + "' name='" + + name + "' after " + attempts + " attempts due to: " + msg); + this.kind = kind; + this.name = name; + this.attempts = attempts; + } + + /** + * CTor + * @param kind The kind of object that couldn't be assigned + * @param name The name of the object that couldn't be assigned + * @param attempts How many attempts were made to assign + * @param ex An exception that caused assignment to fail + */ + public FailedToAssignUniqueIdException(final String kind, final String name, + final int attempts, final Throwable ex) { + super("Failed to assign random ID for kind='" + kind + "' name='" + + name + "' after " + attempts + " attempts", ex); + this.kind = kind; + this.name = name; + this.attempts = attempts; + } + + + /** @return Returns the kind of unique ID that couldn't be assigned. */ + public String kind() { + return kind; + } + + /** @return Returns the name of the object that couldn't be assigned */ + public String name() { + return name; + } + + /** @return Returns how many attempts were made to assign a UID */ + public int attempts() { + return attempts; + } + + private static final long serialVersionUID = 399163221436118367L; + +} diff --git a/src/uid/RandomUniqueId.java b/src/uid/RandomUniqueId.java new file mode 100644 index 0000000000..58e560dfc4 --- /dev/null +++ b/src/uid/RandomUniqueId.java @@ -0,0 +1,76 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.uid; + +import java.security.SecureRandom; +import org.hbase.async.Bytes; +import net.opentsdb.core.TSDB; + +/** + * Generate Random UIDs to be used as unique ID. + * Random metric IDs help to distribute hotspots evenly to region servers. + * It is better to decide whether to use random or serial uid for one type when + * the hbase uid table is empty. If the logic to switch between random or serial + * uid is changed in between writes it will cause frequent id collisions. + * @since 2.2 + */ +public class RandomUniqueId { + /** Use the SecureRandom class to avoid blocking calls */ + private static SecureRandom random_generator = new SecureRandom( + Bytes.fromLong(System.currentTimeMillis())); + + /** Used to limit UIDs to unsigned longs */ + public static final int MAX_WIDTH = 7; + + /** + * Get the next random metric UID, a positive integer greater than zero. + * The default metric ID width is 3 bytes. If it is 3 then it can return + * only up to the max value a 3 byte integer can return, which is 2^31-1. + * In that case, even though it is long, its range will be between 0 + * and 2^31-1. + * NOTE: The caller is responsible for assuring that the UID hasn't been + * assigned yet. + * @return a random UID up to {@link TSDB#metrics_width()} wide + */ + public static long getRandomUID() { + return getRandomUID(TSDB.metrics_width()); + } + + /** + * Get the next random UID. It creates random bytes, then convert it to an + * unsigned long. + * @param width Number of bytes to randomize, it can not be larger + * than {@link MAX_WIDTH} bytes wide + * @return a randomly UID + * @throws IllegalArgumentException if the width is larger than + * {@link MAX_WIDTH} bytes + */ + public static long getRandomUID(final int width) { + if (width > MAX_WIDTH) { + throw new IllegalArgumentException("Expecting to return an unsigned long " + + "random integer, it can not be larger than " + MAX_WIDTH + + " bytes wide"); + } + final byte[] bytes = new byte[width]; + random_generator.nextBytes(bytes); + + long value = 0; + for (int i = 0; i<bytes.length; i++){ + value <<= 8; + value |= bytes[i] & 0xFF; + } + + // make sure we never return 0 as a UID + return value != 0 ? value : value + 1; + } +} \ No newline at end of file diff --git a/src/uid/UniqueId.java b/src/uid/UniqueId.java index 8c324bdc51..04eceb324b 100644 --- a/src/uid/UniqueId.java +++ b/src/uid/UniqueId.java @@ -15,14 +15,20 @@ import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import javax.xml.bind.DatatypeConverter; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.stumbleupon.async.Callback; import com.stumbleupon.async.Deferred; @@ -39,7 +45,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import net.opentsdb.core.Const; +import net.opentsdb.core.Internal; import net.opentsdb.core.TSDB; +import net.opentsdb.core.TSDB.OperationMode; import net.opentsdb.meta.UIDMeta; /** @@ -72,6 +81,8 @@ public enum UniqueIdType { private static final short MAX_ATTEMPTS_ASSIGN_ID = 3; /** How many time do we try to apply an edit before giving up. */ private static final short MAX_ATTEMPTS_PUT = 6; + /** How many time do we try to assign a random ID before giving up. */ + private static final short MAX_ATTEMPTS_ASSIGN_RANDOM_ID = 10; /** Initial delay in ms for exponential backoff to retry failed RPCs. */ private static final short INITIAL_EXP_BACKOFF_DELAY = 800; /** Maximum number of results to return in suggest(). */ @@ -87,24 +98,48 @@ public enum UniqueIdType { private final UniqueIdType type; /** Number of bytes on which each ID is encoded. */ private final short id_width; + /** Whether or not to randomize new IDs */ + private final boolean randomize_id; /** Cache for forward mappings (name to ID). */ - private final ConcurrentHashMap<String, byte[]> name_cache = - new ConcurrentHashMap<String, byte[]>(); + private final ConcurrentHashMap<String, byte[]> name_cache; /** Cache for backward mappings (ID to name). * The ID in the key is a byte[] converted to a String to be Comparable. */ - private final ConcurrentHashMap<String, String> id_cache = - new ConcurrentHashMap<String, String>(); + private final ConcurrentHashMap<String, String> id_cache; + + /** Cache for forward mappings (name to ID). */ + private final Cache<String, byte[]> lru_name_cache; + /** Cache for backward mappings (ID to name). + * The ID in the key is a byte[] converted to a String to be Comparable. */ + private final Cache<String, String> lru_id_cache; + /** Map of pending UID assignments */ private final HashMap<String, Deferred<byte[]>> pending_assignments = new HashMap<String, Deferred<byte[]>>(); + /** Set of UID rename */ + private final Set<String> renaming_id_names = + Collections.synchronizedSet(new HashSet<String>()); /** Number of times we avoided reading from HBase thanks to the cache. */ - private volatile int cache_hits; + private volatile long cache_hits; /** Number of times we had to read from HBase and populate the cache. */ - private volatile int cache_misses; - - /** Whether or not to generate new UIDMetas */ + private volatile long cache_misses; + /** How many times we collided with an existing ID when attempting to + * generate a new UID */ + private volatile int random_id_collisions; + /** How many times assignments have been rejected by the UID filter */ + private volatile int rejected_assignments; + + /** The mode of operation for this TSD. */ + private OperationMode mode; + + /** Whether or not to use the mode for caching IDs. */ + private boolean use_mode; + + /** Whether or not to use the Guava LRU cache for IDs. */ + private boolean use_lru; + + /** TSDB object used for filtering and/or meta generation. */ private TSDB tsdb; /** @@ -118,6 +153,22 @@ public enum UniqueIdType { */ public UniqueId(final HBaseClient client, final byte[] table, final String kind, final int width) { + this(client, table, kind, width, false); + } + + /** + * Constructor. + * @param client The HBase client to use. + * @param table The name of the HBase table to use. + * @param kind The kind of Unique ID this instance will deal with. + * @param width The number of bytes on which Unique IDs should be encoded. + * @param randomize_id Whether or not to randomize new UIDs + * @throws IllegalArgumentException if width is negative or too small/large + * or if kind is an empty string. + * @since 2.2 + */ + public UniqueId(final HBaseClient client, final byte[] table, final String kind, + final int width, final boolean randomize_id) { this.client = client; this.table = table; if (kind.isEmpty()) { @@ -129,23 +180,113 @@ public UniqueId(final HBaseClient client, final byte[] table, final String kind, throw new IllegalArgumentException("Invalid width: " + width); } this.id_width = (short) width; + this.randomize_id = randomize_id; + mode = OperationMode.READWRITE; + name_cache = new ConcurrentHashMap<String, byte[]>(); + id_cache = new ConcurrentHashMap<String, String>(); + lru_name_cache = null; + lru_id_cache = null; + use_lru = false; + } + + /** + * Constructor. + * @param tsdb The TSDB this UID object belongs to + * @param table The name of the HBase table to use. + * @param kind The kind of Unique ID this instance will deal with. + * @param width The number of bytes on which Unique IDs should be encoded. + * @param randomize_id Whether or not to randomize new UIDs + * @throws IllegalArgumentException if width is negative or too small/large + * or if kind is an empty string. + * @since 2.3 + */ + public UniqueId(final TSDB tsdb, final byte[] table, final String kind, + final int width, final boolean randomize_id) { + this.client = tsdb.getClient(); + this.tsdb = tsdb; + this.table = table; + if (kind.isEmpty()) { + throw new IllegalArgumentException("Empty string as 'kind' argument!"); + } + this.kind = toBytes(kind); + type = stringToUniqueIdType(kind); + if (width < 1 || width > 8) { + throw new IllegalArgumentException("Invalid width: " + width); + } + this.id_width = (short) width; + this.randomize_id = randomize_id; + mode = tsdb.getMode(); + use_mode = tsdb.getConfig().getBoolean("tsd.uid.use_mode"); + use_lru = tsdb.getConfig().getBoolean("tsd.uid.lru.enable"); + if (use_lru) { + name_cache = null; + id_cache = null; + lru_name_cache = CacheBuilder.newBuilder() + .maximumSize(tsdb.getConfig().getInt("tsd.uid.lru.name.size")) + .build(); + lru_id_cache = CacheBuilder.newBuilder() + .maximumSize(tsdb.getConfig().getInt("tsd.uid.lru.id.size")) + .build(); + } else { + name_cache = new ConcurrentHashMap<String, byte[]>(); + id_cache = new ConcurrentHashMap<String, String>(); + lru_name_cache = null; + lru_id_cache = null; + } } /** The number of times we avoided reading from HBase thanks to the cache. */ - public int cacheHits() { + public long cacheHits() { return cache_hits; } /** The number of times we had to read from HBase and populate the cache. */ - public int cacheMisses() { + public long cacheMisses() { return cache_misses; } /** Returns the number of elements stored in the internal cache. */ - public int cacheSize() { + public long cacheSize() { + if (use_lru) { + return (int) (lru_name_cache.size() + lru_id_cache.size()); + } return name_cache.size() + id_cache.size(); } + /** + * Resets the cache hits counter before rollover. Note that a few updates + * may be dropped due to race conditions at rollover. + */ + private void incrementCacheHits() { + if (cache_hits >= Long.MAX_VALUE) { + cache_hits = 1; + } else { + cache_hits++; + } + } + + /** + * Resets the cache miss counter before rollover. Note that a few updates + * may be dropped due to race conditions at rollover. + */ + private void incrementCacheMiss() { + if (cache_misses >= Long.MAX_VALUE) { + cache_misses = 1; + } else { + cache_misses++; + } + } + + /** Returns the number of random UID collisions */ + public int randomIdCollisions() { + return random_id_collisions; + } + + /** Returns the number of UID assignments rejected by the filter */ + public int rejectedAssignments() { + return rejected_assignments; + } + public String kind() { return fromBytes(kind); } @@ -157,11 +298,16 @@ public short width() { /** @param tsdb Whether or not to track new UIDMeta objects */ public void setTSDB(final TSDB tsdb) { this.tsdb = tsdb; + mode = tsdb.getMode(); + use_mode = tsdb.getConfig().getBoolean("tsd.uid.use_mode"); } - /** The largest possible ID given the number of bytes the IDs are represented on. */ + /** The largest possible ID given the number of bytes the IDs are + * represented on. + * @deprecated Use {@link Internal#getMaxUnsignedValueOnBytes(int)} + */ public long maxPossibleId() { - return ((long) 1 << id_width * Byte.SIZE) - 1; + return Internal.getMaxUnsignedValueOnBytes(id_width); } /** @@ -169,8 +315,13 @@ public long maxPossibleId() { * @since 1.1 */ public void dropCaches() { - name_cache.clear(); - id_cache.clear(); + if (use_lru) { + lru_name_cache.invalidateAll(); + lru_id_cache.invalidateAll(); + } else { + name_cache.clear(); + id_cache.clear(); + } } /** @@ -216,17 +367,30 @@ public Deferred<String> getNameAsync(final byte[] id) { } final String name = getNameFromCache(id); if (name != null) { - cache_hits++; + incrementCacheHits(); return Deferred.fromResult(name); } - cache_misses++; + incrementCacheMiss(); class GetNameCB implements Callback<String, String> { public String call(final String name) { if (name == null) { throw new NoSuchUniqueId(kind(), id); } - addNameToCache(id, name); - addIdToCache(name, id); + if (use_mode) { + switch(mode) { + case READONLY: + addNameToCache(id, name); + break; + case WRITEONLY: + break; + default: + addNameToCache(id, name); + addIdToCache(name, id); + } + } else { + addNameToCache(id, name); + addIdToCache(name, id); + } return name; } } @@ -234,7 +398,8 @@ public String call(final String name) { } private String getNameFromCache(final byte[] id) { - return id_cache.get(fromBytes(id)); + return use_lru ? lru_id_cache.getIfPresent(fromBytes(id)) : + id_cache.get(fromBytes(id)); } private Deferred<String> getNameFromHBase(final byte[] id) { @@ -248,9 +413,13 @@ public String call(final byte[] name) { private void addNameToCache(final byte[] id, final String name) { final String key = fromBytes(id); - String found = id_cache.get(key); + String found = use_lru ? lru_id_cache.getIfPresent(key) : id_cache.get(key); if (found == null) { - found = id_cache.putIfAbsent(key, name); + if (use_lru) { + lru_id_cache.put(key, name); + } else { + found = id_cache.putIfAbsent(key, name); + } } if (found != null && !found.equals(name)) { throw new IllegalStateException("id=" + Arrays.toString(id) + " => name=" @@ -271,10 +440,10 @@ public byte[] getId(final String name) throws NoSuchUniqueName, HBaseException { public Deferred<byte[]> getIdAsync(final String name) { final byte[] id = getIdFromCache(name); if (id != null) { - cache_hits++; + incrementCacheHits(); return Deferred.fromResult(id); } - cache_misses++; + incrementCacheMiss(); class GetIdCB implements Callback<byte[], byte[]> { public byte[] call(final byte[] id) { if (id == null) { @@ -285,8 +454,21 @@ public byte[] call(final byte[] id) { + " which is != " + id_width + " required for '" + kind() + '\''); } - addIdToCache(name, id); - addNameToCache(id, name); + if (use_mode) { + switch(mode) { + case READONLY: + break; + case WRITEONLY: + addIdToCache(name, id); + break; + default: + addNameToCache(id, name); + addIdToCache(name, id); + } + } else { + addIdToCache(name, id); + addNameToCache(id, name); + } return id; } } @@ -295,7 +477,7 @@ public byte[] call(final byte[] id) { } private byte[] getIdFromCache(final String name) { - return name_cache.get(name); + return use_lru ? lru_name_cache.getIfPresent(name) : name_cache.get(name); } private Deferred<byte[]> getIdFromHBase(final String name) { @@ -303,13 +485,18 @@ private Deferred<byte[]> getIdFromHBase(final String name) { } private void addIdToCache(final String name, final byte[] id) { - byte[] found = name_cache.get(name); + byte[] found = use_lru ? lru_name_cache.getIfPresent(name) : + name_cache.get(name); if (found == null) { - found = name_cache.putIfAbsent(name, - // Must make a defensive copy to be immune - // to any changes the caller may do on the - // array later on. - Arrays.copyOf(id, id.length)); + if (use_lru) { + lru_name_cache.put(name, Arrays.copyOf(id, id.length)); + } else { + found = name_cache.putIfAbsent(name, + // Must make a defensive copy to be immune + // to any changes the caller may do on the + // array later on. + Arrays.copyOf(id, id.length)); + } } if (found != null && !Arrays.equals(found, id)) { throw new IllegalStateException("name=" + name + " => id=" @@ -329,9 +516,16 @@ private void addIdToCache(final String name, final byte[] id) { private final class UniqueIdAllocator implements Callback<Object, Object> { private final String name; // What we're trying to allocate an ID for. private final Deferred<byte[]> assignment; // deferred to call back - private short attempt = MAX_ATTEMPTS_ASSIGN_ID; // Give up when zero. + private short attempt = randomize_id ? // Give up when zero. + MAX_ATTEMPTS_ASSIGN_RANDOM_ID : MAX_ATTEMPTS_ASSIGN_ID; private HBaseException hbe = null; // Last exception caught. + // TODO(manolama) - right now if we retry the assignment it will create a + // callback chain MAX_ATTEMPTS_* long and call the ErrBack that many times. + // This can be cleaned up a fair amount but it may require changing the + // public behavior a bit. For now, the flag will prevent multiple attempts + // to execute the callback. + private boolean called = false; // whether we called the deferred or not private long id = -1; // The ID we'll grab with an atomic increment. private byte row[]; // The same ID, as a byte array. @@ -357,22 +551,29 @@ Deferred<byte[]> tryAllocate() { @SuppressWarnings("unchecked") public Object call(final Object arg) { if (attempt == 0) { - if (hbe == null) { + if (hbe == null && !randomize_id) { throw new IllegalStateException("Should never happen!"); } LOG.error("Failed to assign an ID for kind='" + kind() + "' name='" + name + "'", hbe); + if (hbe == null) { + throw new FailedToAssignUniqueIdException(kind(), name, + MAX_ATTEMPTS_ASSIGN_RANDOM_ID); + } throw hbe; } if (arg instanceof Exception) { - final String msg = ("Failed attempt #" + (MAX_ATTEMPTS_ASSIGN_ID - attempt) - + " to assign an UID for " + kind() + ':' + name - + " at step #" + state); + final String msg = ("Failed attempt #" + (randomize_id + ? (MAX_ATTEMPTS_ASSIGN_RANDOM_ID - attempt) + : (MAX_ATTEMPTS_ASSIGN_ID - attempt)) + + " to assign an UID for " + kind() + ':' + name + + " at step #" + state); if (arg instanceof HBaseException) { LOG.error(msg, (Exception) arg); hbe = (HBaseException) arg; - return tryAllocate(); // Retry from the beginning. + attempt--; + state = ALLOCATE_UID;; // Retry from the beginning. } else { LOG.error("WTF? Unexpected exception! " + msg, (Exception) arg); return arg; // Unexpected exception, let it bubble up. @@ -381,8 +582,11 @@ public Object call(final Object arg) { class ErrBack implements Callback<Object, Exception> { public Object call(final Exception e) throws Exception { - assignment.callback(e); - LOG.warn("Failed pending assignment for: " + name); + if (!called) { + LOG.warn("Failed pending assignment for: " + name, e); + assignment.callback(e); + called = true; + } return assignment; } } @@ -406,17 +610,22 @@ public Object call(final Exception e) throws Exception { return d.addBoth(this).addErrback(new ErrBack()); } + /** Generates either a random or a serial ID. If random, we need to + * make sure that there isn't a UID collision. + */ private Deferred<Long> allocateUid() { - LOG.info("Creating an ID for kind='" + kind() - + "' name='" + name + '\''); + LOG.info("Creating " + (randomize_id ? "a random " : "an ") + + "ID for kind='" + kind() + "' name='" + name + '\''); state = CREATE_REVERSE_MAPPING; - return client.atomicIncrement(new AtomicIncrementRequest(table, MAXID_ROW, - ID_FAMILY, - kind)); + if (randomize_id) { + return Deferred.fromResult(RandomUniqueId.getRandomUID()); + } else { + return client.atomicIncrement(new AtomicIncrementRequest(table, + MAXID_ROW, ID_FAMILY, kind)); + } } - /** * Create the reverse mapping. * We do this before the forward one so that if we die before creating @@ -474,10 +683,20 @@ private Deferred<?> createForwardMapping(final Object arg) { if (!(arg instanceof Boolean)) { throw new IllegalStateException("Expected a Boolean but got " + arg); } - if (!((Boolean) arg)) { // Previous CAS failed. Something is really messed up. - LOG.error("WTF! Failed to CAS reverse mapping: " + reverseMapping() - + " -- run an fsck against the UID table!"); - return tryAllocate(); // Try again from the beginning. + if (!((Boolean) arg)) { // Previous CAS failed. + if (randomize_id) { + // This random Id is already used by another row + LOG.warn("Detected random id collision and retrying kind='" + + kind() + "' name='" + name + "'"); + random_id_collisions++; + } else { + // something is really messed up then + LOG.error("WTF! Failed to CAS reverse mapping: " + reverseMapping() + + " -- run an fsck against the UID table!"); + } + attempt--; + state = ALLOCATE_UID; + return Deferred.fromResult(false); } state = DONE; @@ -504,6 +723,13 @@ private Deferred<byte[]> done(final Object arg) { // manage to CAS this KV into existence. The one that loses the // race will retry and discover the UID assigned by the winner TSD, // and a UID will have been wasted in the process. No big deal. + if (randomize_id) { + // This random Id is already used by another row + LOG.warn("Detected random id collision between two tsdb " + + "servers kind='" + kind() + "' name='" + name + "'"); + random_id_collisions++; + } + class GetIdCB implements Callback<Object, byte[]> { public Object call(final byte[] row) throws Exception { assignment.callback(row); @@ -523,16 +749,12 @@ public Object call(final byte[] row) throws Exception { tsdb.indexUIDMeta(meta); } - synchronized (pending_assignments) { - pending_assignments.remove(name); - } - assignment.callback(row); synchronized(pending_assignments) { - if (pending_assignments.containsKey(name)) { - pending_assignments.remove(name); + if (pending_assignments.remove(name) != null) { LOG.info("Completed pending assignment for: " + name); } } + assignment.callback(row); return assignment; } @@ -562,6 +784,25 @@ public byte[] getOrCreateId(final String name) throws HBaseException { try { return getIdAsync(name).joinUninterruptibly(); } catch (NoSuchUniqueName e) { + if (tsdb != null && tsdb.getUidFilter() != null && + tsdb.getUidFilter().fillterUIDAssignments()) { + try { + if (!tsdb.getUidFilter().allowUIDAssignment(type, name, null, null) + .join()) { + rejected_assignments++; + throw new FailedToAssignUniqueIdException(new String(kind), name, 0, + "Blocked by UID filter."); + } + } catch (FailedToAssignUniqueIdException e1) { + throw e1; + } catch (InterruptedException e1) { + LOG.error("Interrupted", e1); + Thread.currentThread().interrupt(); + } catch (Exception e1) { + throw new RuntimeException("Should never be here", e1); + } + } + Deferred<byte[]> assignment = null; boolean pending = false; synchronized (pending_assignments) { @@ -597,9 +838,10 @@ public byte[] getOrCreateId(final String name) throws HBaseException { } catch (Exception e1) { throw new RuntimeException("Should never be here", e); } finally { - LOG.info("Completed pending assignment for: " + name); synchronized (pending_assignments) { - pending_assignments.remove(name); + if (pending_assignments.remove(name) != null) { + LOG.info("Completed pending assignment for: " + name); + } } } return uid; @@ -607,7 +849,7 @@ public byte[] getOrCreateId(final String name) throws HBaseException { throw new RuntimeException("Should never be here", e); } } - + /** * Finds the ID associated with a given name or creates it. * <p> @@ -621,37 +863,89 @@ public byte[] getOrCreateId(final String name) throws HBaseException { * @since 1.2 */ public Deferred<byte[]> getOrCreateIdAsync(final String name) { + return getOrCreateIdAsync(name, null, null); + } + + /** + * Finds the ID associated with a given name or creates it. + * <p> + * The length of the byte array is fixed in advance by the implementation. + * + * @param name The name to lookup in the table or to assign an ID to. + * @param metric Name of the metric associated with the UID for filtering. + * @param tags Tag set associated with the UID for filtering. + * @throws HBaseException if there is a problem communicating with HBase. + * @throws IllegalStateException if all possible IDs are already assigned. + * @throws IllegalStateException if the ID found in HBase is encoded on the + * wrong number of bytes. + * @since 2.3 + */ + public Deferred<byte[]> getOrCreateIdAsync(final String name, + final String metric, final Map<String, String> tags) { // Look in the cache first. final byte[] id = getIdFromCache(name); if (id != null) { - cache_hits++; + incrementCacheHits(); return Deferred.fromResult(id); } // Not found in our cache, so look in HBase instead. + /** Triggers the assignment if allowed through the filter */ + class AssignmentAllowedCB implements Callback<Deferred<byte[]>, Boolean> { + @Override + public Deferred<byte[]> call(final Boolean allowed) throws Exception { + if (!allowed) { + rejected_assignments++; + return Deferred.fromError(new FailedToAssignUniqueIdException( + new String(kind), name, 0, "Blocked by UID filter.")); + } + + Deferred<byte[]> assignment = null; + synchronized (pending_assignments) { + assignment = pending_assignments.get(name); + if (assignment == null) { + // to prevent UID leaks that can be caused when multiple time + // series for the same metric or tags arrive, we need to write a + // deferred to the pending map as quickly as possible. Then we can + // start the assignment process after we've stashed the deferred + // and released the lock + assignment = new Deferred<byte[]>(); + pending_assignments.put(name, assignment); + } else { + LOG.info("Already waiting for UID assignment: " + name); + return assignment; + } + } + + // start the assignment dance after stashing the deferred + if (metric != null && LOG.isDebugEnabled()) { + LOG.debug("Assigning UID for '" + name + "' of type '" + type + + "' for series '" + metric + ", " + tags + "'"); + } + + // start the assignment dance after stashing the deferred + return new UniqueIdAllocator(name, assignment).tryAllocate(); + } + @Override + public String toString() { + return "AssignmentAllowedCB"; + } + } + + /** Triggers an assignment (possibly through the filter) if the exception + * returned was a NoSuchUniqueName. */ class HandleNoSuchUniqueNameCB implements Callback<Object, Exception> { public Object call(final Exception e) { if (e instanceof NoSuchUniqueName) { - - Deferred<byte[]> assignment = null; - synchronized (pending_assignments) { - assignment = pending_assignments.get(name); - if (assignment == null) { - // to prevent UID leaks that can be caused when multiple time - // series for the same metric or tags arrive, we need to write a - // deferred to the pending map as quickly as possible. Then we can - // start the assignment process after we've stashed the deferred - // and released the lock - assignment = new Deferred<byte[]>(); - pending_assignments.put(name, assignment); - } else { - LOG.info("Already waiting for UID assignment: " + name); - return assignment; - } + if (tsdb != null && tsdb.getUidFilter() != null && + tsdb.getUidFilter().fillterUIDAssignments()) { + return tsdb.getUidFilter() + .allowUIDAssignment(type, name, metric, tags) + .addCallbackDeferring(new AssignmentAllowedCB()); + } else { + return Deferred.fromResult(true) + .addCallbackDeferring(new AssignmentAllowedCB()); } - - // start the assignment dance after stashing the deferred - return new UniqueIdAllocator(name, assignment).tryAllocate(); } return e; // Other unexpected exception, let it bubble up. } @@ -757,7 +1051,8 @@ public Object call(final ArrayList<ArrayList<KeyValue>> rows) { final byte[] key = row.get(0).key(); final String name = fromBytes(key); final byte[] id = row.get(0).value(); - final byte[] cached_id = name_cache.get(name); + final byte[] cached_id = use_lru ? lru_name_cache.getIfPresent(name) : + name_cache.get(name); if (cached_id == null) { cacheMapping(name, id); } else if (!Arrays.equals(id, cached_id)) { @@ -799,6 +1094,7 @@ public Object call(Object ignored) throws Exception { */ public void rename(final String oldname, final String newname) { final byte[] row = getId(oldname); + final String row_string = fromBytes(row); { byte[] id = null; try { @@ -813,6 +1109,15 @@ public void rename(final String oldname, final String newname) { } } + if (renaming_id_names.contains(row_string) + || renaming_id_names.contains(newname)) { + throw new IllegalArgumentException("Ongoing rename on the same ID(\"" + + Arrays.toString(row) + "\") or an identical new name(\"" + newname + + "\")"); + } + renaming_id_names.add(row_string); + renaming_id_names.add(newname); + final byte[] newnameb = toBytes(newname); // Update the reverse mapping first, so that if we die before updating @@ -828,6 +1133,8 @@ public void rename(final String oldname, final String newname) { LOG.error("When trying rename(\"" + oldname + "\", \"" + newname + "\") on " + this + ": Failed to update reverse" + " mapping for ID=" + Arrays.toString(row), e); + renaming_id_names.remove(row_string); + renaming_id_names.remove(newname); throw e; } @@ -841,13 +1148,20 @@ public void rename(final String oldname, final String newname) { LOG.error("When trying rename(\"" + oldname + "\", \"" + newname + "\") on " + this + ": Failed to create the" + " new forward mapping with ID=" + Arrays.toString(row), e); + renaming_id_names.remove(row_string); + renaming_id_names.remove(newname); throw e; } // Update cache. addIdToCache(newname, row); // add new name -> ID - id_cache.put(fromBytes(row), newname); // update ID -> new name - name_cache.remove(oldname); // remove old name -> ID + if (use_lru) { + lru_id_cache.put(fromBytes(row), newname); + lru_name_cache.invalidate(oldname); + } else { + id_cache.put(fromBytes(row), newname); // update ID -> new name + name_cache.remove(oldname); // remove old name -> ID + } // Delete the old forward mapping. try { @@ -865,10 +1179,123 @@ public void rename(final String oldname, final String newname) { + " old forward mapping for ID=" + Arrays.toString(row); LOG.error("WTF? " + msg, e); throw new RuntimeException(msg, e); + } finally { + renaming_id_names.remove(row_string); + renaming_id_names.remove(newname); } // Success! } + /** + * Attempts to remove the mappings for the given string from the UID table + * as well as the cache. If used, the caller should remove the entry from all + * TSD caches as well. + * <p> + * WARNING: This is a best attempt only method in that we'll lookup the UID + * for the given string, then issue two delete requests, one for each mapping. + * If either mapping fails then the cache can be re-populated later on with + * stale data. In that case, please run the FSCK utility. + * <p> + * WARNING 2: This method will NOT delete time series data or TSMeta data + * associated with the UIDs. It only removes them from the UID table. Deleting + * a metric is generally safe as you won't query over it in the future. But + * deleting tag keys or values can cause queries to fail if they find data + * without a corresponding name. + * + * @param name The name of the UID to delete + * @return A deferred to wait on for completion. The result will be null if + * successful, an exception otherwise. + * @throws NoSuchUniqueName if the UID string did not exist in storage + * @throws IllegalStateException if the TSDB wasn't set for this UID object + * @since 2.2 + */ + public Deferred<Object> deleteAsync(final String name) { + if (tsdb == null) { + throw new IllegalStateException("The TSDB is null for this UID object."); + } + final byte[] uid = new byte[id_width]; + final ArrayList<Deferred<Object>> deferreds = + new ArrayList<Deferred<Object>>(2); + + /** Catches errors and still cleans out the cache */ + class ErrCB implements Callback<Object, Exception> { + @Override + public Object call(final Exception ex) throws Exception { + if (use_lru) { + lru_name_cache.invalidate(name); + lru_id_cache.invalidate(fromBytes(uid)); + } else { + name_cache.remove(name); + id_cache.remove(fromBytes(uid)); + } + LOG.error("Failed to delete " + fromBytes(kind) + " UID " + name + + " but still cleared the cache", ex); + return ex; + } + } + + /** Used to wait on the group of delete requests */ + class GroupCB implements Callback<Deferred<Object>, ArrayList<Object>> { + @Override + public Deferred<Object> call(final ArrayList<Object> response) + throws Exception { + if (use_lru) { + lru_name_cache.invalidate(name); + lru_id_cache.invalidate(fromBytes(uid)); + } else { + name_cache.remove(name); + id_cache.remove(fromBytes(uid)); + } + LOG.info("Successfully deleted " + fromBytes(kind) + " UID " + name); + return Deferred.fromResult(null); + } + } + + /** Called after fetching the UID from storage */ + class LookupCB implements Callback<Deferred<Object>, byte[]> { + @Override + public Deferred<Object> call(final byte[] stored_uid) throws Exception { + if (stored_uid == null) { + return Deferred.fromError(new NoSuchUniqueName(kind(), name)); + } + System.arraycopy(stored_uid, 0, uid, 0, id_width); + final DeleteRequest forward = + new DeleteRequest(table, toBytes(name), ID_FAMILY, kind); + deferreds.add(tsdb.getClient().delete(forward)); + + final DeleteRequest reverse = + new DeleteRequest(table, uid, NAME_FAMILY, kind); + deferreds.add(tsdb.getClient().delete(reverse)); + + final DeleteRequest meta = new DeleteRequest(table, uid, NAME_FAMILY, + toBytes((type.toString().toLowerCase() + "_meta"))); + deferreds.add(tsdb.getClient().delete(meta)); + return Deferred.group(deferreds).addCallbackDeferring(new GroupCB()); + } + } + + final byte[] cached_uid = use_lru ? lru_name_cache.getIfPresent(name) : + name_cache.get(name); + if (cached_uid == null) { + return getIdFromHBase(name).addCallbackDeferring(new LookupCB()) + .addErrback(new ErrCB()); + } + System.arraycopy(cached_uid, 0, uid, 0, id_width); + final DeleteRequest forward = + new DeleteRequest(table, toBytes(name), ID_FAMILY, kind); + deferreds.add(tsdb.getClient().delete(forward)); + + final DeleteRequest reverse = + new DeleteRequest(table, uid, NAME_FAMILY, kind); + deferreds.add(tsdb.getClient().delete(reverse)); + + final DeleteRequest meta = new DeleteRequest(table, uid, NAME_FAMILY, + toBytes((type.toString().toLowerCase() + "_meta"))); + deferreds.add(tsdb.getClient().delete(meta)); + return Deferred.group(deferreds).addCallbackDeferring(new GroupCB()) + .addErrback(new ErrCB()); + } + /** The start row to scan on empty search strings. `!' = first ASCII char. */ private static final byte[] START_ROW = new byte[] { '!' }; @@ -1141,15 +1568,26 @@ public static byte[] stringToUid(final String uid, final short uid_length) { * @param row_key The row key to process * @param metric_width The width of the metric * @param timestamp_width The width of the timestamp - * @return The TSUID - * @throws ArrayIndexOutOfBoundsException if the row_key is invalid + * @return The TSUID as a byte array + * @throws IllegalArgumentException if the row key is missing tags or it is + * corrupt such as a salted key when salting is disabled or vice versa. */ public static byte[] getTSUIDFromKey(final byte[] row_key, final short metric_width, final short timestamp_width) { int idx = 0; - final byte[] tsuid = new byte[row_key.length - timestamp_width]; - for (int i = 0; i < row_key.length; i++) { - if (i < metric_width || i >= (metric_width + timestamp_width)) { + // validation + final int tag_pair_width = TSDB.tagk_width() + TSDB.tagv_width(); + final int tags_length = row_key.length - + (Const.SALT_WIDTH() + metric_width + timestamp_width); + if (tags_length < tag_pair_width || (tags_length % tag_pair_width) != 0) { + throw new IllegalArgumentException( + "Row key is missing tags or it is corrupted " + Arrays.toString(row_key)); + } + final byte[] tsuid = new byte[ + row_key.length - timestamp_width - Const.SALT_WIDTH()]; + for (int i = Const.SALT_WIDTH(); i < row_key.length; i++) { + if (i < Const.SALT_WIDTH() + metric_width || + i >= (Const.SALT_WIDTH() + metric_width + timestamp_width)) { tsuid[idx] = row_key[i]; idx++; } @@ -1314,7 +1752,7 @@ public Map<String, Long> call(final ArrayList<KeyValue> row) * @param uid_cache_map A map of {@link UniqueId} objects keyed on the kind. * @throws HBaseException Passes any HBaseException from HBase scanner. * @throws RuntimeException Wraps any non HBaseException from HBase scanner. - * @2.1 + * @since 2.1 */ public static void preloadUidCache(final TSDB tsdb, final ByteMap<UniqueId> uid_cache_map) throws HBaseException { @@ -1354,8 +1792,10 @@ public static void preloadUidCache(final TSDB tsdb, for (UniqueId unique_id_table : uid_cache_map.values()) { LOG.info("After preloading, uid cache '{}' has {} ids and {} names.", unique_id_table.kind(), - unique_id_table.id_cache.size(), - unique_id_table.name_cache.size()); + unique_id_table.use_lru ? unique_id_table.lru_id_cache.size() : + unique_id_table.id_cache.size(), + unique_id_table.use_lru ? unique_id_table.lru_name_cache.size() : + unique_id_table.name_cache.size()); } } catch (Exception e) { if (e instanceof HBaseException) { @@ -1371,4 +1811,24 @@ public static void preloadUidCache(final TSDB tsdb, } } } + + @VisibleForTesting + Map<String, byte[]> nameCache() { + return name_cache; + } + + @VisibleForTesting + Map<String, String> idCache() { + return id_cache; + } + + @VisibleForTesting + Cache<String, byte[]> lruNameCache() { + return lru_name_cache; + } + + @VisibleForTesting + Cache<String, String> lruIdCache() { + return lru_id_cache; + } } diff --git a/src/uid/UniqueIdFilterPlugin.java b/src/uid/UniqueIdFilterPlugin.java new file mode 100644 index 0000000000..60006c0742 --- /dev/null +++ b/src/uid/UniqueIdFilterPlugin.java @@ -0,0 +1,101 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.uid; + +import java.util.Map; + +import com.stumbleupon.async.Deferred; + +import net.opentsdb.core.TSDB; +import net.opentsdb.stats.StatsCollector; +import net.opentsdb.uid.UniqueId.UniqueIdType; + +/** + * A filter that can determine whether or not UIDs should be allowed assignment + * based on their metric and tags. This is useful for such situations as: + * <ul><li>Enforcing naming standards</li> + * <li>Blacklisting certain names or properties</li> + * <li>Preventing cardinality explosions</li></ul> + * <b>Note:</b> Implementations must have a parameterless constructor. The + * {@link #initialize(TSDB)} method will be called immediately after the plugin is + * instantiated and before any other methods are called. + * @since 2.3 + */ +public abstract class UniqueIdFilterPlugin { + + /** + * Called by TSDB to initialize the plugin + * Implementations are responsible for setting up any IO they need as well + * as starting any required background threads. + * <b>Note:</b> Implementations should throw exceptions if they can't start + * up properly. The TSD will then shutdown so the operator can fix the + * problem. Please use IllegalArgumentException for configuration issues. + * @param tsdb The parent TSDB object + * @throws IllegalArgumentException if required configuration parameters are + * missing + * @throws RuntimeException if something else goes wrong + */ + public abstract void initialize(final TSDB tsdb); + + /** + * Called to gracefully shutdown the plugin. Implementations should close + * any IO they have open + * @return A deferred object that indicates the completion of the request. + * The {@link Object} has not special meaning and can be {@code null} + * (think of it as {@code Deferred<Void>}). + */ + public abstract Deferred<Object> shutdown(); + + /** + * Should return the version of this plugin in the format: + * MAJOR.MINOR.MAINT, e.g. "2.3.1". The MAJOR version should match the major + * version of OpenTSDB the plugin is meant to work with. + * @return A version string used to log the loaded version + */ + public abstract String version(); + + /** + * Called by the TSD when a request for statistics collection has come in. The + * implementation may provide one or more statistics. If no statistics are + * available for the implementation, simply stub the method. + * @param collector The collector used for emitting statistics + */ + public abstract void collectStats(final StatsCollector collector); + + /** + * Determine whether or not the UID should be assigned. + * If the UID should not be assigned a value, the implementation can return + * false or an exception in the deferred object. Otherwise it should return + * true and the UID will proceed with assignment. + * NOTE: In some cases the metric and tags may be null, particularly if the + * synchronous APIs were called. In such a situation, make sure the + * implementation handles it properly. + * @param type The type of UID being assigned + * @param value The string value of the UID + * @param metric The metric name associated with the UID for assignment + * @param tags The tag set associated with the UID + * @return True if the UID should be assigned, false if not. + */ + public abstract Deferred<Boolean> allowUIDAssignment( + final UniqueIdType type, + final String value, + final String metric, + final Map<String, String> tags); + + /** + * Whether or not the filter should process UIDs. + * @return True if {@link #allowUIDAssignment(UniqueIdType, String, String, Map)} + * should be called, false if not. + */ + public abstract boolean fillterUIDAssignments(); +} diff --git a/src/uid/UniqueIdWhitelistFilter.java b/src/uid/UniqueIdWhitelistFilter.java new file mode 100644 index 0000000000..371ea7bd50 --- /dev/null +++ b/src/uid/UniqueIdWhitelistFilter.java @@ -0,0 +1,200 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.uid; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +import com.google.common.annotations.VisibleForTesting; +import com.stumbleupon.async.Deferred; + +import net.opentsdb.core.TSDB; +import net.opentsdb.stats.StatsCollector; +import net.opentsdb.uid.UniqueId.UniqueIdType; +import net.opentsdb.utils.Config; + +/** + * A UID filter implementation using regular expression based whitelists. + * Multiple regular expressions can be provided in the configuration file with + * a configurable delimiter. Each expression is compiled into a list per UID + * type and when a new UID passes through the filter, each expression in the + * list is compared to make sure the name satisfies all expressions. + */ +public class UniqueIdWhitelistFilter extends UniqueIdFilterPlugin { + + /** Default delimiter */ + public static final String DEFAULT_REGEX_DELIMITER = ","; + + /** Lists of patterns for each type. */ + private List<Pattern> metric_patterns; + private List<Pattern> tagk_patterns; + private List<Pattern> tagv_patterns; + + /** Counters for tracking stats */ + private final AtomicLong metrics_rejected = new AtomicLong(); + private final AtomicLong metrics_allowed = new AtomicLong(); + private final AtomicLong tagks_rejected = new AtomicLong(); + private final AtomicLong tagks_allowed = new AtomicLong(); + private final AtomicLong tagvs_rejected = new AtomicLong(); + private final AtomicLong tagvs_allowed = new AtomicLong(); + + @Override + public void initialize(final TSDB tsdb) { + final Config config = tsdb.getConfig(); + String delimiter = config.getString("tsd.uidfilter.whitelist.delimiter"); + if (delimiter == null) { + delimiter = DEFAULT_REGEX_DELIMITER; + } + + String raw = config.getString("tsd.uidfilter.whitelist.metric_patterns"); + if (raw != null) { + final String[] splits = raw.split(delimiter); + metric_patterns = new ArrayList<Pattern>(splits.length); + for (final String pattern : splits) { + try { + metric_patterns.add(Pattern.compile(pattern)); + } catch (PatternSyntaxException e) { + throw new IllegalArgumentException("The metric whitelist pattern [" + + pattern + "] does not compile.", e); + } + } + } + + raw = config.getString("tsd.uidfilter.whitelist.tagk_patterns"); + if (raw != null) { + final String[] splits = raw.split(delimiter); + tagk_patterns = new ArrayList<Pattern>(splits.length); + for (final String pattern : splits) { + try { + tagk_patterns.add(Pattern.compile(pattern)); + } catch (PatternSyntaxException e) { + throw new IllegalArgumentException("The tagk whitelist pattern [" + + pattern + "] does not compile.", e); + } + } + } + + raw = config.getString("tsd.uidfilter.whitelist.tagv_patterns"); + if (raw != null) { + final String[] splits = raw.split(delimiter); + tagv_patterns = new ArrayList<Pattern>(splits.length); + for (final String pattern : splits) { + try { + tagv_patterns.add(Pattern.compile(pattern)); + } catch (PatternSyntaxException e) { + throw new IllegalArgumentException("The tagv whitelist pattern [" + + pattern + "] does not compile.", e); + } + } + } + } + + @Override + public Deferred<Object> shutdown() { + return Deferred.fromResult(null); + } + + @Override + public String version() { + return "2.3.0"; + } + + @Override + public void collectStats(final StatsCollector collector) { + collector.record("uid.filter.whitelist.accepted", metrics_allowed.get(), + "type=metrics"); + collector.record("uid.filter.whitelist.accepted", tagks_allowed.get(), + "type=tagk"); + collector.record("uid.filter.whitelist.accepted", tagvs_allowed.get(), + "type=tagv"); + collector.record("uid.filter.whitelist.rejected", metrics_rejected.get(), + "type=metrics"); + collector.record("uid.filter.whitelist.rejected", tagks_rejected.get(), + "type=tagk"); + collector.record("uid.filter.whitelist.rejected", tagvs_rejected.get(), + "type=tagv"); + } + + @Override + public Deferred<Boolean> allowUIDAssignment( + final UniqueIdType type, + final String value, + final String metric, + final Map<String, String> tags) { + + switch (type) { + case METRIC: + if (metric_patterns != null) { + for (final Pattern pattern : metric_patterns) { + if (!pattern.matcher(value).find()) { + metrics_rejected.incrementAndGet(); + return Deferred.fromResult(false); + } + } + } + metrics_allowed.incrementAndGet(); + break; + + case TAGK: + if (tagk_patterns != null) { + for (final Pattern pattern : tagk_patterns) { + if (!pattern.matcher(value).find()) { + tagks_rejected.incrementAndGet(); + return Deferred.fromResult(false); + } + } + } + tagks_allowed.incrementAndGet(); + break; + + case TAGV: + if (tagv_patterns != null) { + for (final Pattern pattern : tagv_patterns) { + if (!pattern.matcher(value).find()) { + tagvs_rejected.incrementAndGet(); + return Deferred.fromResult(false); + } + } + } + tagvs_allowed.incrementAndGet(); + break; + } + + // all patterns passed, yay! + return Deferred.fromResult(true); + } + + @Override + public boolean fillterUIDAssignments() { + return true; + } + + @VisibleForTesting + List<Pattern> metricPatterns() { + return metric_patterns; + } + + @VisibleForTesting + List<Pattern> tagkPatterns() { + return tagk_patterns; + } + + @VisibleForTesting + List<Pattern> tagvPatterns() { + return tagv_patterns; + } +} diff --git a/src/utils/ByteSet.java b/src/utils/ByteSet.java new file mode 100644 index 0000000000..6a1bad1d41 --- /dev/null +++ b/src/utils/ByteSet.java @@ -0,0 +1,110 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.utils; + +import java.util.AbstractSet; +import java.util.Arrays; +import java.util.Iterator; +import java.util.Set; + +import org.hbase.async.Bytes.ByteMap; + +/** + * An implementation of a set based on the AsyncHBase ByteMap. This provides + * a unique set implementation of byte arrays, matching on the contents of + * the arrays, not on the hash codes. + */ +public class ByteSet extends AbstractSet<byte[]> + implements Set<byte[]>, Cloneable, java.io.Serializable { + + private static final long serialVersionUID = -496061795957902656L; + + // Dummy value to associate with an Object in the backing Map + private static final Object PRESENT = new Object(); + + private transient ByteMap<Object> map; + + /** + * Instantiates a unique set of byte arrays based on the array contents. + */ + public ByteSet() { + map = new ByteMap<Object>(); + } + + @Override + public Iterator<byte[]> iterator() { + return map.keySet().iterator(); + } + + @Override + public int size() { + return map.size(); + } + + @Override + public boolean isEmpty() { + return map.isEmpty(); + } + + @Override + public boolean contains(final Object key) { + return map.containsKey(key); + } + + @Override + public boolean add(final byte[] key) { + return map.put(key, PRESENT) == null; + } + + @Override + public boolean remove(final Object key) { + return map.remove(key) == PRESENT; + } + + @Override + public void clear() { + map.clear(); + } + + @Override + public ByteSet clone() { + try { + ByteSet new_set = (ByteSet) super.clone(); + new_set.map = (ByteMap<Object>) map.clone(); + return new_set; + } catch (CloneNotSupportedException e) { + throw new InternalError(); + } + } + + @Override + public String toString() { + final Iterator<byte[]> it = map.keySet().iterator(); + if (!it.hasNext()) { + return "[]"; + } + + final StringBuilder buf = new StringBuilder(); + buf.append('['); + for (;;) { + final byte[] array = it.next(); + buf.append(Arrays.toString(array)); + if (!it.hasNext()) { + return buf.append(']').toString(); + } + buf.append(','); + } + } + + // TODO - writeObject, readObject +} diff --git a/src/utils/Config.java b/src/utils/Config.java index c4ab40ae49..5c9e78b83f 100644 --- a/src/utils/Config.java +++ b/src/utils/Config.java @@ -21,6 +21,7 @@ import java.util.Map; import java.util.Properties; +import net.opentsdb.core.RpcResponder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,18 +29,18 @@ /** * OpenTSDB Configuration Class - * + * * This handles all of the user configurable variables for a TSD. On * initialization default values are configured for all variables. Then * implementations should call the {@link #loadConfig()} methods to search for a * default configuration or try to load one provided by the user. - * + * * To add a configuration, simply set a default value in {@link #setDefaults()}. * Wherever you need to access the config value, use the proper helper to fetch * the value, accounting for exceptions that may be thrown if necessary. - * - * The get<type> number helpers will return NumberFormatExceptions if the - * requested property is null or unparseable. The {@link #getString(String)} + * + * The get<type> number helpers will return NumberFormatExceptions if the + * requested property is null or unparseable. The {@link #getString(String)} * helper will return a NullPointerException if the property isn't found. * <p> * Plugins can extend this class and copy the properties from the main @@ -53,11 +54,11 @@ public class Config { private static final Logger LOG = LoggerFactory.getLogger(Config.class); /** Flag to determine if we're running under Windows or not */ - public static final boolean IS_WINDOWS = + public static final boolean IS_WINDOWS = System.getProperty("os.name", "").contains("Windows"); - + // These are accessed often so need a set address for fast access (faster - // than accessing the map. Their value will be changed when the config is + // than accessing the map. Their value will be changed when the config is // loaded // NOTE: edit the setDefaults() method if you add a public field @@ -66,47 +67,77 @@ public class Config { /** tsd.core.auto_create_tagk */ private boolean auto_tagk = true; - + /** tsd.core.auto_create_tagv */ private boolean auto_tagv = true; - + /** tsd.storage.enable_compaction */ private boolean enable_compactions = true; + /** tsd.storage.enable_appends */ + private boolean enable_appends = false; + + /** tsd.storage.repair_appends */ + private boolean repair_appends = false; + /** tsd.core.meta.enable_realtime_ts */ private boolean enable_realtime_ts = false; - + /** tsd.core.meta.enable_realtime_uid */ private boolean enable_realtime_uid = false; - + /** tsd.core.meta.enable_tsuid_incrementing */ private boolean enable_tsuid_incrementing = false; - + /** tsd.core.meta.enable_tsuid_tracking */ private boolean enable_tsuid_tracking = false; - + /** tsd.http.request.enable_chunked */ private boolean enable_chunked_requests = false; /** tsd.storage.fix_duplicates */ private boolean fix_duplicates = false; + /** tsd.http.header_tag */ + private String http_header_tag = null; + /** tsd.http.request.max_chunk */ - private int max_chunked_requests = 4096; - + private int max_chunked_requests = 4096; + /** tsd.core.tree.enable_processing */ private boolean enable_tree_processing = false; + + /** tsd.storage.hbase.scanner.maxNumRows */ + private int scanner_max_num_rows = 128; + + /** tsd.storage.use_otsdb_timestamp */ + /** Sets the HBase cell timestamp equal to metric timestamp */ + private boolean use_otsdb_timestamp = true; + + /** tsd.storage.get_date_tiered_compaction_start */ + /** Sets the time at which you started using use_otsdb_timestamp + * this value is overriden if you have existing data in your tsdb table + * but don't want to re-write the cell timestamps, therefore you can start + * set this timestamp to when you start using use_otsdb_timestamp + * */ + private long get_date_tiered_compaction_start = 0; + + + /** tsd.storage.use_max_value */ + /** Used for resolving between data coming in at same timestamp */ + /** If set to true, the maximum value will be returned, minimum */ + private boolean use_max_value = true; /** * The list of properties configured to their defaults or modified by users */ - protected final HashMap<String, String> properties = + protected final HashMap<String, String> properties = new HashMap<String, String>(); /** Holds default values for the config */ - protected static final HashMap<String, String> default_map = + protected static final HashMap<String, String> default_map = new HashMap<String, String>(); - + /** Tracks the location of the file that was actually loaded */ protected String config_location; @@ -139,7 +170,7 @@ public Config(final String file) throws IOException { /** * Constructor for plugins or overloaders who want a copy of the parent * properties but without the ability to modify them - * + * * This constructor will not re-read the file, but it will copy the location * so if a child wants to reload the properties periodically, they may do so * @param parent Parent configuration object to load from @@ -151,58 +182,96 @@ public Config(final Config parent) { setDefaults(); } + /** + * Creates a new empty Config + */ + public Config() { + + } + + /** @return The file that generated this config. May be null */ + public String configLocation() { + return config_location; + } + /** @return the auto_metric value */ public boolean auto_metric() { return auto_metric; } - + /** @return the auto_tagk value */ public boolean auto_tagk() { return auto_tagk; } - + /** @return the auto_tagv value */ public boolean auto_tagv() { return auto_tagv; } - + /** @param auto_metric whether or not to auto create metrics */ public void setAutoMetric(boolean auto_metric) { this.auto_metric = auto_metric; - properties.put("tsd.core.auto_create_metrics", + properties.put("tsd.core.auto_create_metrics", Boolean.toString(auto_metric)); } - + /** @return the enable_compaction value */ public boolean enable_compactions() { return enable_compactions; } - + + /** @return whether or not to write data in the append format */ + public boolean enable_appends() { + return enable_appends; + } + + /** @return whether or not to re-write appends with duplicates or out of order + * data when queried. */ + public boolean repair_appends() { + return repair_appends; + } + /** @return whether or not to record new TSMeta objects in real time */ - public boolean enable_realtime_ts() { + public boolean enable_realtime_ts() { return enable_realtime_ts; } - + /** @return whether or not record new UIDMeta objects in real time */ - public boolean enable_realtime_uid() { + public boolean enable_realtime_uid() { return enable_realtime_uid; } - + /** @return whether or not to increment TSUID counters */ - public boolean enable_tsuid_incrementing() { + public boolean enable_tsuid_incrementing() { return enable_tsuid_incrementing; } - + /** @return whether or not to record a 1 for every TSUID */ public boolean enable_tsuid_tracking() { return enable_tsuid_tracking; } + + /** @return maximum number of rows to be fetched per round trip while scanning HBase */ + public int scanner_maxNumRows() { + return scanner_max_num_rows; + } + + /** @return whether or not additional http header tag is allowed */ + public boolean enable_header_tag() { + return http_header_tag != null ; + } + + /** @return the lookup value for additional http header tag */ + public String get_name_header_tag() { + return http_header_tag ; + } /** @return whether or not chunked requests are supported */ public boolean enable_chunked_requests() { return enable_chunked_requests; } - + /** @return max incoming chunk size in bytes */ public int max_chunked_requests() { return max_chunked_requests; @@ -223,13 +292,26 @@ public boolean enable_tree_processing() { return enable_tree_processing; } + public boolean use_otsdb_timestamp() { + return use_otsdb_timestamp; + } + + /** @return the time at which you started storing data using otsdb_timestamp(), if not set defaults to 0 */ + public long get_date_tiered_compaction_start() { + return get_date_tiered_compaction_start; + } + + public boolean use_max_value() { + return use_max_value; + } + /** * Allows for modifying properties after creation or loading. - * - * WARNING: This should only be used on initialization and is meant for - * command line overrides. Also note that it will reset all static config + * + * WARNING: This should only be used on initialization and is meant for + * command line overrides. Also note that it will reset all static config * variables when called. - * + * * @param property The name of the property to override * @param value The value to store */ @@ -260,10 +342,27 @@ public final int getInt(final String property) { } /** - * Returns the given string trimed or null if is null - * @param string The string be trimmed of + * Returns the given property as an integer. + * If no such property is specified, or if the specified value is not a valid + * <code>Int</code>, then <code>default_val</code> is returned. + * + * @param property The property to load + * @param default_val default value + * @return A parsed integer or default_val. + */ + public final int getInt(final String property, final int default_val) { + try { + return getInt(property); + } catch (Exception e) { + return default_val; + } + } + + /** + * Returns the given string trimed or null if is null + * @param string The string be trimmed of * @return The string trimed or null - */ + */ private final String sanitize(final String string) { if (string == null) { return null; @@ -318,12 +417,12 @@ public final double getDouble(final String property) { /** * Returns the given property as a boolean - * + * * Property values are case insensitive and the following values will result * in a True return value: - 1 - True - Yes - * + * * Any other values, including an empty string, will result in a False - * + * * @param property The property to load * @return A parsed boolean * @throws NullPointerException if the property was not found @@ -339,6 +438,23 @@ public final boolean getBoolean(final String property) { return false; } + /** + * Returns the given property as an boolean. + * If no such property is specified, or if the specified value is not a valid + * <code>boolean</code>, then <code>default_val</code> is returned. + * + * @param property The property to load + * @param default_val default value + * @return A parsed boolean or default_val. + */ + public final boolean getBoolean(final String property, final boolean default_val) { + try { + return getBoolean(property); + } catch (Exception e) { + return default_val; + } + } + /** * Returns the directory name, making sure the end is an OS dependent slash * @param property The property to load @@ -347,10 +463,13 @@ public final boolean getBoolean(final String property) { */ public final String getDirectoryName(final String property) { String directory = properties.get(property); + if (directory == null || directory.isEmpty()){ + return null; + } if (IS_WINDOWS) { // Windows swings both ways. If a forward slash was already used, we'll // add one at the end if missing. Otherwise use the windows default of \ - if (directory.charAt(directory.length() - 1) == '\\' || + if (directory.charAt(directory.length() - 1) == '\\' || directory.charAt(directory.length() - 1) == '/') { return directory; } @@ -363,17 +482,17 @@ public final String getDirectoryName(final String property) { throw new IllegalArgumentException( "Unix path names cannot contain a back slash"); } - + if (directory == null || directory.isEmpty()){ return null; } - + if (directory.charAt(directory.length() - 1) == '/') { return directory; } return directory + "/"; } - + /** * Determines if the given propery is in the map * @param property The property to search for @@ -419,10 +538,24 @@ public final String dumpConfiguration() { public final Map<String, String> getMap() { return ImmutableMap.copyOf(properties); } - + + /** + * set enable_compactions to true + */ + public final void enableCompactions() { + this.enable_compactions = true; + } + + /** + * set enable_compactions to false + */ + public final void disableCompactions() { + this.enable_compactions = false; + } + /** * Loads default entries that were not provided by a file or command line - * + * * This should be called in the constructor */ protected void setDefaults() { @@ -437,23 +570,58 @@ protected void setDefaults() { default_map.put("tsd.network.tcp_no_delay", "true"); default_map.put("tsd.network.keep_alive", "true"); default_map.put("tsd.network.reuse_address", "true"); + default_map.put("tsd.core.authentication.enable", "false"); + default_map.put("tsd.core.authentication.plugin", ""); default_map.put("tsd.core.auto_create_metrics", "false"); default_map.put("tsd.core.auto_create_tagks", "true"); default_map.put("tsd.core.auto_create_tagvs", "true"); + default_map.put("tsd.core.connections.limit", "0"); + default_map.put("tsd.core.enable_api", "true"); + default_map.put("tsd.core.enable_ui", "true"); default_map.put("tsd.core.meta.enable_realtime_ts", "false"); default_map.put("tsd.core.meta.enable_realtime_uid", "false"); default_map.put("tsd.core.meta.enable_tsuid_incrementing", "false"); default_map.put("tsd.core.meta.enable_tsuid_tracking", "false"); + default_map.put("tsd.core.meta.cache.enable", "false"); default_map.put("tsd.core.plugin_path", ""); default_map.put("tsd.core.socket.timeout", "0"); default_map.put("tsd.core.tree.enable_processing", "false"); default_map.put("tsd.core.preload_uid_cache", "false"); default_map.put("tsd.core.preload_uid_cache.max_entries", "300000"); + default_map.put("tsd.core.storage_exception_handler.enable", "false"); + default_map.put("tsd.core.uid.random_metrics", "false"); + default_map.put("tsd.core.bulk.allow_out_of_order_timestamps", "false"); + default_map.put("tsd.gnuplot.options.allowlist", ";axis x1y2"); + default_map.put("tsd.query.filter.expansion_limit", "4096"); + default_map.put("tsd.query.skip_unresolved_tagvs", "false"); + default_map.put("tsd.query.allow_simultaneous_duplicates", "true"); + default_map.put("tsd.query.enable_fuzzy_filter", "true"); + default_map.put("tsd.query.limits.bytes.default", "0"); + default_map.put("tsd.query.limits.bytes.allow_override", "false"); + default_map.put("tsd.query.limits.data_points.default", "0"); + default_map.put("tsd.query.limits.data_points.allow_override", "false"); + default_map.put("tsd.query.limits.overrides.interval", "60000"); + default_map.put("tsd.query.multi_get.enable", "false"); + default_map.put("tsd.query.multi_get.limit", "131072"); + default_map.put("tsd.query.multi_get.batch_size", "1024"); + default_map.put("tsd.query.multi_get.concurrent", "20"); + default_map.put("tsd.query.multi_get.get_all_salts", "false"); + default_map.put("tsd.rpc.telnet.return_errors", "true"); + // Rollup related settings + default_map.put("tsd.rollups.enable", "false"); + default_map.put("tsd.rollups.tag_raw", "false"); + default_map.put("tsd.rollups.agg_tag_key", "_aggregate"); + default_map.put("tsd.rollups.raw_agg_tag_value", "RAW"); + default_map.put("tsd.rollups.block_derived", "true"); + default_map.put("tsd.rollups.split_query.enable", "false"); default_map.put("tsd.rtpublisher.enable", "false"); default_map.put("tsd.rtpublisher.plugin", ""); default_map.put("tsd.search.enable", "false"); default_map.put("tsd.search.plugin", ""); default_map.put("tsd.stats.canonical", "false"); + default_map.put("tsd.startup.enable", "false"); + default_map.put("tsd.startup.plugin", ""); + default_map.put("tsd.storage.hbase.scanner.maxNumRows", "128"); default_map.put("tsd.storage.fix_duplicates", "false"); default_map.put("tsd.storage.flush_interval", "1000"); default_map.put("tsd.storage.hbase.data_table", "tsdb"); @@ -462,14 +630,34 @@ protected void setDefaults() { default_map.put("tsd.storage.hbase.meta_table", "tsdb-meta"); default_map.put("tsd.storage.hbase.zk_quorum", "localhost"); default_map.put("tsd.storage.hbase.zk_basedir", "/hbase"); + default_map.put("tsd.storage.hbase.prefetch_meta", "false"); + default_map.put("tsd.storage.enable_appends", "false"); + default_map.put("tsd.storage.repair_appends", "false"); default_map.put("tsd.storage.enable_compaction", "true"); + default_map.put("tsd.storage.compaction.flush_interval", "10"); + default_map.put("tsd.storage.compaction.min_flush_threshold", "100"); + default_map.put("tsd.storage.compaction.max_concurrent_flushes", "10000"); + default_map.put("tsd.storage.compaction.flush_speed", "2"); + default_map.put("tsd.timeseriesfilter.enable", "false"); + default_map.put("tsd.uid.use_mode", "false"); + default_map.put("tsd.uid.lru.enable", "false"); + default_map.put("tsd.uid.lru.name.size", "5000000"); + default_map.put("tsd.uid.lru.id.size", "5000000"); + default_map.put("tsd.uidfilter.enable", "false"); + default_map.put("tsd.core.stats_with_port", "false"); default_map.put("tsd.http.show_stack_trace", "true"); + default_map.put("tsd.http.query.allow_delete", "false"); + default_map.put("tsd.http.header_tag", ""); default_map.put("tsd.http.request.enable_chunked", "false"); default_map.put("tsd.http.request.max_chunk", "4096"); default_map.put("tsd.http.request.cors_domains", ""); default_map.put("tsd.http.request.cors_headers", "Authorization, " + "Content-Type, Accept, Origin, User-Agent, DNT, Cache-Control, " + "X-Mx-ReqToken, Keep-Alive, X-Requested-With, If-Modified-Since"); + default_map.put("tsd.query.timeout", "0"); + default_map.put("tsd.storage.use_otsdb_timestamp", "false"); + default_map.put("tsd.storage.use_max_value", "true"); + default_map.put("tsd.storage.get_date_tiered_compaction_start", "0"); for (Map.Entry<String, String> entry : default_map.entrySet()) { if (!properties.containsKey(entry.getKey())) @@ -481,14 +669,14 @@ protected void setDefaults() { /** * Searches a list of locations for a valid opentsdb.conf file - * + * * The config file must be a standard JAVA properties formatted file. If none * of the locations have a config file, then the defaults or command line * arguments will be used for the configuration - * + * * Defaults for Linux based systems are: ./opentsdb.conf /etc/opentsdb.conf * /etc/opentsdb/opentdsb.conf /opt/opentsdb/opentsdb.conf - * + * * @throws IOException Thrown if there was an issue reading a file */ protected void loadConfig() throws IOException { @@ -517,9 +705,9 @@ protected void loadConfig() throws IOException { FileInputStream file_stream = new FileInputStream(file); Properties props = new Properties(); props.load(file_stream); - + // load the hash map - loadHashMap(props); + loadHashMap(props); } catch (Exception e) { // don't do anything, the file may be missing and that's fine LOG.debug("Unable to find or load " + file, e); @@ -547,10 +735,10 @@ protected void loadConfig(final String file) throws FileNotFoundException, try { final Properties props = new Properties(); props.load(file_stream); - + // load the hash map loadHashMap(props); - + // no exceptions thrown, so save the valid path and exit LOG.info("Successfully loaded configuration file: " + file); config_location = file; @@ -563,35 +751,44 @@ protected void loadConfig(final String file) throws FileNotFoundException, * Loads the static class variables for values that are called often. This * should be called any time the configuration changes. */ - protected void loadStaticVariables() { + public void loadStaticVariables() { auto_metric = this.getBoolean("tsd.core.auto_create_metrics"); auto_tagk = this.getBoolean("tsd.core.auto_create_tagks"); auto_tagv = this.getBoolean("tsd.core.auto_create_tagvs"); enable_compactions = this.getBoolean("tsd.storage.enable_compaction"); + enable_appends = this.getBoolean("tsd.storage.enable_appends"); + repair_appends = this.getBoolean("tsd.storage.repair_appends"); enable_chunked_requests = this.getBoolean("tsd.http.request.enable_chunked"); enable_realtime_ts = this.getBoolean("tsd.core.meta.enable_realtime_ts"); enable_realtime_uid = this.getBoolean("tsd.core.meta.enable_realtime_uid"); - enable_tsuid_incrementing = + enable_tsuid_incrementing = this.getBoolean("tsd.core.meta.enable_tsuid_incrementing"); - enable_tsuid_tracking = + enable_tsuid_tracking = this.getBoolean("tsd.core.meta.enable_tsuid_tracking"); if (this.hasProperty("tsd.http.request.max_chunk")) { max_chunked_requests = this.getInt("tsd.http.request.max_chunk"); } + if (this.hasProperty("tsd.http.header_tag")) { + http_header_tag = this.getString("tsd.http.header_tag"); + } enable_tree_processing = this.getBoolean("tsd.core.tree.enable_processing"); fix_duplicates = this.getBoolean("tsd.storage.fix_duplicates"); + scanner_max_num_rows = this.getInt("tsd.storage.hbase.scanner.maxNumRows"); + use_otsdb_timestamp = this.getBoolean("tsd.storage.use_otsdb_timestamp"); + get_date_tiered_compaction_start = this.getLong("tsd.storage.get_date_tiered_compaction_start"); + use_max_value = this.getBoolean("tsd.storage.use_max_value"); } - + /** * Called from {@link #loadConfig} to copy the properties into the hash map * Tsuna points out that the Properties class is much slower than a hash * map so if we'll be looking up config values more than once, a hash map - * is the way to go + * is the way to go * @param props The loaded Properties object to copy */ private void loadHashMap(final Properties props) { properties.clear(); - + @SuppressWarnings("rawtypes") Enumeration e = props.propertyNames(); while (e.hasMoreElements()) { diff --git a/src/utils/DateTime.java b/src/utils/DateTime.java index dca0de671b..8989ae85f5 100644 --- a/src/utils/DateTime.java +++ b/src/utils/DateTime.java @@ -14,9 +14,11 @@ import java.text.ParseException; import java.text.SimpleDateFormat; +import java.util.Calendar; import java.util.HashMap; import java.util.TimeZone; +import com.google.common.base.Strings; import net.opentsdb.core.Tags; /** @@ -26,7 +28,9 @@ * @since 2.0 */ public class DateTime { - + /** ID of the UTC timezone */ + public static final String UTC_ID = "UTC"; + /** * Immutable cache mapping a timezone name to its object. * We do this because the JDK's TimeZone class was implemented by retards, @@ -64,6 +68,7 @@ public class DateTime { * <li>1355961600.000</li></ul></li> * </ul> * @param datetime The string to parse a value for + * @param tz The timezone to use for parsing. * @return A Unix epoch timestamp in milliseconds * @throws NullPointerException if the timestamp is null * @throws IllegalArgumentException if the request was malformed @@ -72,6 +77,15 @@ public static final long parseDateTimeString(final String datetime, final String tz) { if (datetime == null || datetime.isEmpty()) return -1; + + if (datetime.matches("^[0-9]+ms$")) { + return Tags.parseLong(datetime.replaceFirst("^([0-9]+)(ms)$", "$1")); + } + + if (datetime.toLowerCase().equals("now")) { + return System.currentTimeMillis(); + } + if (datetime.toLowerCase().endsWith("-ago")) { long interval = DateTime.parseDuration( datetime.substring(0, datetime.length() - 4)); @@ -121,8 +135,14 @@ public static final long parseDateTimeString(final String datetime, } else { try { long time; - if (datetime.contains(".")) { - if (datetime.charAt(10) != '.' || datetime.length() != 14) { + final boolean contains_dot = datetime.contains("."); + // [0-9]{10} ten digits + // \\. a dot + // [0-9]{1,3} one to three digits + final boolean valid_dotted_ms = + datetime.matches("^[0-9]{10}\\.[0-9]{1,3}$"); + if (contains_dot) { + if (!valid_dotted_ms) { throw new IllegalArgumentException("Invalid time: " + datetime + ". Millisecond timestamps must be in the format " + "<seconds>.<ms> where the milliseconds are limited to 3 digits"); @@ -137,8 +157,9 @@ public static final long parseDateTimeString(final String datetime, } // this is a nasty hack to determine if the incoming request is // in seconds or milliseconds. This will work until November 2286 - if (datetime.length() <= 10) + if (datetime.length() <= 10) { time *= 1000; + } return time; } catch (NumberFormatException e) { throw new IllegalArgumentException("Invalid time: " + datetime @@ -164,12 +185,20 @@ public static final long parseDateTimeString(final String datetime, * @throws IllegalArgumentException if the interval was malformed. */ public static final long parseDuration(final String duration) { + if (duration == null || duration.isEmpty()) { + throw new IllegalArgumentException("Cannot parse null or empty duration"); + } + long interval; long multiplier; double temp; int unit = 0; while (Character.isDigit(duration.charAt(unit))) { unit++; + if (unit >= duration.length()) { + throw new IllegalArgumentException("Invalid duration, must have an " + + "integer and unit: " + duration); + } } try { interval = Long.parseLong(duration.substring(0, unit)); @@ -201,6 +230,66 @@ public static final long parseDuration(final String duration) { return interval * multiplier; } + /** + * Returns the suffix or "units" of the duration as a string. The result will + * be ms, s, m, h, d, w, n or y. + * @param duration The duration in the format #units, e.g. 1d or 6h + * @return Just the suffix, e.g. 'd' or 'h' + * @throws IllegalArgumentException if the duration is null, empty or if + * the units are invalid. + * @since 2.4 + */ + public static final String getDurationUnits(final String duration) { + if (duration == null || duration.isEmpty()) { + throw new IllegalArgumentException("Duration cannot be null or empty"); + } + int unit = 0; + while (unit < duration.length() && + Character.isDigit(duration.charAt(unit))) { + unit++; + } + final String units = duration.substring(unit).toLowerCase(); + if (units.equals("ms") || units.equals("s") || units.equals("m") || + units.equals("h") || units.equals("d") || units.equals("w") || + units.equals("n") || units.equals("y")) { + return units; + } + throw new IllegalArgumentException("Invalid units in the duration: " + units); + } + + /** + * Parses the prefix of the duration, the interval and returns it as a number. + * E.g. if you supply "1d" it will return "1". If you supply "60m" it will + * return "60". + * @param duration The duration to parse in the format #units, e.g. "1d" or "60m" + * @return The interval as an integer, regardless of units. + * @throws IllegalArgumentException if the duration is null, empty or parsing + * of the integer failed. + * @since 2.4 + */ + public static final int getDurationInterval(final String duration) { + if (duration == null || duration.isEmpty()) { + throw new IllegalArgumentException("Duration cannot be null or empty"); + } + if (duration.contains(".")) { + throw new IllegalArgumentException("Floating point intervals are not supported"); + } + int unit = 0; + while (Character.isDigit(duration.charAt(unit))) { + unit++; + } + int interval; + try { + interval = Integer.parseInt(duration.substring(0, unit)); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid duration (number): " + duration); + } + if (interval <= 0) { + throw new IllegalArgumentException("Zero or negative duration: " + duration); + } + return interval; + } + /** * Returns whether or not a date is specified in a relative fashion. * <p> @@ -256,4 +345,303 @@ public static void setDefaultTimezone(final String tzname) { throw new IllegalArgumentException("Invalid timezone name: " + tzname); } } + + /** + * Pass through to {@link System#currentTimeMillis()} for use in classes to + * make unit testing easier. Mocking System.class is a bad idea in general + * so placing this here and mocking DateTime.class is MUCH cleaner. + * @return The current epoch time in milliseconds + * @since 2.1 + */ + public static long currentTimeMillis() { + return System.currentTimeMillis(); + } + + /** + * Pass through to {@link System#nanoTime()} for use in classes to + * make unit testing easier. Mocking System.class is a bad idea in general + * so placing this here and mocking DateTime.class is MUCH cleaner. + * @return The current epoch time in milliseconds + * @since 2.2 + */ + public static long nanoTime() { + return System.nanoTime(); + } + + /** + * Converts the long nanosecond value to a double in milliseconds + * @param ts The timestamp or value in nanoseconds + * @return The timestamp in milliseconds + * @since 2.2 + */ + public static double msFromNano(final long ts) { + return (double)ts / 1000000; + } + + /** + * Calculates the difference between two values and returns the time in + * milliseconds as a double. + * @param end The end timestamp + * @param start The start timestamp + * @return The value in milliseconds + * @throws IllegalArgumentException if end is less than start + * @since 2.2 + */ + public static double msFromNanoDiff(final long end, final long start) { + if (end < start) { + throw new IllegalArgumentException("End (" + end + ") cannot be less " + + "than start (" + start + ")"); + } + return ((double) end - (double) start) / 1000000; + } + + /** + * Returns a calendar set to the previous interval time based on the + * units and UTC the timezone. This allows for snapping to day, week, + * monthly, etc. boundaries. + * NOTE: It uses a calendar for snapping so isn't as efficient as a simple + * modulo calculation. + * NOTE: For intervals that don't nicely divide into their given unit (e.g. + * a 23s interval where 60 seconds is not divisible by 23) the base time may + * start at the top of the day (for ms and s) or from Unix epoch 0. In the + * latter case, setting up the base timestamp may be slow if the caller does + * something silly like "23m" where we iterate 23 minutes at a time from 0 + * till we find the proper timestamp. + * TODO - There is likely a better way to do all of this + * @param ts The timestamp to find an interval for, in milliseconds as + * a Unix epoch. + * @param interval The interval as a measure of units. + * @param unit The unit. This must cast to a Calendar time unit. + * @return A calendar set to the timestamp aligned to the proper interval + * before the given ts + * @throws IllegalArgumentException if the timestamp is negative, if the + * interval is less than 1 or the unit is unrecognized. + * @since 2.3 + */ + public static Calendar previousInterval(final long ts, final int interval, + final int unit) { + return previousInterval(ts, interval, unit, null); + } + + /** + * Returns a calendar set to the previous interval time based on the + * units and timezone. This allows for snapping to day, week, monthly, etc. + * boundaries. + * NOTE: It uses a calendar for snapping so isn't as efficient as a simple + * modulo calculation. + * NOTE: For intervals that don't nicely divide into their given unit (e.g. + * a 23s interval where 60 seconds is not divisible by 23) the base time may + * start at the top of the day (for ms and s) or from Unix epoch 0. In the + * latter case, setting up the base timestamp may be slow if the caller does + * something silly like "23m" where we iterate 23 minutes at a time from 0 + * till we find the proper timestamp. + * TODO - There is likely a better way to do all of this + * @param ts The timestamp to find an interval for, in milliseconds as + * a Unix epoch. + * @param interval The interval as a measure of units. + * @param unit The unit. This must cast to a Calendar time unit. + * @param tz An optional timezone. + * @return A calendar set to the timestamp aligned to the proper interval + * before the given ts + * @throws IllegalArgumentException if the timestamp is negative, if the + * interval is less than 1 or the unit is unrecognized. + * @since 2.3 + */ + public static Calendar previousInterval(final long ts, final int interval, + final int unit, final TimeZone tz) { + if (ts < 0) { + throw new IllegalArgumentException("Timestamp cannot be less than zero"); + } + if (interval < 1) { + throw new IllegalArgumentException("Interval must be greater than zero"); + } + + int unit_override = unit; + int interval_override = interval; + final Calendar calendar; + if (tz == null) { + calendar = Calendar.getInstance(timezones.get(UTC_ID)); + } else { + calendar = Calendar.getInstance(tz); + } + + switch (unit_override) { + case Calendar.MILLISECOND: + if (1000 % interval_override == 0) { + calendar.setTimeInMillis(ts); + calendar.set(Calendar.MILLISECOND, 0); + if (interval_override > 1000) { + calendar.add(Calendar.MILLISECOND, -interval_override); + } + } else { + // from top of minute + calendar.setTimeInMillis(ts); + calendar.set(Calendar.MILLISECOND, 0); + calendar.set(Calendar.SECOND, 0); + } + break; + case Calendar.SECOND: + if (60 % interval_override == 0) { + calendar.setTimeInMillis(ts); + calendar.set(Calendar.MILLISECOND, 0); + calendar.set(Calendar.SECOND, 0); + if (interval_override > 60) { + calendar.add(Calendar.SECOND, -interval_override); + } + } else { + // from top of hour + calendar.setTimeInMillis(ts); + calendar.set(Calendar.MILLISECOND, 0); + calendar.set(Calendar.SECOND, 0); + calendar.set(Calendar.MINUTE, 0); + } + break; + case Calendar.MINUTE: + if (60 % interval_override == 0) { + calendar.setTimeInMillis(ts); + calendar.set(Calendar.MILLISECOND, 0); + calendar.set(Calendar.SECOND, 0); + calendar.set(Calendar.MINUTE, 0); + if (interval_override > 60) { + calendar.add(Calendar.MINUTE, -interval_override); + } + } else { + // from top of day + calendar.setTimeInMillis(ts); + calendar.set(Calendar.MILLISECOND, 0); + calendar.set(Calendar.SECOND, 0); + calendar.set(Calendar.MINUTE, 0); + calendar.set(Calendar.HOUR_OF_DAY, 0); + } + break; + case Calendar.HOUR_OF_DAY: + if (24 % interval_override == 0) { + calendar.setTimeInMillis(ts); + calendar.set(Calendar.MILLISECOND, 0); + calendar.set(Calendar.SECOND, 0); + calendar.set(Calendar.MINUTE, 0); + calendar.set(Calendar.HOUR_OF_DAY, 0); + if (interval_override > 24) { + calendar.add(Calendar.HOUR_OF_DAY, -interval_override); + } + } else { + // from top of month + calendar.setTimeInMillis(ts); + calendar.set(Calendar.MILLISECOND, 0); + calendar.set(Calendar.SECOND, 0); + calendar.set(Calendar.MINUTE, 0); + calendar.set(Calendar.HOUR_OF_DAY, 0); + calendar.set(Calendar.DAY_OF_MONTH, 1); + } + break; + case Calendar.DAY_OF_MONTH: + if (interval_override == 1) { + calendar.setTimeInMillis(ts); + calendar.set(Calendar.MILLISECOND, 0); + calendar.set(Calendar.SECOND, 0); + calendar.set(Calendar.MINUTE, 0); + calendar.set(Calendar.HOUR_OF_DAY, 0); + calendar.set(Calendar.DAY_OF_MONTH, 1); + } else { + // from top of year + calendar.setTimeInMillis(ts); + calendar.set(Calendar.MILLISECOND, 0); + calendar.set(Calendar.SECOND, 0); + calendar.set(Calendar.MINUTE, 0); + calendar.set(Calendar.HOUR_OF_DAY, 0); + calendar.set(Calendar.DAY_OF_MONTH, 1); + calendar.set(Calendar.MONTH, 0); + } + break; + case Calendar.DAY_OF_WEEK: + if (2 % interval_override == 0) { + calendar.setTimeInMillis(ts); + calendar.set(Calendar.MILLISECOND, 0); + calendar.set(Calendar.SECOND, 0); + calendar.set(Calendar.MINUTE, 0); + calendar.set(Calendar.HOUR_OF_DAY, 0); + calendar.set(Calendar.DAY_OF_WEEK, calendar.getFirstDayOfWeek()); + } else { + // from top of year + calendar.setTimeInMillis(ts); + calendar.set(Calendar.MILLISECOND, 0); + calendar.set(Calendar.SECOND, 0); + calendar.set(Calendar.MINUTE, 0); + calendar.set(Calendar.HOUR_OF_DAY, 0); + calendar.set(Calendar.MONTH, 0); + calendar.set(Calendar.DAY_OF_WEEK, calendar.getFirstDayOfWeek()); + } + unit_override = Calendar.DAY_OF_MONTH; + interval_override = 7; + break; + case Calendar.WEEK_OF_YEAR: + // from top of year + calendar.setTimeInMillis(ts); + calendar.set(Calendar.MILLISECOND, 0); + calendar.set(Calendar.SECOND, 0); + calendar.set(Calendar.MINUTE, 0); + calendar.set(Calendar.HOUR_OF_DAY, 0); + calendar.set(Calendar.DAY_OF_MONTH, 1); + calendar.set(Calendar.MONTH, 0); + break; + case Calendar.MONTH: + case Calendar.YEAR: + calendar.setTimeInMillis(ts); + calendar.set(Calendar.MILLISECOND, 0); + calendar.set(Calendar.SECOND, 0); + calendar.set(Calendar.MINUTE, 0); + calendar.set(Calendar.HOUR_OF_DAY, 0); + calendar.set(Calendar.DAY_OF_MONTH, 1); + calendar.set(Calendar.MONTH, 0); + break; + default: + throw new IllegalArgumentException("Unexpected unit_overrides of type: " + + unit_override); + } + + if (calendar.getTimeInMillis() == ts) { + return calendar; + } + // TODO optimize a bit. We probably don't need to go past then back. + while (calendar.getTimeInMillis() <= ts) { + calendar.add(unit_override, interval_override); + } + calendar.add(unit_override, -interval_override); + return calendar; + } + + /** + * Return the proper Calendar time unit as an integer given the string + * @param units The unit to parse + * @return An integer matching a Calendar.<UNIT> enum + * @throws IllegalArgumentException if the unit is null, empty or doesn't + * match one of the configured units. + * @since 2.3 + */ + public static int unitsToCalendarType(final String units) { + if (Strings.isNullOrEmpty(units)) { + throw new IllegalArgumentException("Units cannot be null or empty"); + } + + final String lc = units.toLowerCase(); + if (lc.equals("ms")) { + return Calendar.MILLISECOND; + } else if (lc.equals("s")) { + return Calendar.SECOND; + } else if (lc.equals("m")) { + return Calendar.MINUTE; + } else if (lc.equals("h")) { + return Calendar.HOUR_OF_DAY; + } else if (lc.equals("d")) { + return Calendar.DAY_OF_MONTH; + } else if (lc.equals("w")) { + return Calendar.DAY_OF_WEEK; + } else if (lc.equals("n")) { + return Calendar.MONTH; + } else if (lc.equals("y")) { + return Calendar.YEAR; + } + throw new IllegalArgumentException("Unrecognized unit type: " + units); + } + } diff --git a/src/utils/Exceptions.java b/src/utils/Exceptions.java new file mode 100644 index 0000000000..4f52111329 --- /dev/null +++ b/src/utils/Exceptions.java @@ -0,0 +1,41 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.utils; + +import com.stumbleupon.async.DeferredGroupException; + +/** + * A class with utility methods for dealing with Exceptions in OpenTSDB + * @since 2.2 + */ +public class Exceptions { + + /** + * Iterates through the stack trace, looking for the actual cause of the + * deferred group exception. These traces can be huge and truncated in the + * logs so it's really useful to be able to spit out the source. + * @param e A DeferredGroupException to parse + * @return The root cause of the exception if found. + */ + public static Throwable getCause(final DeferredGroupException e) { + Throwable ex = e; + while (ex.getClass().equals(DeferredGroupException.class)) { + if (ex.getCause() == null) { + break; + } else { + ex = ex.getCause(); + } + } + return ex; + } +} diff --git a/src/utils/JSON.java b/src/utils/JSON.java index dd72c12125..eef88b4f20 100644 --- a/src/utils/JSON.java +++ b/src/utils/JSON.java @@ -269,7 +269,6 @@ public static final JsonParser parseToStream(final InputStream json) { * @return A JSON formatted string * @throws IllegalArgumentException if the object was null * @throws JSONException if the object could not be serialized - * @throws IOException Thrown when there was an issue reading the object */ public static final String serializeToString(final Object object) { if (object == null) @@ -287,7 +286,6 @@ public static final String serializeToString(final Object object) { * @return A JSON formatted byte array * @throws IllegalArgumentException if the object was null * @throws JSONException if the object could not be serialized - * @throws IOException Thrown when there was an issue reading the object */ public static final byte[] serializeToBytes(final Object object) { if (object == null) @@ -301,7 +299,7 @@ public static final byte[] serializeToBytes(final Object object) { /** * Serializes the given object and wraps it in a callback function - * i.e. <callback>(<json>) + * i.e. <callback>(<json>) * Note: This will not append a trailing semicolon * @param callback The name of the Javascript callback to prepend * @param object The object to serialize @@ -309,7 +307,6 @@ public static final byte[] serializeToBytes(final Object object) { * @throws IllegalArgumentException if the callback method name was missing * or object was null * @throws JSONException if the object could not be serialized - * @throws IOException Thrown when there was an issue reading the object */ public static final String serializeToJSONPString(final String callback, final Object object) { @@ -326,7 +323,7 @@ public static final String serializeToJSONPString(final String callback, /** * Serializes the given object and wraps it in a callback function - * i.e. <callback>(<json>) + * i.e. <callback>(<json>) * Note: This will not append a trailing semicolon * @param callback The name of the Javascript callback to prepend * @param object The object to serialize @@ -334,7 +331,6 @@ public static final String serializeToJSONPString(final String callback, * @throws IllegalArgumentException if the callback method name was missing * or object was null * @throws JSONException if the object could not be serialized - * @throws IOException Thrown when there was an issue reading the object */ public static final byte[] serializeToJSONPBytes(final String callback, final Object object) { diff --git a/src/utils/PluginLoader.java b/src/utils/PluginLoader.java index 8dadfc5ec3..2d0b8b3bc5 100644 --- a/src/utils/PluginLoader.java +++ b/src/utils/PluginLoader.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.ServiceConfigurationError; import java.util.ServiceLoader; import org.slf4j.Logger; @@ -88,7 +89,7 @@ public final class PluginLoader { * @return An instantiated object of the given type if found, null if the * class could not be found * @throws ServiceConfigurationError if the plugin cannot be instantiated - * @throws IllegalArgumentName if the plugin name is null or empty + * @throws IllegalArgumentException if the plugin name is null or empty */ public static <T> T loadSpecificPlugin(final String name, final Class<T> type) { @@ -104,7 +105,7 @@ public static <T> T loadSpecificPlugin(final String name, while(it.hasNext()) { T plugin = it.next(); - if (plugin.getClass().getName().equals(name)) { + if (plugin.getClass().getName().equals(name) || plugin.getClass().getSuperclass().getName().equals(name)) { return plugin; } } diff --git a/src/utils/Threads.java b/src/utils/Threads.java new file mode 100644 index 0000000000..93a0e02547 --- /dev/null +++ b/src/utils/Threads.java @@ -0,0 +1,99 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.utils; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; + +import org.jboss.netty.util.HashedWheelTimer; +import org.jboss.netty.util.ThreadNameDeterminer; + +/** + * Utilities dealing with threads, timers and the like. + */ +public class Threads { + /** Used to count HashedWheelTimers */ + final static AtomicInteger TIMER_ID = new AtomicInteger(); + + /** Helps give useful names to the Netty threads */ + public static class BossThreadNamer implements ThreadNameDeterminer { + final static AtomicInteger tid = new AtomicInteger(); + @Override + public String determineThreadName(String currentThreadName, + String proposedThreadName) throws Exception { + return "OpenTSDB I/O Boss #" + tid.incrementAndGet(); + } + } + + /** Helps give useful names to the Netty threads */ + public static class WorkerThreadNamer implements ThreadNameDeterminer { + final static AtomicInteger tid = new AtomicInteger(); + @Override + public String determineThreadName(String currentThreadName, + String proposedThreadName) throws Exception { + return "OpenTSDB I/O Worker #" + tid.incrementAndGet(); + } + } + + /** Simple prepends "OpenTSDB" to all threads */ + public static class PrependThreadNamer implements ThreadNameDeterminer { + @Override + public String determineThreadName(String currentThreadName, String proposedThreadName) + throws Exception { + return "OpenTSDB " + proposedThreadName; + } + } + + /** + * Returns a new HashedWheelTimer with a name and default ticks + * @param name The name to add to the thread name + * @return A timer + */ + public static HashedWheelTimer newTimer(final String name) { + return newTimer(100, name); + } + + /** + * Returns a new HashedWheelTimer with a name and default ticks + * @param ticks How many ticks per second to sleep between executions, in ms + * @param name The name to add to the thread name + * @return A timer + */ + public static HashedWheelTimer newTimer(final int ticks, final String name) { + return newTimer(ticks, 512, name); + } + + /** + * Returns a new HashedWheelTimer with a name and default ticks + * @param ticks How many ticks per second to sleep between executions, in ms + * @param ticks_per_wheel The size of the wheel + * @param name The name to add to the thread name + * @return A timer + */ + public static HashedWheelTimer newTimer(final int ticks, + final int ticks_per_wheel, final String name) { + class TimerThreadNamer implements ThreadNameDeterminer { + @Override + public String determineThreadName(String currentThreadName, + String proposedThreadName) throws Exception { + return "OpenTSDB Timer " + name + " #" + TIMER_ID.incrementAndGet(); + } + } + return new HashedWheelTimer(Executors.defaultThreadFactory(), + new TimerThreadNamer(), ticks, MILLISECONDS, ticks_per_wheel); + } + + +} diff --git a/test/META-INF/services/net.opentsdb.tsd.HttpRpcPlugin b/test/META-INF/services/net.opentsdb.tsd.HttpRpcPlugin new file mode 100644 index 0000000000..a673a90a34 --- /dev/null +++ b/test/META-INF/services/net.opentsdb.tsd.HttpRpcPlugin @@ -0,0 +1 @@ +net.opentsdb.tsd.DummyHttpRpcPlugin diff --git a/test/META-INF/services/net.opentsdb.tsd.StorageExceptionHandler b/test/META-INF/services/net.opentsdb.tsd.StorageExceptionHandler new file mode 100644 index 0000000000..64e9aa9e46 --- /dev/null +++ b/test/META-INF/services/net.opentsdb.tsd.StorageExceptionHandler @@ -0,0 +1 @@ +net.opentsdb.tsd.DummySEHPlugin diff --git a/test/auth/AllowAllAuthenticatingAuthorizerTest.java b/test/auth/AllowAllAuthenticatingAuthorizerTest.java new file mode 100644 index 0000000000..64e2ce213b --- /dev/null +++ b/test/auth/AllowAllAuthenticatingAuthorizerTest.java @@ -0,0 +1,107 @@ +package net.opentsdb.auth; + +import net.opentsdb.query.pojo.Query; +import net.opentsdb.core.TSDB; +import net.opentsdb.core.TSQuery; +import net.opentsdb.utils.Config; +import org.jboss.netty.channel.Channel; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import static org.junit.Assert.*; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.when; +import static org.powermock.api.mockito.PowerMockito.mock; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({TSDB.class, TSQuery.class}) +public class AllowAllAuthenticatingAuthorizerTest { + private TSDB tsdb = null; + private AllowAllAuthenticatingAuthorizer authenticatingAuthorizer; + + private Channel getMockedChannel() { + Channel channel = mock(Channel.class); + when(channel.getAttachment()).thenReturn(AllowAllAuthenticatingAuthorizer.accessGranted); + return channel; + } + + @Before + public void setUp() throws Exception { + this.authenticatingAuthorizer = new AllowAllAuthenticatingAuthorizer(); + this.tsdb = mock(TSDB.class); + + final Config config = new Config(false); + config.overrideConfig("tsd.core.authentication.enable", "true"); + when(tsdb.getConfig()).thenReturn(config); + + final AuthState state = mock(AuthState.class); + when(state.getStatus()).thenReturn(AllowAllAuthenticatingAuthorizer.accessGranted.getStatus()); + + final Authorization authorization = mock(Authorization.class); + when(authorization.allowQuery(any(AuthState.class), any(TSQuery.class))).thenReturn(state); + + final Authentication authentication = mock(Authentication.class); + when(authentication.authorization()).thenReturn(authorization); + when(authentication.isReady(any(TSDB.class), any(Channel.class))).thenReturn(true); + when(tsdb.getAuth()).thenReturn(authentication); + } + + @Test + public void isReady() throws Exception { + Channel channel = getMockedChannel(); + assertTrue(authenticatingAuthorizer.isReady(tsdb, channel)); + } + + @Test + public void hasPermissionAdministrator() throws Exception { + AuthState authState = mock(AuthState.class); + this.authenticatingAuthorizer.setRoles(new Roles(Roles.ADMINISTRATOR)); + assertEquals(AllowAllAuthenticatingAuthorizer.accessGranted, this.authenticatingAuthorizer.hasPermission(authState, Permissions.TELNET_PUT)); + } + + @Test + public void hasPermissionGuest() throws Exception { + AuthState authState = mock(AuthState.class); + this.authenticatingAuthorizer.setRoles(new Roles(Roles.GUEST)); + assertEquals(AllowAllAuthenticatingAuthorizer.accessDenied, this.authenticatingAuthorizer.hasPermission(authState, Permissions.TELNET_PUT)); + } + + @Test + public void allowTSQueryAdministrator() throws Exception { + AuthState authState = mock(AuthState.class); + TSQuery tsQuery = mock(TSQuery.class); + Roles roles = new Roles(Roles.ADMINISTRATOR); + this.authenticatingAuthorizer.setRoles(roles); + assertEquals(AllowAllAuthenticatingAuthorizer.accessGranted, this.authenticatingAuthorizer.allowQuery(authState, tsQuery)); + } + + @Test + public void allowTSQueryGuest() throws Exception { + AuthState authState = mock(AuthState.class); + TSQuery tsQuery = mock(TSQuery.class); + Roles roles = new Roles(Roles.GUEST); + this.authenticatingAuthorizer.setRoles(roles); + assertEquals(AllowAllAuthenticatingAuthorizer.accessDenied, this.authenticatingAuthorizer.allowQuery(authState, tsQuery)); + } + + @Test + public void allowQueryAdministrator() throws Exception { + AuthState authState = mock(AuthState.class); + Query query = mock(Query.class); + Roles roles = new Roles(Roles.ADMINISTRATOR); + this.authenticatingAuthorizer.setRoles(roles); + assertEquals(AllowAllAuthenticatingAuthorizer.accessGranted, this.authenticatingAuthorizer.allowQuery(authState, query)); + } + + @Test + public void allowQueryGuest() throws Exception { + AuthState authState = mock(AuthState.class); + Query query = mock(Query.class); + Roles roles = new Roles(Roles.GUEST); + this.authenticatingAuthorizer.setRoles(roles); + assertEquals(AllowAllAuthenticatingAuthorizer.accessDenied, this.authenticatingAuthorizer.allowQuery(authState, query)); + } +} \ No newline at end of file diff --git a/test/core/BaseTsdbTest.java b/test/core/BaseTsdbTest.java new file mode 100644 index 0000000000..bafe0eae95 --- /dev/null +++ b/test/core/BaseTsdbTest.java @@ -0,0 +1,1006 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015-2016 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.*; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.when; +import static org.powermock.api.mockito.PowerMockito.mock; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import net.opentsdb.auth.AuthState; +import net.opentsdb.auth.Authentication; +import net.opentsdb.auth.Authorization; +import net.opentsdb.meta.Annotation; +import net.opentsdb.rollup.RollupInterval; +import net.opentsdb.rollup.RollupQuery; +import net.opentsdb.storage.MockBase; +import net.opentsdb.uid.NoSuchUniqueId; +import net.opentsdb.uid.NoSuchUniqueName; +import net.opentsdb.uid.UniqueId; +import net.opentsdb.uid.UniqueId.UniqueIdType; +import net.opentsdb.utils.Config; +import net.opentsdb.utils.Threads; + +import org.hbase.async.Bytes; +import org.hbase.async.HBaseClient; +import org.hbase.async.Scanner; +import org.jboss.netty.channel.Channel; +import org.jboss.netty.util.HashedWheelTimer; +import org.jboss.netty.util.Timeout; +import org.jboss.netty.util.TimerTask; +import org.junit.Before; +import org.junit.runner.RunWith; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +import com.google.common.collect.Maps; +import com.stumbleupon.async.Deferred; + +/** + * Sets up a real TSDB with mocked client, compaction queue and timer along + * with mocked UID assignment, fetches for common unit tests. + */ +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({TSDB.class, Config.class, UniqueId.class, HBaseClient.class, + HashedWheelTimer.class, Scanner.class, Const.class, Threads.class }) +public class BaseTsdbTest { + + public static final String METRIC_STRING = "sys.cpu.user"; + public static final byte[] METRIC_BYTES = new byte[] { 0, 0, 1 }; + public static final String METRIC_B_STRING = "sys.cpu.system"; + public static final byte[] METRIC_B_BYTES = new byte[] { 0, 0, 2 }; + public static final String NSUN_METRIC = "sys.cpu.nice"; + public static final byte[] NSUI_METRIC = new byte[] { 0, 0, 3 }; + + public static final String TAGK_STRING = "host"; + public static final byte[] TAGK_BYTES = new byte[] { 0, 0, 1 }; + public static final String TAGK_B_STRING = "owner"; + public static final byte[] TAGK_B_BYTES = new byte[] { 0, 0, 3 }; + public static final String NSUN_TAGK = "dc"; + public static final byte[] NSUI_TAGK = new byte[] { 0, 0, 4 }; + + public static final String TAGV_STRING = "web01"; + public static final byte[] TAGV_BYTES = new byte[] { 0, 0, 1 }; + public static final String TAGV_B_STRING = "web02"; + public static final byte[] TAGV_B_BYTES = new byte[] { 0, 0, 2 }; + public static final String NSUN_TAGV = "web03"; + public static final byte[] NSUI_TAGV = new byte[] { 0, 0, 3 }; + + static final String NOTE_DESCRIPTION = "Hello DiscWorld!"; + static final String NOTE_NOTES = "Millenium hand and shrimp"; + + //histgoram metric + public static final String HISTOGRAM_METRIC_STRING = "msg.end2end.latency"; + public static final byte[] HISTOGRAM_METRIC_BYTES = new byte[] { 0, 0, 5 }; + + public static final Map<String, byte[]> UIDS = new HashMap<String, byte[]>(26); + static { + char letter = 'A'; + byte[] uid = new byte[] { 0, 0, 10 }; + for (int i = 0; i < 26; i++) { + UIDS.put(Character.toString(letter++), Arrays.copyOf(uid, uid.length)); + uid[2]++; + } + } + + protected FakeTaskTimer timer; + protected Config config; + protected TSDB tsdb; + protected HBaseClient client = mock(HBaseClient.class); + protected UniqueId metrics = mock(UniqueId.class); + protected UniqueId tag_names = mock(UniqueId.class); + protected UniqueId tag_values = mock(UniqueId.class); + protected Map<String, String> tags; + protected MockBase storage; + protected Map<String, byte[]> uid_map; + + @Before + public void before() throws Exception { + uid_map = Maps.newHashMap(); + PowerMockito.mockStatic(Threads.class); + timer = new FakeTaskTimer(); + PowerMockito.when(Threads.newTimer(anyString())).thenReturn(timer); + PowerMockito.when(Threads.newTimer(anyInt(), anyString())).thenReturn(timer); + + PowerMockito.whenNew(HashedWheelTimer.class).withNoArguments() + .thenReturn(timer); + PowerMockito.whenNew(HBaseClient.class).withAnyArguments() + .thenReturn(client); + + final AuthState state = mock(AuthState.class); + when(state.getStatus()).thenReturn(AuthState.AuthStatus.SUCCESS); + + final Authorization authorization = mock(Authorization.class); + when(authorization.allowQuery(any(AuthState.class), any(TSQuery.class))).thenReturn(state); + + final Authentication authentication = mock(Authentication.class); + when(authentication.authorization()).thenReturn(authorization); + when(authentication.isReady(any(TSDB.class), any(Channel.class))).thenReturn(true); + + config = new Config(false); + config.overrideConfig("tsd.storage.enable_compaction", "false"); + config.overrideConfig("tsd.core.authentication.enable", "false"); + tsdb = PowerMockito.spy(new TSDB(config)); + when(tsdb.getAuth()).thenReturn(authentication); + + config.setAutoMetric(true); + + Whitebox.setInternalState(tsdb, "metrics", metrics); + Whitebox.setInternalState(tsdb, "tag_names", tag_names); + Whitebox.setInternalState(tsdb, "tag_values", tag_values); + + setupMetricMaps(); + setupTagkMaps(); + setupTagvMaps(); + + mockUID(UniqueIdType.METRIC, HISTOGRAM_METRIC_STRING, HISTOGRAM_METRIC_BYTES); + + // add metrics and tags to the UIDs list for other functions to share + uid_map.put(METRIC_STRING, METRIC_BYTES); + uid_map.put(METRIC_B_STRING, METRIC_B_BYTES); + uid_map.put(NSUN_METRIC, NSUI_METRIC); + uid_map.put(HISTOGRAM_METRIC_STRING, HISTOGRAM_METRIC_BYTES); + + uid_map.put(TAGK_STRING, TAGK_BYTES); + uid_map.put(TAGK_B_STRING, TAGK_B_BYTES); + uid_map.put(NSUN_TAGK, NSUI_TAGK); + + uid_map.put(TAGV_STRING, TAGV_BYTES); + uid_map.put(TAGV_B_STRING, TAGV_B_BYTES); + uid_map.put(NSUN_TAGV, NSUI_TAGV); + + uid_map.putAll(UIDS); + + when(metrics.width()).thenReturn((short)3); + when(tag_names.width()).thenReturn((short)3); + when(tag_values.width()).thenReturn((short)3); + + tags = new HashMap<String, String>(1); + tags.put(TAGK_STRING, TAGV_STRING); + } + + /** Adds the static UIDs to the metrics UID mock object */ + public void setupMetricMaps() { + mockUID(UniqueIdType.METRIC, METRIC_STRING, METRIC_BYTES); + mockUID(UniqueIdType.METRIC, METRIC_B_STRING, METRIC_B_BYTES); + + final NoSuchUniqueName nsun = new NoSuchUniqueName(NSUN_METRIC, "metric"); + + when(metrics.getId(NSUN_METRIC)).thenThrow(nsun); + when(metrics.getIdAsync(NSUN_METRIC)) + .thenReturn(Deferred.<byte[]> fromError(nsun)); + when(metrics.getOrCreateId(NSUN_METRIC)).thenThrow(nsun); + final NoSuchUniqueName nsunic = new NoSuchUniqueName(NSUN_METRIC, "metric"); + when(metrics.getOrCreateIdAsync(eq(NSUN_METRIC))).thenThrow(nsunic); + when(metrics.getNameAsync(NSUI_METRIC)).thenReturn( + Deferred.<String>fromError(new NoSuchUniqueId("metrics", NSUI_METRIC))); + + for (final Map.Entry<String, byte[]> uid : UIDS.entrySet()) { + mockUID(UniqueIdType.METRIC, uid.getKey(), uid.getValue()); + } + } + + /** Adds the static UIDs to the tag keys UID mock object */ + public void setupTagkMaps() { + mockUID(UniqueIdType.TAGK, TAGK_STRING, TAGK_BYTES); + mockUID(UniqueIdType.TAGK, TAGK_B_STRING, TAGK_B_BYTES); + + final NoSuchUniqueName nsunic = new NoSuchUniqueName(NSUN_TAGK, "tagk"); + when(tag_names.getIdAsync(NSUN_TAGK)).thenReturn( + Deferred.<byte[]> fromError(nsunic)); + when(tag_names.getOrCreateId(eq(NSUN_TAGK))).thenThrow(nsunic); + when(tag_names.getOrCreateIdAsync(eq(NSUN_TAGK))).thenReturn( + Deferred.<byte[]> fromError(nsunic)); + when(tag_names.getName(NSUI_TAGK)) + .thenThrow(new NoSuchUniqueId("tagk", NSUI_TAGK)); + when(tag_names.getNameAsync(NSUI_TAGK)).thenReturn( + Deferred.<String>fromError(new NoSuchUniqueId("tagk", NSUI_TAGK))); + + for (final Map.Entry<String, byte[]> uid : UIDS.entrySet()) { + mockUID(UniqueIdType.TAGK, uid.getKey(), uid.getValue()); + } + } + + /** Adds the static UIDs to the tag values UID mock object */ + public void setupTagvMaps() { + mockUID(UniqueIdType.TAGV, TAGV_STRING, TAGV_BYTES); + mockUID(UniqueIdType.TAGV, TAGV_B_STRING, TAGV_B_BYTES); + + final NoSuchUniqueName nsun = new NoSuchUniqueName(NSUN_TAGV, "tagv"); + final NoSuchUniqueId nsui = new NoSuchUniqueId("tagv", NSUI_TAGV); + when(tag_values.getId(NSUN_TAGV)).thenThrow(nsun); + when(tag_values.getIdAsync(NSUN_TAGV)) + .thenReturn(Deferred.<byte[]> fromError(nsun)); + when(tag_values.getName(NSUI_TAGV)).thenThrow(nsui); + when(tag_values.getNameAsync(NSUI_TAGV)) + .thenReturn(Deferred.<String>fromError(nsui)); + final NoSuchUniqueName nsunic = new NoSuchUniqueName(NSUN_TAGV, "tagv"); + when(tag_values.getOrCreateId(eq(NSUN_TAGV))).thenThrow(nsunic); + when(tag_values.getOrCreateIdAsync(eq(NSUN_TAGV))).thenReturn( + Deferred.<byte[]> fromError(nsunic)); + + for (final Map.Entry<String, byte[]> uid : UIDS.entrySet()) { + mockUID(UniqueIdType.TAGV, uid.getKey(), uid.getValue()); + } + } + + /** + * Helper method that sets up UIDs for rollup and pre-agg testing. + */ + protected void setupGroupByTagValues() { + // set the aggregate tag and value + mockUID(UniqueIdType.TAGK, config.getString("tsd.rollups.agg_tag_key"), + new byte[] { 0, 0, 42 }); + uid_map.put(config.getString("tsd.rollups.agg_tag_key"), new byte[] { 0, 0, 42 }); + mockUID(UniqueIdType.TAGV, config.getString("tsd.rollups.raw_agg_tag_value"), + new byte[] { 0, 0, 42 }); + uid_map.put(config.getString("tsd.rollups.raw_agg_tag_value"), + new byte[] { 0, 0, 42 }); + mockUID(UniqueIdType.TAGV, "SUM", new byte[] { 0, 0, 43 }); + uid_map.put("SUM", new byte[] { 0, 0, 43 }); + mockUID(UniqueIdType.TAGV, "MAX", new byte[] { 0, 0, 44 }); + uid_map.put("MAX", new byte[] { 0, 0, 44 }); + mockUID(UniqueIdType.TAGV, "MIN", new byte[] { 0, 0, 45 }); + uid_map.put("MIN", new byte[] { 0, 0, 45 }); + mockUID(UniqueIdType.TAGV, "COUNT", new byte[] { 0, 0, 46 }); + uid_map.put("COUNT", new byte[] { 0, 0, 46 }); + mockUID(UniqueIdType.TAGV, "AVG", new byte[] { 0, 0, 47 }); + uid_map.put("AVG", new byte[] { 0, 0, 47 }); + mockUID(UniqueIdType.TAGV, "NOSUCHAGG", new byte[] { 0, 0, 0, 48 }); + uid_map.put("NOSUCHAGG", new byte[] { 0, 0, 48 }); + } + + // ----------------- // + // Helper functions. // + // ----------------- // + + /** + * Mocks out the UID calls to match keys and values + * + * @param type + * The type of UID to deal with + * @param key + * The String name of the UID + * @param uid + * The byte array UID to pair up with + */ + protected void mockUID(final UniqueIdType type, final String key, + final byte[] uid) { + switch (type) { + case METRIC: + when(metrics.getId(key)).thenReturn(uid); + when(metrics.getIdAsync(key)) + .thenAnswer(new Answer<Deferred<byte[]>>() { + @Override + public Deferred<byte[]> answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult(uid); + } + }); + when(metrics.getOrCreateId(key)).thenReturn(uid); + when(metrics.getOrCreateIdAsync(key)) + .thenAnswer(new Answer<Deferred<byte[]>>() { + @Override + public Deferred<byte[]> answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult(uid); + } + }); + when(metrics.getName(uid)).thenReturn(key); + when(metrics.getNameAsync(uid)).thenAnswer(new Answer<Deferred<String>>() { + @Override + public Deferred<String> answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult(key); + } + }); + break; + case TAGK: + when(tag_names.getId(key)).thenReturn(uid); + when(tag_names.getIdAsync(key)) + .thenAnswer(new Answer<Deferred<byte[]>>() { + @Override + public Deferred<byte[]> answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult(uid); + } + }); + when(tag_names.getOrCreateId(key)).thenReturn(uid); + when(tag_names.getOrCreateIdAsync(key)) + .thenAnswer(new Answer<Deferred<byte[]>>() { + @Override + public Deferred<byte[]> answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult(uid); + } + }); + when(tag_names.getName(uid)).thenReturn(key); + when(tag_names.getNameAsync(uid)).thenAnswer(new Answer<Deferred<String>>() { + @Override + public Deferred<String> answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult(key); + } + }); + break; + case TAGV: + when(tag_values.getId(key)).thenReturn(uid); + when(tag_values.getIdAsync(key)) + .thenAnswer(new Answer<Deferred<byte[]>>() { + @Override + public Deferred<byte[]> answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult(uid); + } + }); + when(tag_values.getOrCreateId(key)).thenReturn(uid); + when(tag_values.getOrCreateIdAsync(key)) + .thenAnswer(new Answer<Deferred<byte[]>>() { + @Override + public Deferred<byte[]> answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult(uid); + } + }); + when(tag_values.getName(uid)).thenReturn(key); + when(tag_values.getNameAsync(uid)).thenAnswer(new Answer<Deferred<String>>() { + @Override + public Deferred<String> answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult(key); + } + }); + break; + } + } + + /** @return a row key template with the default metric and tags */ + protected byte[] getRowKeyTemplate() { + return IncomingDataPoints.rowKeyTemplate(tsdb, METRIC_STRING, tags); + } + + /** + * Generates a proper key storage row key based on the metric, base time + * and tags. Adds salting when mocked properly. + * @param metric A non-null byte array representing the metric. + * @param base_time The base time for the row. + * @param tags A non-null list of tag key/value pairs as UIDs. + * @return A row key to check mock storage for. + */ + public static byte[] getRowKey(final byte[] metric, final int base_time, + final byte[]... tags) { + int tags_length = 0; + for (final byte[] tag : tags) { + tags_length += tag.length; + } + final byte[] key = new byte[Const.SALT_WIDTH() + metric.length + + Const.TIMESTAMP_BYTES + tags_length]; + + System.arraycopy(metric, 0, key, Const.SALT_WIDTH(), metric.length); + System.arraycopy(Bytes.fromInt(base_time), 0, key, + Const.SALT_WIDTH() + metric.length, Const.TIMESTAMP_BYTES); + int offset = Const.SALT_WIDTH() + metric.length + Const.TIMESTAMP_BYTES; + for (final byte[] tag : tags) { + System.arraycopy(tag, 0, key, offset, tag.length); + offset += tag.length; + } + RowKey.prefixKeyWithSalt(key); + return key; + } + + /** + * Generates a proper key storage row key based on the metric, base time + * and tags. Adds salting when mocked properly. + * @param metric + * @param base_time + * @param tags + * @return + */ + protected byte[] getRowKey(final String metric, final int base_time, + final String... tags) { + final int m = TSDB.metrics_width(); + final int tk = TSDB.tagk_width(); + final int tv = TSDB.tagv_width(); + + final byte[] key = new byte[Const.SALT_WIDTH() + m + 4 + + (tags.length / 2) * tk + (tags.length / 2) * tv]; + byte[] uid = uid_map.get(metric); + + // metrics first + if (uid != null) { + System.arraycopy(uid, 0, key, Const.SALT_WIDTH(), m); + } else { + throw new IllegalArgumentException("No METRIC UID was mocked for: " + metric); + } + + // timestamp + System.arraycopy(Bytes.fromInt(base_time), 0, key, Const.SALT_WIDTH() + m, + Const.TIMESTAMP_BYTES); + + // shortcut for offsets + final int pl = Const.SALT_WIDTH() + m + Const.TIMESTAMP_BYTES; + int ctr = 0; + int offset = 0; + for (final String tag : tags) { + uid = uid_map.get(tag); + + if (ctr % 2 == 0) { + // TAGK + if (uid != null) { + System.arraycopy(uid, 0, key, pl + offset, tk); + } else { + throw new IllegalArgumentException("No TAGK UID was mocked for: " + tag); + } + offset += tk; + } else { + // TAGV + if (uid != null) { + System.arraycopy(uid, 0, key, pl + offset, tv); + } else { + throw new IllegalArgumentException("No TAGK UID was mocked for: " + tag); + } + offset += tv; + } + + ctr++; + } + + RowKey.prefixKeyWithSalt(key); + return key; + } + + /** + * Generates a TSUID given the metric and tag UIDs. + * @param metric A metric UID. + * @param tags A set of UIDs + * @return A TSUID byte array + */ + public static byte[] getTSUID(final byte[] metric, final byte[]... tags) { + int tags_length = 0; + for (final byte[] tag : tags) { + tags_length += tag.length; + } + final byte[] tsuid = new byte[metric.length + tags_length]; + System.arraycopy(metric, 0, tsuid, 0, metric.length); + int offset = metric.length; + for (final byte[] tag : tags) { + System.arraycopy(tag, 0, tsuid, offset, tag.length); + offset += tag.length; + } + RowKey.prefixKeyWithSalt(tsuid); + return tsuid; + } + + /** + * Generates a UID of the proper length given a type and ID. + * @param type The type of UID. + * @param id The ID to set (just tweaks the last byte) + * @return A Unique ID of the proper width. + */ + public static byte[] generateUID(final UniqueIdType type, byte id) { + final byte[] uid; + switch (type) { + case METRIC: + uid = new byte[TSDB.metrics_width()]; + break; + case TAGK: + uid = new byte[TSDB.tagk_width()]; + break; + case TAGV: + uid = new byte[TSDB.tagv_width()]; + break; + default: + throw new IllegalArgumentException("Yo! You have to mock out " + type + "!"); + } + uid[uid.length - 1] = id; + return uid; + } + + /** + * Generates a UID of the proper length given a type and ID. + * @param type The type of UID. + * @param id The ID to set (just tweaks the last byte) + * @return A Unique ID of the proper width. + */ + public static String generateUIDString(final UniqueIdType type, byte id) { + return UniqueId.uidToString(generateUID(type, id)); + } + + /** + * Generates a TSUID given the metric and tag UIDs. + * @param metric A metric UID. + * @param tags A set of UIDs + * @return A TSUID as a hex string + */ + public static String getTSUIDString(final byte[] metric, final byte[]... tags) { + return UniqueId.uidToString(getTSUID(metric, tags)); + } + + /** + * Generates a TSUID given the mocked UID strings. + * @param metric A mocked metric name. + * @param tags A set of mocked tag key and values. + * @return A TSUID byte array + */ + protected byte[] getTSUID(final String metric, final String... tags) { + final int m = TSDB.metrics_width(); + final int tk = TSDB.tagk_width(); + final int tv = TSDB.tagv_width(); + + final byte[] tsuid = new byte[m + (tags.length / 2) * tk + (tags.length / 2) * tv]; + byte[] uid = uid_map.get(metric); + + // metrics first + if (uid != null) { + System.arraycopy(uid, 0, tsuid, 0, m); + } else { + throw new IllegalArgumentException("No METRIC UID was mocked for: " + metric); + } + + int ctr = 0; + int offset = 0; + for (final String tag : tags) { + uid = uid_map.get(tag); + + if (ctr % 2 == 0) { + // TAGK + if (uid != null) { + System.arraycopy(uid, 0, tsuid, m + offset, tk); + } else { + throw new IllegalArgumentException("No TAGK UID was mocked for: " + tag); + } + offset += tk; + } else { + // TAGV + if (uid != null) { + System.arraycopy(uid, 0, tsuid, m + offset, tv); + } else { + throw new IllegalArgumentException("No TAGK UID was mocked for: " + tag); + } + offset += tv; + } + + ctr++; + } + + return tsuid; + } + + /** + * Generates a TSUID given the mocked UID strings. + * @param metric A mocked metric name. + * @param tags A set of mocked tag key and values. + * @return A TSUID hex string. + */ + protected String getTSUIDString(final String metric, final String... tags) { + return UniqueId.uidToString(getTSUID(metric, tags)); + } + + protected void setDataPointStorage() throws Exception { + storage = new MockBase(tsdb, client, true, true, true, true); + storage.setFamily("t".getBytes(MockBase.ASCII())); + } + + protected void storeLongTimeSeriesSeconds(final boolean two_metrics, + final boolean offset) throws Exception { + setDataPointStorage(); + + // dump a bunch of rows of two metrics so that we can test filtering out + // on the metric + HashMap<String, String> tags_local = new HashMap<String, String>(tags); + long timestamp = 1356998400; + for (int i = 1; i <= 300; i++) { + tsdb.addPoint(METRIC_STRING, timestamp += 30, i, tags_local) + .joinUninterruptibly(); + if (two_metrics) { + tsdb.addPoint(METRIC_B_STRING, timestamp, i, tags_local) + .joinUninterruptibly(); + } + } + + // dump a parallel set but invert the values + tags_local.clear(); + tags_local.put(TAGK_STRING, TAGV_B_STRING); + timestamp = offset ? 1356998415 : 1356998400; + for (int i = 300; i > 0; i--) { + tsdb.addPoint(METRIC_STRING, timestamp += 30, i, tags_local) + .joinUninterruptibly(); + if (two_metrics) { + tsdb.addPoint(METRIC_B_STRING, timestamp, i, tags_local) + .joinUninterruptibly(); + } + } + } + + protected void storeLongTimeSeriesMs() throws Exception { + setDataPointStorage(); + // dump a bunch of rows of two metrics so that we can test filtering out + // on the metric + HashMap<String, String> tags_local = new HashMap<String, String>(tags); + long timestamp = 1356998400000L; + for (int i = 1; i <= 300; i++) { + tsdb.addPoint(METRIC_STRING, timestamp += 500, i, tags_local) + .joinUninterruptibly(); + tsdb.addPoint(METRIC_B_STRING, timestamp, i, tags_local) + .joinUninterruptibly(); + } + + // dump a parallel set but invert the values + tags_local.clear(); + tags_local.put(TAGK_STRING, TAGV_B_STRING); + timestamp = 1356998400000L; + for (int i = 300; i > 0; i--) { + tsdb.addPoint(METRIC_STRING, timestamp += 500, i, tags_local) + .joinUninterruptibly(); + tsdb.addPoint(METRIC_B_STRING, timestamp, i, tags_local) + .joinUninterruptibly(); + } + } + + /** + * Create two metrics with same name, skipping every third point in host=web01 + * and every other point in host=web02. To wit: + * + * METRIC TAG t0 t1 t2 t3 t4 t5 ... + * sys.cpu.user web01 X 2 3 X 5 6 ... + * sys.cpu.user web02 X 299 X 297 X 295 ... + */ + protected void storeLongTimeSeriesWithMissingData() throws Exception { + setDataPointStorage(); + + // host=web01 + HashMap<String, String> tags_local = new HashMap<String, String>(tags); + long timestamp = 1356998400L; + for (int i = 0; i < 300; ++i) { + // Skip every third point. + if (0 != (i % 3)) { + tsdb.addPoint(METRIC_STRING, timestamp, i + 1, tags_local) + .joinUninterruptibly(); + } + timestamp += 10L; + } + + // host=web02 + tags_local.clear(); + tags_local.put(TAGK_STRING, TAGV_B_STRING); + timestamp = 1356998400L; + for (int i = 300; i > 0; --i) { + // Skip every other point. + if (0 != (i % 2)) { + tsdb.addPoint(METRIC_STRING, timestamp, i, tags_local) + .joinUninterruptibly(); + } + timestamp += 10L; + } + } + + protected void storeFloatTimeSeriesSeconds(final boolean two_metrics, + final boolean offset) throws Exception { + setDataPointStorage(); + // dump a bunch of rows of two metrics so that we can test filtering out + // on the metric + HashMap<String, String> tags_local = new HashMap<String, String>(tags); + long timestamp = 1356998400; + for (float i = 1.25F; i <= 76; i += 0.25F) { + tsdb.addPoint(METRIC_STRING, timestamp += 30, i, tags_local) + .joinUninterruptibly(); + if (two_metrics) { + tsdb.addPoint(METRIC_B_STRING, timestamp, i, tags_local) + .joinUninterruptibly(); + } + } + + // dump a parallel set but invert the values + tags_local.clear(); + tags_local.put(TAGK_STRING, TAGV_B_STRING); + timestamp = offset ? 1356998415 : 1356998400; + for (float i = 75F; i > 0; i -= 0.25F) { + tsdb.addPoint(METRIC_STRING, timestamp += 30, i, tags_local) + .joinUninterruptibly(); + if (two_metrics) { + tsdb.addPoint(METRIC_B_STRING, timestamp, i, tags_local) + .joinUninterruptibly(); + } + } + } + + protected void storeFloatTimeSeriesMs() throws Exception { + setDataPointStorage(); + // dump a bunch of rows of two metrics so that we can test filtering out + // on the metric + HashMap<String, String> tags_local = new HashMap<String, String>(tags); + long timestamp = 1356998400000L; + for (float i = 1.25F; i <= 76; i += 0.25F) { + tsdb.addPoint(METRIC_STRING, timestamp += 500, i, tags_local) + .joinUninterruptibly(); + tsdb.addPoint(METRIC_B_STRING, timestamp, i, tags_local) + .joinUninterruptibly(); + } + + // dump a parallel set but invert the values + tags_local.clear(); + tags_local.put(TAGK_STRING, TAGV_B_STRING); + timestamp = 1356998400000L; + for (float i = 75F; i > 0; i -= 0.25F) { + tsdb.addPoint(METRIC_STRING, timestamp += 500, i, tags_local) + .joinUninterruptibly(); + tsdb.addPoint(METRIC_B_STRING, timestamp, i, tags_local) + .joinUninterruptibly(); + } + } + + protected void storeMixedTimeSeriesSeconds() throws Exception { + setDataPointStorage(); + HashMap<String, String> tags_local = new HashMap<String, String>(tags); + long timestamp = 1356998400; + for (float i = 1.25F; i <= 76; i += 0.25F) { + if (i % 2 == 0) { + tsdb.addPoint(METRIC_STRING, timestamp += 30, (long)i, tags_local) + .joinUninterruptibly(); + } else { + tsdb.addPoint(METRIC_STRING, timestamp += 30, i, tags_local) + .joinUninterruptibly(); + } + } + } + + // dumps ints, floats, seconds and ms + protected void storeMixedTimeSeriesMsAndS() throws Exception { + setDataPointStorage(); + HashMap<String, String> tags_local = new HashMap<String, String>(tags); + long timestamp = 1356998400000L; + for (float i = 1.25F; i <= 76; i += 0.25F) { + long ts = timestamp += 500; + if (ts % 1000 == 0) { + ts /= 1000; + } + if (i % 2 == 0) { + tsdb.addPoint(METRIC_STRING, ts, (long)i, tags_local).joinUninterruptibly(); + } else { + tsdb.addPoint(METRIC_STRING, ts, i, tags_local).joinUninterruptibly(); + } + } + } + + //store histogram data points of {@link LongHistogramDataPointForTest} with second timestamp + protected void storeTestHistogramTimeSeriesSeconds(final boolean offset) throws Exception { + setDataPointStorage(); + + // dump a bunch of rows of two metrics so that we can test filtering out + // on the metric + HashMap<String, String> tags_local = new HashMap<String, String>(); + tags_local.put("host", "web01"); + + // note that the mock must have been configured properly + final int id = tsdb.histogramManager() + .getCodec(LongHistogramDataPointForTestDecoder.class); + + long timestamp = 1356998400; + for (int i = 1; i <= 300; i++) { + final LongHistogramDataPointForTest hdp = + new LongHistogramDataPointForTest(id, i); + tsdb.addHistogramPoint(HISTOGRAM_METRIC_STRING, timestamp += 30, + hdp.histogram(true), tags_local).joinUninterruptibly(); + } + + // dump a parallel set but invert the values + tags_local.clear(); + tags_local.put("host", "web02"); + timestamp = offset ? 1356998415 : 1356998400; + for (int i = 300; i > 0; i--) { + final LongHistogramDataPointForTest hdp = + new LongHistogramDataPointForTest(id, i); + tsdb.addHistogramPoint(HISTOGRAM_METRIC_STRING, timestamp += 30, + hdp.histogram(true), tags_local).joinUninterruptibly(); + } + } + + // store histogram data points of {@link LongHistogramDataPointForTest} with ms timestamp + protected void storeTestHistogramTimeSeriesMs() throws Exception { + setDataPointStorage(); + + // note that the mock must have been configured properly + final int id = tsdb.histogramManager() + .getCodec(LongHistogramDataPointForTestDecoder.class); + + // dump a bunch of rows of two metrics so that we can test filtering out + // on the metric + HashMap<String, String> tags = new HashMap<String, String>(1); + tags.put("host", "web01"); + long timestamp = 1356998400000L; + for (int i = 1; i <= 300; i++) { + timestamp += 500; + final LongHistogramDataPointForTest hdp = + new LongHistogramDataPointForTest(id, i); + tsdb.addHistogramPoint("msg.end2end.latency", timestamp, + hdp.histogram(true), tags).joinUninterruptibly(); + } // end for + + // dump a parallel set but invert the values + tags.clear(); + tags.put("host", "web02"); + timestamp = 1356998400000L; + for (int i = 300; i > 0; i--) { + timestamp += 500; + final LongHistogramDataPointForTest hdp = + new LongHistogramDataPointForTest(id, i); + tsdb.addHistogramPoint("msg.end2end.latency", timestamp, + hdp.histogram(true), tags).joinUninterruptibly(); + } // end for + } + + /** + * Validates the metric name, tags and annotations + * @param dps The datapoints array returned from the query + * @param index The index to peek into the array + * @param agged_tags Whether or not the tags were aggregated out + */ + protected void assertMeta(final DataPoints[] dps, final int index, + final boolean agged_tags) { + assertMeta(dps, index, agged_tags, false); + } + + /** + * Validates the metric name, tags and annotations + * @param dps The datapoints array returned from the query + * @param index The index to peek into the array + * @param agged_tags Whether or not the tags were aggregated out + * @param annotation Whether we're expecting a note or not + */ + protected void assertMeta(final DataPoints[] dps, final int index, + final boolean agged_tags, final boolean annotation) { + assertNotNull(dps); + assertEquals(METRIC_STRING, dps[index].metricName()); + + if (agged_tags) { + assertTrue(dps[index].getTags().isEmpty()); + assertEquals(TAGK_STRING, dps[index].getAggregatedTags().get(0)); + } else { + if (index == 0) { + assertTrue(dps[index].getAggregatedTags().isEmpty()); + assertEquals(TAGV_STRING, dps[index].getTags().get(TAGK_STRING)); + } else { + assertEquals(TAGV_B_STRING, dps[index].getTags().get(TAGK_STRING)); + } + } + + if (annotation) { + assertEquals(1, dps[index].getAnnotations().size()); + assertEquals(NOTE_DESCRIPTION, dps[index].getAnnotations().get(0) + .getDescription()); + assertEquals(NOTE_NOTES, dps[index].getAnnotations().get(0).getNotes()); + } else { + assertNull(dps[index].getAnnotations()); + } + } + + /** + * Stores a single annotation in the given row + * @param timestamp The time to store the data point at + * @throws Exception + */ + protected void storeAnnotation(final long timestamp) throws Exception { + final Annotation note = new Annotation(); + note.setTSUID("000001000001000001"); + note.setStartTime(timestamp); + note.setDescription(NOTE_DESCRIPTION); + note.setNotes(NOTE_NOTES); + note.syncToStorage(tsdb, false).joinUninterruptibly(); + } + + RollupQuery makeRollupQuery() { + Whitebox.setInternalState(tsdb, "rollups_split_queries", true); + final RollupInterval oneHourWithDelay = RollupInterval.builder() + .setTable("fake-rollup-table") + .setPreAggregationTable("fake-preagg-table") + .setInterval("1h") + .setRowSpan("1d") + .setDelaySla("2d") + .build(); + return new RollupQuery( + oneHourWithDelay, + Aggregators.SUM, + 3600000, + Aggregators.SUM + ); + } + + /** + * A fake {@link org.jboss.netty.util.Timer} implementation. + * Instead of executing the task it will store that task in a internal state + * and provides a function to start the execution of the stored task. + * This implementation thus allows the flexibility of simulating the + * things that will be going on during the time out period of a TimerTask. + * This was mainly return to simulate the timeout period for + * alreadyNSREdRegion test, where the region will be in the NSREd mode only + * during this timeout period, which was difficult to simulate using the + * above {@link FakeTimer} implementation, as we don't get back the control + * during the timeout period + * + * Here it will hold at most two Tasks. We have two tasks here because when + * one is being executed, it may call for newTimeOut for another task. + */ + public static final class FakeTaskTimer extends HashedWheelTimer { + + public TimerTask newPausedTask = null; + public TimerTask pausedTask = null; + public Timeout timeout = null; + + @Override + public synchronized Timeout newTimeout(final TimerTask task, + final long delay, + final TimeUnit unit) { + if (pausedTask == null) { + pausedTask = task; + } else if (newPausedTask == null) { + newPausedTask = task; + } else { + throw new IllegalStateException("Cannot Pause Two Timer Tasks"); + } + timeout = mock(Timeout.class); + return timeout; + } + + @Override + public Set<Timeout> stop() { + return null; + } + + public boolean continuePausedTask() { + if (pausedTask == null) { + return false; + } + try { + if (newPausedTask != null) { + throw new IllegalStateException("Cannot be in this state"); + } + pausedTask.run(null); // Argument never used in this code base + pausedTask = newPausedTask; + newPausedTask = null; + return true; + } catch (Exception e) { + throw new RuntimeException("Timer task failed: " + pausedTask, e); + } + } + } + + /** + * A little class used to throw a very specific type of exception for matching + * in Unit Tests. + */ + public static class UnitTestException extends RuntimeException { + public UnitTestException() { } + public UnitTestException(final String msg) { + super(msg); + } + private static final long serialVersionUID = -4404095849459619922L; + } +} diff --git a/test/core/HistogramSeekableViewForTest.java b/test/core/HistogramSeekableViewForTest.java new file mode 100644 index 0000000000..e68e5b3d67 --- /dev/null +++ b/test/core/HistogramSeekableViewForTest.java @@ -0,0 +1,257 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.NoSuchElementException; + +import org.hbase.async.Bytes; +import org.junit.Test; + +/** Helper class to mock HistogramSeekableView. */ + +public class HistogramSeekableViewForTest { + + /** + * Creates a {@link HistogramSeekableView} object to iterate the given data + * points. + * @param data_points Test data. + * @return A {@link HistogramSeekableView} object + */ + public static HistogramSeekableView fromArray( + final HistogramDataPoint[] data_points) { + return new MockHistogramSeekableView(data_points); + } + + /** + * Creates a {@link HistogramSeekableView} that generates a sequence of data + * points. + * @param start_time Starting timestamp + * @param sample_period Average sample period of data points + * @param num_data_points Total number of data points to generate + * @return A {@link HistogramSeekableView} object + */ + public static HistogramSeekableView generator(final long start_time, + final long sample_period, + final int num_data_points) { + return new DataPointGenerator(start_time, sample_period, num_data_points); + } + + /** Iterates an array of data points. */ + public static class MockHistogramSeekableView implements HistogramSeekableView { + + private final HistogramDataPoint[] data_points; + private int index = 0; + + MockHistogramSeekableView(final HistogramDataPoint[] data_points2) { + this.data_points = data_points2; + } + + @Override + public boolean hasNext() { + return data_points.length > index; + } + + @Override + public HistogramDataPoint next() { + if (hasNext()) { + return data_points[index++]; + } + throw new NoSuchElementException("no more values"); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + @Override + public void seek(long timestamp) { + for (index = 0; index < data_points.length; ++index) { + if (data_points[index].timestamp() >= timestamp) { + break; + } + } + } + + public void resetIndex() { + index = 0; + } + } + + /** Generates a sequence of data points. */ + private static class DataPointGenerator implements HistogramSeekableView { + + private final long start_time_ms; + private final long sample_period_ms; + private final int num_data_points; + private SimpleHistogramDataPointAdapter current_data; + private int current = 0; + + DataPointGenerator(final long start_time_ms, final long sample_period_ms, + final int num_data_points) { + this.start_time_ms = start_time_ms; + this.sample_period_ms = sample_period_ms; + this.num_data_points = num_data_points; + current_data = new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(1, 0), 100L); + + rewind(); + } + + @Override + public boolean hasNext() { + return current < num_data_points; + } + + @Override + public HistogramDataPoint next() { + if (hasNext()) { + generateData(); + ++current; + return current_data; + } + throw new NoSuchElementException("no more values"); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + @Override + public void seek(long timestamp) { + rewind(); + current = (int)((timestamp -1 - start_time_ms) / sample_period_ms); + if (current < 0) { + current = 0; + } + while (generateTimestamp() < timestamp) { + ++current; + } + } + + private void rewind() { + current = 0; + generateData(); + } + + private void generateData() { + current_data = new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(1, current), generateTimestamp()); + } + + private long generateTimestamp() { + long timestamp = start_time_ms + sample_period_ms * current; + return timestamp + (((current % 2) == 0) ? -1000 : 1000); + } + } + + @Test + public void testDataPointGenerator() { + DataPointGenerator hdpg = new DataPointGenerator(100000, 10000, 5); + HistogramDataPoint[] expected_data_points = new HistogramDataPoint[] { + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(1, 0), 99000), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(1, 1), 111000), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(1, 2), 119000), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(1, 3), 131000), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(1, 4), 139000), + }; + for (HistogramDataPoint expected: expected_data_points) { + assertTrue(hdpg.hasNext()); + HistogramDataPoint dp = hdpg.next(); + assertEquals(expected.timestamp(), dp.timestamp()); + assertEquals(Bytes.getLong(expected.getRawData(false)), + Bytes.getLong(dp.getRawData(false))); + } + assertFalse(hdpg.hasNext()); + } + + @Test + public void testDataPointGenerator_seek() { + DataPointGenerator hdpg = new DataPointGenerator(100000, 10000, 5); + hdpg.seek(119000); + HistogramDataPoint[] expected_data_points = new HistogramDataPoint[] { + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(1, 2), 119000), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(1, 3), 131000), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(1, 4), 139000), + }; + for (HistogramDataPoint expected: expected_data_points) { + assertTrue(hdpg.hasNext()); + HistogramDataPoint hdp = hdpg.next(); + assertEquals(expected.timestamp(), hdp.timestamp()); + assertEquals(Bytes.getLong(expected.getRawData(false)), + Bytes.getLong(hdp.getRawData(false))); + } + assertFalse(hdpg.hasNext()); + } + + @Test + public void testDataPointGenerator_seekToFirst() { + DataPointGenerator hdpg = new DataPointGenerator(100000, 10000, 5); + hdpg.seek(100000); + HistogramDataPoint[] expected_data_points = new HistogramDataPoint[] { + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(1, 1), 111000), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(1, 2), 119000), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(1, 3), 131000), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(1, 4), 139000), + }; + for (HistogramDataPoint expected: expected_data_points) { + assertTrue(hdpg.hasNext()); + HistogramDataPoint hdp = hdpg.next(); + assertEquals(expected.timestamp(), hdp.timestamp()); + assertEquals(Bytes.getLong(expected.getRawData(false)), + Bytes.getLong(hdp.getRawData(false))); + } + assertFalse(hdpg.hasNext()); + } + + @Test + public void testDataPointGenerator_seekToSecond() { + DataPointGenerator hdpg = new DataPointGenerator(100000, 10000, 5); + hdpg.seek(100001); + HistogramDataPoint[] expected_data_points = new HistogramDataPoint[] { + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(1, 1), 111000), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(1, 2), 119000), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(1, 3), 131000), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(1, 4), 139000), + }; + for (HistogramDataPoint expected: expected_data_points) { + assertTrue(hdpg.hasNext()); + HistogramDataPoint hdp = hdpg.next(); + assertEquals(expected.timestamp(), hdp.timestamp()); + assertEquals(Bytes.getLong(expected.getRawData(false)), + Bytes.getLong(hdp.getRawData(false))); + } + assertFalse(hdpg.hasNext()); + } +} diff --git a/test/core/LongHistogramDataPointForTest.java b/test/core/LongHistogramDataPointForTest.java new file mode 100644 index 0000000000..f716ad9f2f --- /dev/null +++ b/test/core/LongHistogramDataPointForTest.java @@ -0,0 +1,119 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2023 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. + +package net.opentsdb.core; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.hbase.async.Bytes; +import org.junit.Ignore; + +@Ignore +public class LongHistogramDataPointForTest implements Histogram { + private final int id; + private long data; + + public LongHistogramDataPointForTest(final int id) { + this.id = id; + } + + LongHistogramDataPointForTest(final int id, final long value) { + this.id = id; + this.data = value; + } + + protected LongHistogramDataPointForTest(final LongHistogramDataPointForTest rhs) { + this.id = rhs.id; + this.data = rhs.data; + } + + protected LongHistogramDataPointForTest(final LongHistogramDataPointForTest rhs, + final long timestamp) { + this.id = rhs.id; + this.data = rhs.data; + } + + public void setRawData(final byte[] data) { + this.data = Bytes.getLong(data); + } + + @Override + public double percentile(double p) { + return data * p; + } + + @Override + public List<Double> percentiles(List<Double> p) { + List<Double> rs = new ArrayList<Double>(); + for (Double d : p) { + rs.add(d.doubleValue() * data); + } + return rs; + } + + @Override + public void aggregate(Histogram histo, HistogramAggregation func) { + if (!(histo instanceof LongHistogramDataPointForTest)) { + throw new IllegalArgumentException("The object must be an instance of the " + + "LongHistogramDataPointForTest"); + } + + long agg = this.data + Bytes.getLong(histo.histogram(false)); + this.data = agg; + } + + @Override + public Histogram clone() { + return new LongHistogramDataPointForTest(this); + } + + @Override + public int getId() { + return id; + } + + @Override + public byte[] histogram(boolean include_id) { + if (include_id) { + final byte[] result = new byte[9]; + result[0] = (byte) id; + System.arraycopy(Bytes.fromLong(data), 0, result, 1, 8); + return result; + } + return Bytes.fromLong(data); + } + + @Override + public void fromHistogram(byte[] raw, boolean includes_id) { + if (includes_id) { + data = Bytes.getLong(raw, 1); + } else { + data = Bytes.getLong(raw); + } + } + + @Override + public Map getHistogram() { + // TODO Auto-generated method stub + return null; + } + + @Override + public void aggregate(List<Histogram> histos, HistogramAggregation func) { + // TODO Auto-generated method stub + + } + +} diff --git a/test/core/LongHistogramDataPointForTestDecoder.java b/test/core/LongHistogramDataPointForTestDecoder.java new file mode 100644 index 0000000000..e7dd40dca9 --- /dev/null +++ b/test/core/LongHistogramDataPointForTestDecoder.java @@ -0,0 +1,34 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2023 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import org.junit.Ignore; + +@Ignore +public class LongHistogramDataPointForTestDecoder extends HistogramDataPointCodec { + + @Override + public Histogram decode(byte[] raw_data, final boolean includes_id) { + final LongHistogramDataPointForTest dp = new LongHistogramDataPointForTest(id); + dp.fromHistogram(raw_data, includes_id); + return dp; + } + + + @Override + public byte[] encode(Histogram data_point, final boolean include_id) { + // TODO Auto-generated method stub + return null; + } + +} diff --git a/test/core/SeekableViewsForTest.java b/test/core/SeekableViewsForTest.java index dab66d7c37..bc6463da1c 100644 --- a/test/core/SeekableViewsForTest.java +++ b/test/core/SeekableViewsForTest.java @@ -32,7 +32,8 @@ public static SeekableView fromArray(final DataPoint[] data_points) { } /** - * Creates a {@link SeekableView} that generates a sequence of data points. + * Creates a {@link SeekableView} that generates a sequence of data points + * where the starting value is 1 and it is incremented by 1 each iteration. * @param start_time Starting timestamp * @param sample_period Average sample period of data points * @param num_data_points Total number of data points to generate @@ -43,12 +44,56 @@ public static SeekableView generator(final long start_time, final long sample_period, final int num_data_points, final boolean is_integer) { + return generator(start_time, sample_period, num_data_points, + is_integer, 0, 1); + } + + /** + * Creates a {@link SeekableView} that generates a sequence of data points. + * @param start_time Starting timestamp + * @param sample_period Average sample period of data points + * @param num_data_points Total number of data points to generate + * @param is_integer True to generate a sequence of integer data points. + * @param starting_value The starting data point value. + * @param increment How much to increment the values each iteration. + * @return A {@link SeekableView} object + */ + public static SeekableView generator(final long start_time, + final long sample_period, + final int num_data_points, + final boolean is_integer, + final double starting_value, + final double increment) { + return generator(start_time, sample_period, num_data_points, + is_integer, starting_value, increment, false); + } + + /** + * Creates a {@link SeekableView} that generates a sequence of data points. + * @param start_time Starting timestamp + * @param sample_period Average sample period of data points + * @param num_data_points Total number of data points to generate + * @param is_integer True to generate a sequence of integer data points. + * @param starting_value The starting data point value. + * @param increment How much to increment the values each iteration. + * @param wholes_as_integer Whether or not to return whole numbers (1.0, 2.0, + * etc) as integers to test for functions that should support both. + * Note: Ignored if is_integer is true. + * @return A {@link SeekableView} object + */ + public static SeekableView generator(final long start_time, + final long sample_period, + final int num_data_points, + final boolean is_integer, + final double starting_value, + final double increment, + final boolean wholes_as_integer) { return new DataPointGenerator(start_time, sample_period, num_data_points, - is_integer); + is_integer, starting_value, increment, wholes_as_integer); } /** Iterates an array of data points. */ - private static class MockSeekableView implements SeekableView { + public static class MockSeekableView implements SeekableView { private final DataPoint[] data_points; private int index = 0; @@ -83,37 +128,56 @@ public void seek(long timestamp) { } } } + + public void resetIndex() { + index = 0; + } } /** Generates a sequence of data points. */ private static class DataPointGenerator implements SeekableView { - private final long start_time_ms; private final long sample_period_ms; private final int num_data_points; private final boolean is_integer; + private final double increment; private final MutableDataPoint current_data = new MutableDataPoint(); - private int current = 0; - + private final MutableDataPoint next_data = new MutableDataPoint(); + private final boolean wholes_as_integer; + private int dps_emitted = 0; + DataPointGenerator(final long start_time_ms, final long sample_period_ms, - final int num_data_points, final boolean is_integer) { - this.start_time_ms = start_time_ms; + final int num_data_points, final boolean is_integer, + final double starting_value, final double increment, + final boolean wholes_as_integer) { this.sample_period_ms = sample_period_ms; this.num_data_points = num_data_points; this.is_integer = is_integer; - rewind(); + this.increment = increment; + this.wholes_as_integer = wholes_as_integer; + if (is_integer) { + next_data.reset(start_time_ms, (long)starting_value); + } else { + if (wholes_as_integer && + (starting_value == Math.floor(starting_value)) && + !Double.isInfinite(starting_value)) { + next_data.reset(start_time_ms, (long)starting_value); + } else { + next_data.reset(start_time_ms, starting_value); + } + } } @Override public boolean hasNext() { - return current < num_data_points; + return dps_emitted < num_data_points; } @Override public DataPoint next() { if (hasNext()) { - generateData(); - ++current; + current_data.reset(next_data); + advance(); return current_data; } throw new NoSuchElementException("no more values"); @@ -126,44 +190,39 @@ public void remove() { @Override public void seek(long timestamp) { - rewind(); - current = (int)((timestamp -1 - start_time_ms) / sample_period_ms); - if (current < 0) { - current = 0; - } - while (generateTimestamp() < timestamp) { - ++current; + while (next_data.timestamp() < timestamp && dps_emitted < num_data_points) { + advance(); } } - - private void rewind() { - current = 0; - generateData(); - } - - private void generateData() { + + private void advance() { if (is_integer) { - current_data.reset(generateTimestamp(), current); + next_data.reset(next_data.timestamp() + sample_period_ms, + next_data.longValue() + (long)increment); } else { - current_data.reset(generateTimestamp(), (double)current); + final double next = next_data.toDouble() + increment; + if (wholes_as_integer && + (next == Math.floor(next)) && !Double.isInfinite(next)) { + next_data.reset(next_data.timestamp() + sample_period_ms, (long)next); + } else { + next_data.reset(next_data.timestamp() + sample_period_ms, next); + } } + dps_emitted++; } - - private long generateTimestamp() { - long timestamp = start_time_ms + sample_period_ms * current; - return timestamp + (((current % 2) == 0) ? -1000 : 1000); - } + + } - + @Test public void testDataPointGenerator() { - DataPointGenerator dpg = new DataPointGenerator(100000, 10000, 5, true); + SeekableView dpg = generator(100000, 10000, 5, true); DataPoint[] expected_data_points = new DataPoint[] { - MutableDataPoint.ofLongValue(99000, 0), - MutableDataPoint.ofLongValue(111000, 1), - MutableDataPoint.ofLongValue(119000, 2), - MutableDataPoint.ofLongValue(131000, 3), - MutableDataPoint.ofLongValue(139000, 4), + MutableDataPoint.ofLongValue(100000, 0), + MutableDataPoint.ofLongValue(110000, 1), + MutableDataPoint.ofLongValue(120000, 2), + MutableDataPoint.ofLongValue(130000, 3), + MutableDataPoint.ofLongValue(140000, 4), }; for (DataPoint expected: expected_data_points) { assertTrue(dpg.hasNext()); @@ -176,13 +235,13 @@ public void testDataPointGenerator() { @Test public void testDataPointGenerator_double() { - DataPointGenerator dpg = new DataPointGenerator(100000, 10000, 5, false); + SeekableView dpg = generator(100000, 10000, 5, false); DataPoint[] expected_data_points = new DataPoint[] { - MutableDataPoint.ofDoubleValue(99000, 0), - MutableDataPoint.ofDoubleValue(111000, 1), - MutableDataPoint.ofDoubleValue(119000, 2), - MutableDataPoint.ofDoubleValue(131000, 3), - MutableDataPoint.ofDoubleValue(139000, 4), + MutableDataPoint.ofDoubleValue(100000, 0), + MutableDataPoint.ofDoubleValue(110000, 1), + MutableDataPoint.ofDoubleValue(120000, 2), + MutableDataPoint.ofDoubleValue(130000, 3), + MutableDataPoint.ofDoubleValue(140000, 4), }; for (DataPoint expected: expected_data_points) { assertTrue(dpg.hasNext()); @@ -195,12 +254,12 @@ public void testDataPointGenerator_double() { @Test public void testDataPointGenerator_seek() { - DataPointGenerator dpg = new DataPointGenerator(100000, 10000, 5, true); + SeekableView dpg = generator(100000, 10000, 5, true); dpg.seek(119000); DataPoint[] expected_data_points = new DataPoint[] { - MutableDataPoint.ofLongValue(119000, 2), - MutableDataPoint.ofLongValue(131000, 3), - MutableDataPoint.ofLongValue(139000, 4), + MutableDataPoint.ofLongValue(120000, 2), + MutableDataPoint.ofLongValue(130000, 3), + MutableDataPoint.ofLongValue(140000, 4), }; for (DataPoint expected: expected_data_points) { assertTrue(dpg.hasNext()); @@ -213,13 +272,14 @@ public void testDataPointGenerator_seek() { @Test public void testDataPointGenerator_seekToFirst() { - DataPointGenerator dpg = new DataPointGenerator(100000, 10000, 5, true); + SeekableView dpg = generator(100000, 10000, 5, true); dpg.seek(100000); DataPoint[] expected_data_points = new DataPoint[] { - MutableDataPoint.ofLongValue(111000, 1), - MutableDataPoint.ofLongValue(119000, 2), - MutableDataPoint.ofLongValue(131000, 3), - MutableDataPoint.ofLongValue(139000, 4), + MutableDataPoint.ofLongValue(100000, 0), + MutableDataPoint.ofLongValue(110000, 1), + MutableDataPoint.ofLongValue(120000, 2), + MutableDataPoint.ofLongValue(130000, 3), + MutableDataPoint.ofLongValue(140000, 4), }; for (DataPoint expected: expected_data_points) { assertTrue(dpg.hasNext()); @@ -232,13 +292,13 @@ public void testDataPointGenerator_seekToFirst() { @Test public void testDataPointGenerator_seekToSecond() { - DataPointGenerator dpg = new DataPointGenerator(100000, 10000, 5, true); + SeekableView dpg = generator(100000, 10000, 5, true); dpg.seek(100001); DataPoint[] expected_data_points = new DataPoint[] { - MutableDataPoint.ofLongValue(111000, 1), - MutableDataPoint.ofLongValue(119000, 2), - MutableDataPoint.ofLongValue(131000, 3), - MutableDataPoint.ofLongValue(139000, 4), + MutableDataPoint.ofLongValue(110000, 1), + MutableDataPoint.ofLongValue(120000, 2), + MutableDataPoint.ofLongValue(130000, 3), + MutableDataPoint.ofLongValue(140000, 4), }; for (DataPoint expected: expected_data_points) { assertTrue(dpg.hasNext()); @@ -248,4 +308,27 @@ public void testDataPointGenerator_seekToSecond() { } assertFalse(dpg.hasNext()); } + + @Test + public void testDataPointGeneratorWholes() { + SeekableView dpg = generator(100000, 10000, 5, false, 0, 1.5, true); + DataPoint[] expected_data_points = new DataPoint[] { + MutableDataPoint.ofLongValue(100000, 0), + MutableDataPoint.ofDoubleValue(110000, 1.5), + MutableDataPoint.ofLongValue(120000, 3), + MutableDataPoint.ofDoubleValue(130000, 4.5), + MutableDataPoint.ofLongValue(140000, 6), + }; + for (DataPoint expected: expected_data_points) { + assertTrue(dpg.hasNext()); + DataPoint dp = dpg.next(); + assertEquals(expected.timestamp(), dp.timestamp()); + if (expected.isInteger()) { + assertEquals(expected.longValue(), dp.longValue()); + } else { + assertEquals(expected.doubleValue(), dp.doubleValue(), 0.001); + } + } + assertFalse(dpg.hasNext()); + } } diff --git a/test/core/TestAggregationIterator.java b/test/core/TestAggregationIterator.java index 9cf49fb574..1d1d229873 100644 --- a/test/core/TestAggregationIterator.java +++ b/test/core/TestAggregationIterator.java @@ -285,4 +285,35 @@ public void testAggregate10000Spans() { public void testAggregate250000Spans() { testMeasureAggregationLatency(250000, 10.0); } + + @Test + public void pfsum() { + // TODO - More UTs around this one. + iterators = new SeekableView[] { + SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(BASE_TIME, 40), + //MutableDataPoint.ofLongValue(BASE_TIME + 10000, 50), // skip one + MutableDataPoint.ofLongValue(BASE_TIME + 30000, 70) + }), + SeekableViewsForTest.fromArray(DATA_POINTS_2), + }; + AggregationIterator sgai = AggregationIterator.createForTesting(iterators, + start_time_ms, end_time_ms, SUM, Interpolation.PREV, rate); + // Checks if all the distinct timestamps of both spans appear and missing + // data point of one span for a timestamp of one span was interpolated. + DataPoint[] expected_data_points = new DataPoint[] { + MutableDataPoint.ofLongValue(BASE_TIME, 40), + MutableDataPoint.ofLongValue(BASE_TIME + 10000, 37 + 40), + // 60 is the interpolated value. + MutableDataPoint.ofLongValue(BASE_TIME + 20000, 48 + 40), + MutableDataPoint.ofLongValue(BASE_TIME + 30000, 70) + }; + for (DataPoint expected: expected_data_points) { + assertTrue(sgai.hasNext()); + DataPoint dp = sgai.next(); + assertEquals(expected.timestamp(), dp.timestamp()); + assertEquals(expected.longValue(), dp.longValue()); + } + assertFalse(sgai.hasNext()); + } } diff --git a/test/core/TestAggregators.java b/test/core/TestAggregators.java index 6d01e0bd2c..54273dea7c 100644 --- a/test/core/TestAggregators.java +++ b/test/core/TestAggregators.java @@ -1,5 +1,5 @@ // This file is part of OpenTSDB. -// Copyright (C) 2012 The OpenTSDB Authors. +// Copyright (C) 2012-2016 The OpenTSDB Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU Lesser General Public License as published by @@ -12,6 +12,10 @@ // see <http://www.gnu.org/licenses/>. package net.opentsdb.core; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import java.util.Random; import org.junit.Assert; @@ -22,7 +26,6 @@ public final class TestAggregators { private static final Random random; static { final long seed = System.nanoTime(); - System.out.println("Random seed: " + seed); random = new Random(seed); } @@ -37,26 +40,37 @@ public final class TestAggregators { /** Helper class to hold a bunch of numbers we can iterate on. */ private static final class Numbers implements Aggregator.Longs, Aggregator.Doubles { - private final long[] numbers; + private final long[] longs; + private final double[] doubles; private int i = 0; public Numbers(final long[] numbers) { - this.numbers = numbers; + longs = numbers; + doubles = null; + } + + public Numbers(final double[] numbers) { + longs = null; + doubles = numbers; } + public boolean isInteger() { + return longs != null ? true : false; + } + @Override public boolean hasNextValue() { - return i < numbers.length; + return longs != null ? i < longs.length : i < doubles.length; } @Override public long nextLongValue() { - return numbers[i++]; + return longs[i++]; } @Override public double nextDoubleValue() { - return numbers[i++]; + return doubles[i++]; } void reset() { @@ -113,9 +127,6 @@ private static void checkSimilarStdDev(final long[] values, final double epsilon) { final Numbers numbers = new Numbers(values); final Aggregator agg = Aggregators.get("dev"); - - Assert.assertEquals(expected, agg.runDouble(numbers), epsilon); - numbers.reset(); Assert.assertEquals(expected, agg.runLong(numbers), Math.max(epsilon, 1.0)); } @@ -134,4 +145,141 @@ private static double naiveStdDev(long[] values) { return Math.sqrt(variance); } + @Test + public void testPercentiles() { + final long[] longValues = new long[1000]; + for (int i = 0; i < longValues.length; i++) { + longValues[i] = i+1; + } + + Numbers values = new Numbers(longValues); + assertAggregatorEquals(500, Aggregators.get("p50"), values); + assertAggregatorEquals(750, Aggregators.get("p75"), values); + assertAggregatorEquals(900, Aggregators.get("p90"), values); + assertAggregatorEquals(950, Aggregators.get("p95"), values); + assertAggregatorEquals(990, Aggregators.get("p99"), values); + assertAggregatorEquals(999, Aggregators.get("p999"), values); + + assertAggregatorEquals(500, Aggregators.get("ep50r3"), values); + assertAggregatorEquals(750, Aggregators.get("ep75r3"), values); + assertAggregatorEquals(900, Aggregators.get("ep90r3"), values); + assertAggregatorEquals(950, Aggregators.get("ep95r3"), values); + assertAggregatorEquals(990, Aggregators.get("ep99r3"), values); + assertAggregatorEquals(999, Aggregators.get("ep999r3"), values); + + assertAggregatorEquals(500, Aggregators.get("ep50r7"), values); + assertAggregatorEquals(750, Aggregators.get("ep75r7"), values); + assertAggregatorEquals(900, Aggregators.get("ep90r7"), values); + assertAggregatorEquals(950, Aggregators.get("ep95r7"), values); + assertAggregatorEquals(990, Aggregators.get("ep99r7"), values); + assertAggregatorEquals(999, Aggregators.get("ep999r7"), values); + } + + @Test + public void testFirst() { + final long[] values = new long[10]; + for (int i = 0; i < values.length; i++) { + values[i] = i; + } + + Aggregator agg = Aggregators.FIRST; + Numbers numbers = new Numbers(values); + assertEquals(0, agg.runLong(numbers)); + + final double[] doubles = new double[10]; + double val = 0.5; + for (int i = 0; i < doubles.length; i++) { + doubles[i] = val++; + } + + numbers = new Numbers(doubles); + assertEquals(0.5, agg.runDouble(numbers), EPSILON_PERCENTAGE); + } + + @Test + public void testLast() { + final long[] values = new long[10]; + for (int i = 0; i < values.length; i++) { + values[i] = i; + } + + Aggregator agg = Aggregators.LAST; + Numbers numbers = new Numbers(values); + assertEquals(9, agg.runLong(numbers)); + + final double[] doubles = new double[10]; + double val = 0.5; + for (int i = 0; i < doubles.length; i++) { + doubles[i] = val++; + } + + numbers = new Numbers(doubles); + assertEquals(9.5, agg.runDouble(numbers), EPSILON_PERCENTAGE); + } + + @Test + public void testMedian() { + final Aggregator agg = Aggregators.get("median"); + Numbers numbers = new Numbers(new long[] { 5, 2, -1, 400, 3 }); + assertEquals(3, agg.runLong(numbers)); + + numbers = new Numbers(new long[] { 5, 2, -1, 400, 3, -42 }); + assertEquals(3, agg.runLong(numbers)); + + numbers = new Numbers(new long[] { 42 }); + assertEquals(42, agg.runLong(numbers)); + + numbers = new Numbers(new long[] { }); + try { + assertEquals(42, agg.runLong(numbers)); + fail("Expected IllegalStateException"); + } catch (IllegalStateException e) { } + + numbers = new Numbers(new double[] { 5.1, 2.434, -1.99, 400.69487, 3.15168 }); + assertEquals(3.15168, agg.runDouble(numbers), 0.0001); + + numbers = new Numbers(new double[] { 5.1, 2.434, -1.99, 400.69487, + 3.15168, -42 }); + assertEquals(3.15168, agg.runDouble(numbers), 0.0001); + + numbers = new Numbers(new double[] { 42.5 }); + assertEquals(42.5, agg.runDouble(numbers), 0.0001); + + numbers = new Numbers(new double[] { }); + assertTrue(Double.isNaN(agg.runDouble(numbers))); + } + + private void assertAggregatorEquals(long value, Aggregator agg, Numbers numbers) { + if (numbers.isInteger()) { + Assert.assertEquals(value, agg.runLong(numbers)); + } else { + Assert.assertEquals((double)value, agg.runDouble(numbers), 1.0); + } + numbers.reset(); + } + + @Test + public void testSquareSumFewDataInputs(){ + final long[] longValues = new long[2]; + for (int i = 0; i < longValues.length; i++) { + longValues[i] = i + 1; + } + + Numbers values = new Numbers(longValues); + assertAggregatorEquals(5, net.opentsdb.core.Aggregators.get("squareSum"), values); + } + + @Test + public void testSquareSumRandomInputs(){ + final long[] longValues = new long[100]; + long summ = 0; + for (int i = 0; i < longValues.length; i++) { + long temp = random.nextLong(); + longValues[i] = temp; + summ += temp * temp; + } + + Numbers values = new Numbers(longValues); + assertAggregatorEquals(summ, net.opentsdb.core.Aggregators.get("squareSum"), values); + } } diff --git a/test/core/TestAppendDataPoints.java b/test/core/TestAppendDataPoints.java new file mode 100644 index 0000000000..ee89501575 --- /dev/null +++ b/test/core/TestAppendDataPoints.java @@ -0,0 +1,373 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.util.Collection; +import java.util.Iterator; + +import net.opentsdb.core.Internal.Cell; +import net.opentsdb.storage.MockBase; + +import org.hbase.async.HBaseClient; +import org.hbase.async.KeyValue; +import org.hbase.async.PutRequest; +import org.junit.Test; +import org.powermock.reflect.Whitebox; + +public class TestAppendDataPoints extends BaseTsdbTest { + + private static final byte[] CF = "t".getBytes(); + private static final byte[] DPQ_S = new byte[] { 0, 0x20 }; + private static final byte[] DPQ_MS = new byte[] { (byte) 0xF0, 0, 0x20, 0 }; + private static final byte[] DPV = new byte[] { 42 }; + private static final byte[] DPV2 = new byte[] { 24 }; + private static final byte[] ROW_KEY = new byte[] { + 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, 0, 0, 1, 0, 0, 1 }; + + @Test + public void ctorForWrites() throws Exception { + assertNotNull(new AppendDataPoints(DPQ_S, DPV)); + assertNotNull(new AppendDataPoints(DPQ_MS, DPV)); + + try { + assertNotNull(new AppendDataPoints(DPQ_S, null)); + fail("Expected an IllegalArgumentException"); + } catch (IllegalArgumentException iae) { } + + try { + assertNotNull(new AppendDataPoints(null, DPV)); + fail("Expected an IllegalArgumentException"); + } catch (IllegalArgumentException iae) { } + + try { + assertNotNull(new AppendDataPoints(null, null)); + fail("Expected an IllegalArgumentException"); + } catch (IllegalArgumentException iae) { } + } + + @Test + public void toByteArray() throws Exception { + AppendDataPoints adp = new AppendDataPoints(DPQ_S, DPV); + assertArrayEquals(MockBase.concatByteArrays(DPQ_S, DPV), adp.getBytes()); + + adp = new AppendDataPoints(DPQ_MS, DPV); + assertArrayEquals(MockBase.concatByteArrays(DPQ_MS, DPV), adp.getBytes()); + + adp = new AppendDataPoints(MockBase.concatByteArrays(DPQ_MS, DPV), + MockBase.concatByteArrays(DPQ_S, DPV2)); + assertArrayEquals(MockBase.concatByteArrays(DPQ_MS, DPV, DPQ_S, DPV2), + adp.getBytes()); + } + + @Test + public void parseKeyValue() throws Exception { + KeyValue kv = new KeyValue(ROW_KEY, CF, + AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(DPQ_S, DPV)); + AppendDataPoints adp = new AppendDataPoints(); + Collection<Cell> cells = adp.parseKeyValue(tsdb, kv); + assertEquals(1, cells.size()); + Cell cell = cells.iterator().next(); + assertArrayEquals(DPV, cell.value); + assertEquals(1356998402000L, cell.timestamp(1356998400)); + assertArrayEquals(DPQ_S, adp.qualifier()); + assertArrayEquals(DPV, adp.value()); + verify(client, never()).put(any(PutRequest.class)); + + kv = new KeyValue(ROW_KEY, CF, + AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(DPQ_MS, DPV)); + adp = new AppendDataPoints(); + cells = adp.parseKeyValue(tsdb, kv); + assertEquals(1, cells.size()); + cell = cells.iterator().next(); + assertArrayEquals(DPV, cell.value); + assertEquals(1356998400128L, cell.timestamp(1356998400)); + assertArrayEquals(DPQ_MS, adp.qualifier()); + assertArrayEquals(DPV, adp.value()); + verify(client, never()).put(any(PutRequest.class)); + + // some odd offset + kv = new KeyValue(ROW_KEY, CF, + new byte[] { AppendDataPoints.APPEND_COLUMN_PREFIX, 42, 42 }, + MockBase.concatByteArrays(DPQ_MS, DPV)); + adp = new AppendDataPoints(); + cells = adp.parseKeyValue(tsdb, kv); + assertEquals(1, cells.size()); + cell = cells.iterator().next(); + assertArrayEquals(DPV, cell.value); + assertEquals(1356998400128L, cell.timestamp(1356998400)); + assertArrayEquals(DPQ_MS, adp.qualifier()); + assertArrayEquals(DPV, adp.value()); + verify(client, never()).put(any(PutRequest.class)); + + kv = new KeyValue(ROW_KEY, CF, + AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(DPQ_MS, DPV, DPQ_S, DPV2)); + adp = new AppendDataPoints(); + cells = adp.parseKeyValue(tsdb, kv); + assertEquals(2, cells.size()); + Iterator<Cell> iterator = cells.iterator(); + cell = iterator.next(); + assertArrayEquals(DPV, cell.value); + assertEquals(1356998400128L, cell.timestamp(1356998400)); + cell = iterator.next(); + assertArrayEquals(DPV2, cell.value); + assertEquals(1356998402000L, cell.timestamp(1356998400)); + assertArrayEquals(MockBase.concatByteArrays(DPQ_MS, DPQ_S), adp.qualifier()); + assertArrayEquals(MockBase.concatByteArrays(DPV, DPV2), adp.value()); + verify(client, never()).put(any(PutRequest.class)); + + // out of order + kv = new KeyValue(ROW_KEY, CF, + AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(DPQ_S, DPV2, DPQ_MS, DPV)); + adp = new AppendDataPoints(); + cells = adp.parseKeyValue(tsdb, kv); + assertEquals(2, cells.size()); + iterator = cells.iterator(); + cell = iterator.next(); + assertArrayEquals(DPV, cell.value); + assertEquals(1356998400128L, cell.timestamp(1356998400)); + cell = iterator.next(); + assertArrayEquals(DPV2, cell.value); + assertEquals(1356998402000L, cell.timestamp(1356998400)); + assertArrayEquals(MockBase.concatByteArrays(DPQ_MS, DPQ_S), adp.qualifier()); + assertArrayEquals(MockBase.concatByteArrays(DPV, DPV2), adp.value()); + verify(client, never()).put(any(PutRequest.class)); + + // duplicates + kv = new KeyValue(ROW_KEY, CF, + AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(DPQ_MS, DPV, DPQ_S, DPV2, DPQ_S, DPV2)); + adp = new AppendDataPoints(); + cells = adp.parseKeyValue(tsdb, kv); + assertEquals(2, cells.size()); + iterator = cells.iterator(); + cell = iterator.next(); + assertArrayEquals(DPV, cell.value); + assertEquals(1356998400128L, cell.timestamp(1356998400)); + cell = iterator.next(); + assertArrayEquals(DPV2, cell.value); + assertEquals(1356998402000L, cell.timestamp(1356998400)); + assertArrayEquals(MockBase.concatByteArrays(DPQ_MS, DPQ_S), adp.qualifier()); + assertArrayEquals(MockBase.concatByteArrays(DPV, DPV2), adp.value()); + verify(client, never()).put(any(PutRequest.class)); + + // duplicates AND out of order, just a mess + kv = new KeyValue(ROW_KEY, CF, + AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays( + DPQ_S, DPV2, DPQ_MS, DPV, DPQ_MS, DPV, DPQ_S, DPV2, DPQ_MS, DPV)); + adp = new AppendDataPoints(); + cells = adp.parseKeyValue(tsdb, kv); + assertEquals(2, cells.size()); + iterator = cells.iterator(); + cell = iterator.next(); + assertArrayEquals(DPV, cell.value); + assertEquals(1356998400128L, cell.timestamp(1356998400)); + cell = iterator.next(); + assertArrayEquals(DPV2, cell.value); + assertEquals(1356998402000L, cell.timestamp(1356998400)); + assertArrayEquals(MockBase.concatByteArrays(DPQ_MS, DPQ_S), adp.qualifier()); + assertArrayEquals(MockBase.concatByteArrays(DPV, DPV2), adp.value()); + verify(client, never()).put(any(PutRequest.class)); + } + + @Test + public void parseKeyValueNotAppends() throws Exception { + // regular data points + try { + KeyValue kv = new KeyValue(ROW_KEY, CF, DPQ_S, DPV); + new AppendDataPoints().parseKeyValue(tsdb, kv); + fail("Expected an IllegalArgumentException"); + } catch (IllegalArgumentException iae) { } + + try { + KeyValue kv = new KeyValue(ROW_KEY, CF, DPQ_MS, DPV); + new AppendDataPoints().parseKeyValue(tsdb, kv); + fail("Expected an IllegalArgumentException"); + } catch (IllegalArgumentException iae) { } + + // different object + try { + KeyValue kv = new KeyValue(ROW_KEY, CF, new byte[] { 1, 0, 0 }, DPV); + new AppendDataPoints().parseKeyValue(tsdb, kv); + fail("Expected an IllegalArgumentException"); + } catch (IllegalArgumentException iae) { } + + // bad coder! + try { + new AppendDataPoints().parseKeyValue(tsdb, null); + fail("Expected an NullPointerException"); + } catch (NullPointerException iae) { } + } + + @Test + public void parseKeyValueCorrupt() throws Exception { + // shouldn't happen, but who knows? + try { + KeyValue kv = new KeyValue(ROW_KEY, CF, null, DPV); + new AppendDataPoints().parseKeyValue(tsdb, kv); + fail("Expected an NullPointerException"); + } catch (NullPointerException iae) { } + + try { + KeyValue kv = new KeyValue(ROW_KEY, CF, HBaseClient.EMPTY_ARRAY, DPV); + new AppendDataPoints().parseKeyValue(tsdb, kv); + fail("Expected an IllegalArgumentException"); + } catch (IllegalArgumentException iae) { } + + try { + KeyValue kv = new KeyValue(ROW_KEY, CF, DPQ_S, null); + new AppendDataPoints().parseKeyValue(tsdb, kv); + fail("Expected an NullPointerException"); + } catch (NullPointerException iae) { } + + try { + KeyValue kv = new KeyValue(ROW_KEY, CF, DPQ_S, HBaseClient.EMPTY_ARRAY); + new AppendDataPoints().parseKeyValue(tsdb, kv); + fail("Expected an IllegalArgumentException"); + } catch (IllegalArgumentException iae) { } + + try { + KeyValue kv = new KeyValue(null, CF, DPQ_S, DPV); + new AppendDataPoints().parseKeyValue(tsdb, kv); + fail("Expected an NullPointerException"); + } catch (NullPointerException iae) { } + + try { + KeyValue kv = new KeyValue(METRIC_BYTES, CF, + AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(DPQ_S, DPV2)); + new AppendDataPoints().parseKeyValue(tsdb, kv); + fail("Expected an IllegalDataException"); + } catch (IllegalDataException iae) { } + + // bad values + try { + KeyValue kv = new KeyValue(ROW_KEY, CF, + AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(DPQ_MS, /*DPV, oops*/ DPQ_S, DPV2)); + new AppendDataPoints().parseKeyValue(tsdb, kv); + fail("Expected an IllegalDataException"); + } catch (IllegalDataException iae) { } + + try { + KeyValue kv = new KeyValue(ROW_KEY, CF, + AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(DPQ_MS, DPV2, new byte[] { 0, 0, 0, 0, })); + new AppendDataPoints().parseKeyValue(tsdb, kv); + fail("Expected an IllegalDataException"); + } catch (IllegalDataException iae) { } + + try { + KeyValue kv = new KeyValue(ROW_KEY, CF, + AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(DPV2, DPQ_MS)); + new AppendDataPoints().parseKeyValue(tsdb, kv); + fail("Expected an IllegalDataException"); + } catch (IllegalDataException iae) { } + } + + @Test + public void repairDuplicates() throws Exception { + setDataPointStorage(); + Whitebox.setInternalState(config, "repair_appends", true); + final KeyValue kv = new KeyValue(ROW_KEY, CF, + AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(DPQ_MS, DPV, DPQ_S, DPV2, DPQ_S, DPV2)); + final AppendDataPoints adp = new AppendDataPoints(); + final Collection<Cell>cells = adp.parseKeyValue(tsdb, kv); + assertEquals(2, cells.size()); + final Iterator<Cell> iterator = cells.iterator(); + Cell cell = iterator.next(); + assertArrayEquals(DPV, cell.value); + assertEquals(1356998400128L, cell.timestamp(1356998400)); + cell = iterator.next(); + assertArrayEquals(DPV2, cell.value); + assertEquals(1356998402000L, cell.timestamp(1356998400)); + assertArrayEquals(MockBase.concatByteArrays(DPQ_MS, DPQ_S), adp.qualifier()); + assertArrayEquals(MockBase.concatByteArrays(DPV, DPV2), adp.value()); + verify(client, times(1)).put(any(PutRequest.class)); + + adp.repairedDeferred().join(); + assertArrayEquals( + MockBase.concatByteArrays(DPQ_MS, DPV, DPQ_S, DPV2), + storage.getColumn(ROW_KEY, AppendDataPoints.APPEND_COLUMN_QUALIFIER)); + } + + @Test + public void repairOutOfOrder() throws Exception { + setDataPointStorage(); + Whitebox.setInternalState(config, "repair_appends", true); + final KeyValue kv = new KeyValue(ROW_KEY, CF, + AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(DPQ_S, DPV2, DPQ_MS, DPV)); + final AppendDataPoints adp = new AppendDataPoints(); + final Collection<Cell>cells = adp.parseKeyValue(tsdb, kv); + assertEquals(2, cells.size()); + final Iterator<Cell> iterator = cells.iterator(); + Cell cell = iterator.next(); + assertArrayEquals(DPV, cell.value); + assertEquals(1356998400128L, cell.timestamp(1356998400)); + cell = iterator.next(); + assertArrayEquals(DPV2, cell.value); + assertEquals(1356998402000L, cell.timestamp(1356998400)); + assertArrayEquals(MockBase.concatByteArrays(DPQ_MS, DPQ_S), adp.qualifier()); + assertArrayEquals(MockBase.concatByteArrays(DPV, DPV2), adp.value()); + verify(client, times(1)).put(any(PutRequest.class)); + + adp.repairedDeferred().join(); + assertArrayEquals( + MockBase.concatByteArrays(DPQ_MS, DPV, DPQ_S, DPV2), + storage.getColumn(ROW_KEY, AppendDataPoints.APPEND_COLUMN_QUALIFIER)); + } + + @Test + public void repairOutOfOrderAndDuplicates() throws Exception { + setDataPointStorage(); + Whitebox.setInternalState(config, "repair_appends", true); + final KeyValue kv = new KeyValue(ROW_KEY, CF, + AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays( + DPQ_S, DPV2, DPQ_MS, DPV, DPQ_MS, DPV, DPQ_S, DPV2, DPQ_MS, DPV)); + final AppendDataPoints adp = new AppendDataPoints(); + final Collection<Cell>cells = adp.parseKeyValue(tsdb, kv); + assertEquals(2, cells.size()); + final Iterator<Cell> iterator = cells.iterator(); + Cell cell = iterator.next(); + assertArrayEquals(DPV, cell.value); + assertEquals(1356998400128L, cell.timestamp(1356998400)); + cell = iterator.next(); + assertArrayEquals(DPV2, cell.value); + assertEquals(1356998402000L, cell.timestamp(1356998400)); + assertArrayEquals(MockBase.concatByteArrays(DPQ_MS, DPQ_S), adp.qualifier()); + assertArrayEquals(MockBase.concatByteArrays(DPV, DPV2), adp.value()); + verify(client, times(1)).put(any(PutRequest.class)); + + adp.repairedDeferred().join(); + assertArrayEquals( + MockBase.concatByteArrays(DPQ_MS, DPV, DPQ_S, DPV2), + storage.getColumn(ROW_KEY, AppendDataPoints.APPEND_COLUMN_QUALIFIER)); + } +} diff --git a/test/core/TestBatchedDataPoints.java b/test/core/TestBatchedDataPoints.java new file mode 100644 index 0000000000..517fd86aae --- /dev/null +++ b/test/core/TestBatchedDataPoints.java @@ -0,0 +1,188 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2014 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.when; +import static org.powermock.api.mockito.PowerMockito.mock; + +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; + +import net.opentsdb.uid.UniqueId; +import net.opentsdb.utils.Config; + +import org.hbase.async.HBaseClient; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.stumbleupon.async.Deferred; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({ TSDB.class, Config.class, UniqueId.class, HBaseClient.class, + IncomingDataPoints.class }) +public class TestBatchedDataPoints { + private static Config config; + private static TSDB tsdb = null; + private HBaseClient client = mock(HBaseClient.class); + private UniqueId metrics = mock(UniqueId.class); + private UniqueId tag_names = mock(UniqueId.class); + private UniqueId tag_values = mock(UniqueId.class); + private BatchedDataPoints bdp = null; + + @SuppressWarnings("unchecked") + @Before + public void before() throws Exception { + config = new Config(false); + tsdb = new TSDB(client, config); + + // replace the "real" field objects with mocks + Field met = tsdb.getClass().getDeclaredField("metrics"); + met.setAccessible(true); + met.set(tsdb, metrics); + + Field tagk = tsdb.getClass().getDeclaredField("tag_names"); + tagk.setAccessible(true); + tagk.set(tsdb, tag_names); + + Field tagv = tsdb.getClass().getDeclaredField("tag_values"); + tagv.setAccessible(true); + tagv.set(tsdb, tag_values); + + when(metrics.width()).thenReturn((short) 3); + when(tag_names.width()).thenReturn((short) 3); + when(tag_values.width()).thenReturn((short) 3); + + when(metrics.getId("foo")).thenReturn(new byte[] { 0, 0, 1 }); + when(metrics.getNameAsync(new byte[] { 0, 0, 1 })).thenReturn( + Deferred.fromResult("foo")); + + PowerMockito.mockStatic(IncomingDataPoints.class); + final byte[] row = new byte[] { 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1 }; + PowerMockito.doAnswer(new Answer<byte[]>() { + public byte[] answer(final InvocationOnMock unused) throws Exception { + return row; + } + }).when(IncomingDataPoints.class, "rowKeyTemplate", (TSDB) any(), + anyString(), (Map<String, String>) any()); + + Map<String, String> tags = new HashMap<String, String>(); + tags.put("host", "web01"); + bdp = new BatchedDataPoints(tsdb, "foo", tags); + } + + @Test + public void timestamp() { + bdp.addPoint(1388534400L, 1); + bdp.addPoint(1388534400500L, 2); + bdp.addPoint(1388534400750L, 2); + + assertEquals(1388534400000L, bdp.timestamp(0)); + assertEquals(1388534400500L, bdp.timestamp(1)); + assertEquals(1388534400750L, bdp.timestamp(2)); + } + + @Test + public void isInteger() { + bdp.addPoint(1388534400L, 1); + bdp.addPoint(1388534400500L, Short.MIN_VALUE); + bdp.addPoint(1388534401L, Integer.MIN_VALUE); + bdp.addPoint(1388534401750L, 2.0f); + + assertTrue(bdp.isInteger(0)); + assertTrue(bdp.isInteger(1)); + assertTrue(bdp.isInteger(2)); + assertFalse(bdp.isInteger(3)); + } + + @Test + public void longValue() { + bdp.addPoint(1388534400L, 1); + bdp.addPoint(1388534400500L, Short.MIN_VALUE); + bdp.addPoint(1388534401L, Integer.MIN_VALUE); + bdp.addPoint(1388534401750L, 2.0f); + + assertEquals(1, bdp.longValue(0)); + assertEquals(Short.MIN_VALUE, bdp.longValue(1)); + assertEquals(Integer.MIN_VALUE, bdp.longValue(2)); + } + + @Test + public void doubleValue() { + bdp.addPoint(1388534400L, 1); + bdp.addPoint(1388534400500L, Short.MIN_VALUE); + bdp.addPoint(1388534401L, Integer.MIN_VALUE); + bdp.addPoint(1388534401750L, 2.0f); + + Assert.assertEquals(2.0f, bdp.doubleValue(3), 0.00001f); + } + + @Test + public void inBounds() { + long timestamp = 1388534400L; + long base_time = (timestamp - (timestamp % Const.MAX_TIMESPAN)); + + // test that the limit values are fine + bdp.addPoint(base_time, 2); + bdp.addPoint(base_time + Const.MAX_TIMESPAN - 1, 3); + } + + @Test(expected = IllegalArgumentException.class) + public void addPrevTimestamp() { + bdp.addPoint(1388534400L, 1); + bdp.addPoint(1388534350L, 2); + } + + @Test(expected = IllegalDataException.class) + public void outOfBounds() { + long timestamp = 1388534400L; + long base_time = (timestamp - (timestamp % Const.MAX_TIMESPAN)); + + bdp.addPoint(timestamp, 1); // this set's the batch's baseTime + + // outside the limits addPoint throws an IllegalArgumentException + bdp.addPoint(base_time + Const.MAX_TIMESPAN, 2); + } + + @Test + public void fullLoad() { + long timestamp = 1388534400L; + long base_time = (timestamp - (timestamp % Const.MAX_TIMESPAN)); + + for (int i = 0; i < Const.MAX_TIMESPAN; i++) { + bdp.addPoint(base_time + i, i); + } + } + + @Test + public void fullLoadMs() { + long timestamp = 1388534400L; + long base_time = (timestamp - (timestamp % Const.MAX_TIMESPAN)); + + for (long i = 0; i < Const.MAX_TIMESPAN * 1000; i++) { + bdp.addPoint(base_time * 1000 + i, i); + } + } +} diff --git a/test/core/TestCompactionQueue.java b/test/core/TestCompactionQueue.java index d1c1a0e84e..6080802e37 100644 --- a/test/core/TestCompactionQueue.java +++ b/test/core/TestCompactionQueue.java @@ -23,7 +23,9 @@ import java.util.Arrays; import java.util.HashSet; +import java.util.Random; import java.util.Set; + import org.hbase.async.Bytes; import org.hbase.async.KeyValue; @@ -47,6 +49,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.anyLong; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; @@ -72,9 +75,9 @@ public final class TestCompactionQueue { private static final byte[] FAMILY = { 't' }; private static final byte[] ZERO = { 0 }; private static final byte[] MIXED_FLAG = { Const.MS_MIXED_COMPACT }; - private static final String annotation = - "{\"tsuid\":\"ABCD\",\"description\":\"Description\"," + - "\"notes\":\"Notes\",\"custom\":null,\"endTime\":1328140801,\"startTime" + + private static final String annotation = + "{\"tsuid\":\"ABCD\",\"description\":\"Description\"," + + "\"notes\":\"Notes\",\"custom\":null,\"endTime\":1328140801,\"startTime" + "\":1328140800}"; private static final byte[] note = annotation.getBytes(Charset.forName("UTF-8")); private static final byte[] note_qual = { 1, 0, 0 }; @@ -88,6 +91,7 @@ public void before() throws Exception { Whitebox.setInternalState(config, "enable_compactions", true); Whitebox.setInternalState(config, "fix_duplicates", true); Whitebox.setInternalState(tsdb, "config", config); + when(tsdb.getConfig()).thenReturn(config); // Stub out the compaction thread, so it doesn't even start. PowerMockito.whenNew(CompactionQueue.Thrd.class).withNoArguments() .thenReturn(mock(CompactionQueue.Thrd.class)); @@ -95,22 +99,47 @@ public void before() throws Exception { PowerMockito.when(config.fix_duplicates()).thenReturn(true); compactionq = new CompactionQueue(tsdb); - when(tsdb.put(anyBytes(), anyBytes(), anyBytes())) + when(tsdb.put(anyBytes(), anyBytes(), anyBytes(), anyLong())) .thenAnswer(newDeferred()); when(tsdb.delete(anyBytes(), any(byte[][].class))) .thenAnswer(newDeferred()); } + @Test + public void useMaxTsWhileCompacting() throws Exception { + Random rnd = new Random(); + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(2); + ArrayList<Annotation> annotations = new ArrayList<Annotation>(0); + long ts1 = Math.abs(rnd.nextLong()); + final byte[] qual1 = { (byte) 0xF0, 0x00, 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + kvs.add(makekvWithTs(qual1, ts1, val1)); + long ts2 = Math.abs(rnd.nextLong()); + final byte[] qual2 = { (byte) 0xF0, 0x00, 0x01, 0x07 }; + final byte[] val2 = Bytes.fromLong(5L); + kvs.add(makekvWithTs(qual2, ts2, val2)); + long ts3 = Math.abs(rnd.nextLong()); + final byte[] qual3 = { (byte) 0xF0, 0x00, 0x02, 0x07 }; + final byte[] val3 = Bytes.fromLong(2L); + kvs.add(makekvWithTs(qual3, ts3, val3)); + + when(tsdb.getConfig().getBoolean("tsd.storage.use_otsdb_timestamp")).thenReturn(true); + final KeyValue kv = compactionq.compact(kvs, annotations, null); + assertArrayEquals(MockBase.concatByteArrays(qual1, qual2, qual3), kv.qualifier()); + assertArrayEquals(MockBase.concatByteArrays(val1, val2, val3, ZERO), kv.value()); + assert(kv.timestamp() == Math.max(ts1, Math.max(ts2, ts3))); + } + @Test public void emptyRow() throws Exception { ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(0); ArrayList<Annotation> annotations = new ArrayList<Annotation>(0); - final KeyValue kv = compactionq.compact(kvs, annotations); + final KeyValue kv = compactionq.compact(kvs, annotations, null); assertNull(kv); - + // We had nothing to do so... // ... verify there were no put. - verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes()); + verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes(), anyLong()); // ... verify there were no delete. verify(tsdb, never()).delete(anyBytes(), any(byte[][].class)); } @@ -122,17 +151,36 @@ public void oneCellRow() throws Exception { final byte[] qual = { 0x00, 0x07 }; final byte[] val = Bytes.fromLong(42L); kvs.add(makekv(qual, val)); - final KeyValue kv = compactionq.compact(kvs, annotations); + final KeyValue kv = compactionq.compact(kvs, annotations, null); assertArrayEquals(qual, kv.qualifier()); assertArrayEquals(val, kv.value()); - + // We had nothing to do so... // ... verify there were no put. - verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes()); + verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes(), anyLong()); // ... verify there were no delete. verify(tsdb, never()).delete(anyBytes(), any(byte[][].class)); } - + + @Test + public void oneCellAppend() throws Exception { + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); + ArrayList<Annotation> annotations = new ArrayList<Annotation>(0); + final byte[] qual = { 0x00, 0x07 }; + final byte[] val = Bytes.fromLong(42L); + kvs.add(makekv(AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(qual, val))); + final KeyValue kv = compactionq.compact(kvs, annotations, null); + assertArrayEquals(qual, kv.qualifier()); + assertArrayEquals(val, kv.value()); + + // We had nothing to do so... + // ... verify there were no put. + verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes(), anyLong()); + // ... verify there were no delete. + verify(tsdb, never()).delete(anyBytes(), any(byte[][].class)); + } + @Test public void oneCellRowWAnnotation() throws Exception { ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); @@ -141,18 +189,39 @@ public void oneCellRowWAnnotation() throws Exception { final byte[] qual = { 0x00, 0x07 }; final byte[] val = Bytes.fromLong(42L); kvs.add(makekv(qual, val)); - final KeyValue kv = compactionq.compact(kvs, annotations); + final KeyValue kv = compactionq.compact(kvs, annotations, null); assertArrayEquals(qual, kv.qualifier()); assertArrayEquals(val, kv.value()); assertEquals(1, annotations.size()); - + // We had nothing to do so... // ... verify there were no put. - verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes()); + verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes(), anyLong()); // ... verify there were no delete. verify(tsdb, never()).delete(anyBytes(), any(byte[][].class)); } - + + @Test + public void oneCellAppendWAnnotiation() throws Exception { + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); + ArrayList<Annotation> annotations = new ArrayList<Annotation>(1); + kvs.add(makekv(note_qual, note)); + final byte[] qual = { 0x00, 0x07 }; + final byte[] val = Bytes.fromLong(42L); + kvs.add(makekv(AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(qual, val))); + final KeyValue kv = compactionq.compact(kvs, annotations, null); + assertArrayEquals(qual, kv.qualifier()); + assertArrayEquals(val, kv.value()); + assertEquals(1, annotations.size()); + + // We had nothing to do so... + // ... verify there were no put. + verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes(), anyLong()); + // ... verify there were no delete. + verify(tsdb, never()).delete(anyBytes(), any(byte[][].class)); + } + @Test public void oneCellRowWAnnotationMS() throws Exception { ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); @@ -161,14 +230,14 @@ public void oneCellRowWAnnotationMS() throws Exception { final byte[] qual = { 0x00, 0x07 }; final byte[] val = Bytes.fromLong(42L); kvs.add(makekv(qual, val)); - final KeyValue kv = compactionq.compact(kvs, annotations); + final KeyValue kv = compactionq.compact(kvs, annotations, null); assertArrayEquals(qual, kv.qualifier()); assertArrayEquals(val, kv.value()); assertEquals(1, annotations.size()); - + // We had nothing to do so... // ... verify there were no put. - verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes()); + verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes(), anyLong()); // ... verify there were no delete. verify(tsdb, never()).delete(anyBytes(), any(byte[][].class)); } @@ -181,12 +250,12 @@ public void oneCellRowBadLength() throws Exception { final byte[] cqual = { 0x00, 0x07 }; byte[] val = Bytes.fromLong(42L); kvs.add(makekv(qual, val)); - final KeyValue kv = compactionq.compact(kvs, annotations); + final KeyValue kv = compactionq.compact(kvs, annotations, null); assertArrayEquals(cqual, kv.qualifier()); assertArrayEquals(val, kv.value()); // The old one needed the length fixed up, so verify that we wrote the new one - verify(tsdb, times(1)).put(KEY, cqual, val); + verify(tsdb, times(1)).put(KEY, cqual, val, kvCount - 1); // ... and deleted the old one verify(tsdb, times(1)).delete(KEY, new byte[][] { qual }); } @@ -198,13 +267,13 @@ public void oneCellRowMS() throws Exception { final byte[] qual = { (byte) 0xF0, 0x00, 0x00, 0x07 }; byte[] val = Bytes.fromLong(42L); kvs.add(makekv(qual, val)); - final KeyValue kv = compactionq.compact(kvs, annotations); + final KeyValue kv = compactionq.compact(kvs, annotations, null); assertArrayEquals(qual, kv.qualifier()); assertArrayEquals(val, kv.value()); - + // We had nothing to do so... // ... verify there were no put. - verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes()); + verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes(), anyLong()); // ... verify there were no delete. verify(tsdb, never()).delete(anyBytes(), any(byte[][].class)); } @@ -220,17 +289,38 @@ public void twoCellRow() throws Exception { final byte[] val2 = Bytes.fromLong(5L); kvs.add(makekv(qual2, val2)); - final KeyValue kv = compactionq.compact(kvs, annotations); + final KeyValue kv = compactionq.compact(kvs, annotations, null); assertArrayEquals(MockBase.concatByteArrays(qual1, qual2), kv.qualifier()); assertArrayEquals(MockBase.concatByteArrays(val1, val2, ZERO), kv.value()); - + // We had one row to compact, so one put to do. verify(tsdb, times(1)).put(KEY, MockBase.concatByteArrays(qual1, qual2), - MockBase.concatByteArrays(val1, val2, ZERO)); + MockBase.concatByteArrays(val1, val2, ZERO), kvCount - 1); // And we had to delete individual cells. verify(tsdb, times(1)).delete(eq(KEY), eqAnyOrder(new byte[][] { qual1, qual2 })); } - + + @Test + public void twoCellAppend() throws Exception { + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); + ArrayList<Annotation> annotations = new ArrayList<Annotation>(0); + final byte[] qual = { 0x00, 0x07 }; + final byte[] val = Bytes.fromLong(42L); + final byte[] qual2 = { 0x00, 0x17 }; + final byte[] val2 = Bytes.fromLong(5L); + kvs.add(makekv(AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(qual, val, qual2, val2))); + final KeyValue kv = compactionq.compact(kvs, annotations, null); + assertArrayEquals(MockBase.concatByteArrays(qual, qual2), kv.qualifier()); + assertArrayEquals(MockBase.concatByteArrays(val, val2, ZERO), kv.value()); + + // We had nothing to do so... + // ... verify there were no put. + verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes(), anyLong()); + // ... verify there were no delete. + verify(tsdb, never()).delete(anyBytes(), any(byte[][].class)); + } + @Test public void twoCellRowWAnnotation() throws Exception { ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(2); @@ -243,45 +333,68 @@ public void twoCellRowWAnnotation() throws Exception { final byte[] val2 = Bytes.fromLong(5L); kvs.add(makekv(qual2, val2)); - final KeyValue kv = compactionq.compact(kvs, annotations); + final KeyValue kv = compactionq.compact(kvs, annotations, null); assertArrayEquals(MockBase.concatByteArrays(qual1, qual2), kv.qualifier()); assertArrayEquals(MockBase.concatByteArrays(val1, val2, ZERO), kv.value()); assertEquals(1, annotations.size()); - + // We had one row to compact, so one put to do. verify(tsdb, times(1)).put(KEY, MockBase.concatByteArrays(qual1, qual2), - MockBase.concatByteArrays(val1, val2, ZERO)); + MockBase.concatByteArrays(val1, val2, ZERO), kvCount - 1); // And we had to delete individual cells. verify(tsdb, times(1)).delete(eq(KEY), eqAnyOrder(new byte[][] { qual1, qual2 })); } + @Test + public void twoCellAppendWAnnotations() throws Exception { + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); + ArrayList<Annotation> annotations = new ArrayList<Annotation>(1); + kvs.add(makekv(note_qual, note)); + final byte[] qual = { 0x00, 0x07 }; + final byte[] val = Bytes.fromLong(42L); + final byte[] qual2 = { 0x00, 0x17 }; + final byte[] val2 = Bytes.fromLong(5L); + kvs.add(makekv(AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(qual, val, qual2, val2))); + final KeyValue kv = compactionq.compact(kvs, annotations, null); + assertArrayEquals(MockBase.concatByteArrays(qual, qual2), kv.qualifier()); + assertArrayEquals(MockBase.concatByteArrays(val, val2, ZERO), kv.value()); + assertEquals(1, annotations.size()); + + // We had nothing to do so... + // ... verify there were no put. + verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes(), anyLong()); + // ... verify there were no delete. + verify(tsdb, never()).delete(anyBytes(), any(byte[][].class)); + } + @Test public void fullRowSeconds() throws Exception { ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(3600); ArrayList<Annotation> annotations = new ArrayList<Annotation>(0); - + byte[] qualifiers = new byte[] {}; byte[] values = new byte[] {}; - + for (int i = 0; i < 3600; i++) { final short qualifier = (short) (i << Const.FLAG_BITS | 0x07); kvs.add(makekv(Bytes.fromShort(qualifier), Bytes.fromLong(i))); - qualifiers = MockBase.concatByteArrays(qualifiers, + qualifiers = MockBase.concatByteArrays(qualifiers, Bytes.fromShort(qualifier)); values = MockBase.concatByteArrays(values, Bytes.fromLong(i)); } - final KeyValue kv = compactionq.compact(kvs, annotations); + final KeyValue kv = compactionq.compact(kvs, annotations, null); assertArrayEquals(MockBase.concatByteArrays(qualifiers), kv.qualifier()); assertArrayEquals(MockBase.concatByteArrays(values, ZERO), kv.value()); - + // We had one row to compact, so one put to do. verify(tsdb, times(1)).put(KEY, qualifiers, - MockBase.concatByteArrays(values, ZERO)); + MockBase.concatByteArrays(values, ZERO), kvCount - 1); // And we had to delete individual cells. verify(tsdb, times(1)).delete((byte[])any(), (byte[][])any()); } - + @Test public void bigRowMs() throws Exception { ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(3599999); @@ -292,22 +405,22 @@ public void bigRowMs() throws Exception { for (int i = 0; i < 3599999; i++) { final int qualifier = (((i << Const.MS_FLAG_BITS ) | 0x07) | 0xF0000000); kvs.add(makekv(Bytes.fromInt(qualifier), Bytes.fromLong(i))); - qualifiers = MockBase.concatByteArrays(qualifiers, + qualifiers = MockBase.concatByteArrays(qualifiers, Bytes.fromInt(qualifier)); values = MockBase.concatByteArrays(values, Bytes.fromLong(i)); i += 100; } - final KeyValue kv = compactionq.compact(kvs, annotations); + final KeyValue kv = compactionq.compact(kvs, annotations, null); assertArrayEquals(MockBase.concatByteArrays(qualifiers), kv.qualifier()); assertArrayEquals(MockBase.concatByteArrays(values, ZERO), kv.value()); - + // We had one row to compact, so one put to do. verify(tsdb, times(1)).put(KEY, qualifiers, - MockBase.concatByteArrays(values, ZERO)); + MockBase.concatByteArrays(values, ZERO), kvCount - 1); // And we had to delete individual cells. verify(tsdb, times(1)).delete((byte[])any(), (byte[][])any()); } - + @Test public void twoCellRowMS() throws Exception { ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(2); @@ -319,17 +432,17 @@ public void twoCellRowMS() throws Exception { final byte[] val2 = Bytes.fromLong(5L); kvs.add(makekv(qual2, val2)); - final KeyValue kv = compactionq.compact(kvs, annotations); + final KeyValue kv = compactionq.compact(kvs, annotations, null); assertArrayEquals(MockBase.concatByteArrays(qual1, qual2), kv.qualifier()); assertArrayEquals(MockBase.concatByteArrays(val1, val2, ZERO), kv.value()); - + // We had one row to compact, so one put to do. verify(tsdb, times(1)).put(KEY, MockBase.concatByteArrays(qual1, qual2), - MockBase.concatByteArrays(val1, val2, ZERO)); + MockBase.concatByteArrays(val1, val2, ZERO), kvCount - 1); // And we had to delete individual cells. verify(tsdb, times(1)).delete(eq(KEY), eqAnyOrder(new byte[][] { qual1, qual2 })); } - + @Test public void sortMsAndS() throws Exception { ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(2); @@ -343,21 +456,21 @@ public void sortMsAndS() throws Exception { final byte[] qual3 = { (byte) 0xF0, 0x00, 0x01, 0x07 }; final byte[] val3 = Bytes.fromLong(5L); kvs.add(makekv(qual3, val3)); - - final KeyValue kv = compactionq.compact(kvs, annotations); - assertArrayEquals(MockBase.concatByteArrays(qual1, qual3, qual2), + + final KeyValue kv = compactionq.compact(kvs, annotations, null); + assertArrayEquals(MockBase.concatByteArrays(qual1, qual3, qual2), kv.qualifier()); - assertArrayEquals(MockBase.concatByteArrays(val1, val3, val2, + assertArrayEquals(MockBase.concatByteArrays(val1, val3, val2, new byte[] { 1 }), kv.value()); // We had one row to compact, so one put to do. verify(tsdb, times(1)).put(KEY, MockBase.concatByteArrays(qual1, qual3, qual2), - MockBase.concatByteArrays(val1, val3, val2, new byte[] { 1 })); + MockBase.concatByteArrays(val1, val3, val2, new byte[] { 1 }), kvCount - 1); // And we had to delete individual cells. - verify(tsdb, times(1)).delete(eq(KEY), eqAnyOrder(new byte[][] { qual1, + verify(tsdb, times(1)).delete(eq(KEY), eqAnyOrder(new byte[][] { qual1, qual2, qual3 })); } - + @Test public void secondsOutOfOrder() throws Exception { ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(3); @@ -372,23 +485,23 @@ public void secondsOutOfOrder() throws Exception { final byte[] val3 = Bytes.fromLong(6L); kvs.add(makekv(qual3, val3)); - final KeyValue kv = compactionq.compact(kvs, annotations); - assertArrayEquals(MockBase.concatByteArrays(qual2, qual3, qual1), + final KeyValue kv = compactionq.compact(kvs, annotations, null); + assertArrayEquals(MockBase.concatByteArrays(qual2, qual3, qual1), kv.qualifier()); - assertArrayEquals(MockBase.concatByteArrays(val2, val3, val1, ZERO), + assertArrayEquals(MockBase.concatByteArrays(val2, val3, val1, ZERO), kv.value()); // We compacted all columns to one, so one put to do. verify(tsdb, times(1)).put(KEY, MockBase.concatByteArrays(qual2, qual3, qual1), - MockBase.concatByteArrays(val2, val3, val1, ZERO)); + MockBase.concatByteArrays(val2, val3, val1, ZERO), kvCount - 1); // And we had to delete individual cells. - verify(tsdb, times(1)).delete(eq(KEY), eqAnyOrder(new byte[][] { qual1, + verify(tsdb, times(1)).delete(eq(KEY), eqAnyOrder(new byte[][] { qual1, qual2, qual3 })); } - + @Test public void msOutOfOrder() throws Exception { - // all rows with an ms qualifier will go through the compaction + // all rows with an ms qualifier will go through the compaction // process and they'll be sorted ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(3); ArrayList<Annotation> annotations = new ArrayList<Annotation>(0); @@ -402,19 +515,19 @@ public void msOutOfOrder() throws Exception { final byte[] val3 = Bytes.fromLong(6L); kvs.add(makekv(qual3, val3)); - final KeyValue kv = compactionq.compact(kvs, annotations); - assertArrayEquals(MockBase.concatByteArrays(qual2, qual3, qual1), + final KeyValue kv = compactionq.compact(kvs, annotations, null); + assertArrayEquals(MockBase.concatByteArrays(qual2, qual3, qual1), kv.qualifier()); - assertArrayEquals(MockBase.concatByteArrays(val2, val3, val1, ZERO), + assertArrayEquals(MockBase.concatByteArrays(val2, val3, val1, ZERO), kv.value()); // We had one row to compact, so one put to do. verify(tsdb, times(1)).put(KEY, MockBase.concatByteArrays(qual2, qual3, qual1), - MockBase.concatByteArrays(val2, val3, val1, ZERO)); + MockBase.concatByteArrays(val2, val3, val1, ZERO), kvCount - 1); // And we had to delete individual cells. verify(tsdb, times(1)).delete(eq(KEY), eqAnyOrder(new byte[][] { qual1, qual2, qual3 })); } - + @Test public void secondAndMs() throws Exception { ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(2); @@ -426,18 +539,18 @@ public void secondAndMs() throws Exception { final byte[] val2 = Bytes.fromLong(5L); kvs.add(makekv(qual2, val2)); - final KeyValue kv = compactionq.compact(kvs, annotations); + final KeyValue kv = compactionq.compact(kvs, annotations, null); assertArrayEquals(MockBase.concatByteArrays(qual1, qual2), kv.qualifier()); - assertArrayEquals(MockBase.concatByteArrays(val1, val2, new byte[] { 1 }), + assertArrayEquals(MockBase.concatByteArrays(val1, val2, new byte[] { 1 }), kv.value()); // We had one row to compact, so one put to do. verify(tsdb, times(1)).put(KEY, MockBase.concatByteArrays(qual1, qual2), - MockBase.concatByteArrays(val1, val2, new byte[] { 1 })); + MockBase.concatByteArrays(val1, val2, new byte[] { 1 }), kvCount - 1); // And we had to delete individual cells. verify(tsdb, times(1)).delete(eq(KEY), eqAnyOrder(new byte[][] { qual1, qual2 })); } - + @Test public void secondAndMsWAnnotation() throws Exception { ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(2); @@ -450,15 +563,15 @@ public void secondAndMsWAnnotation() throws Exception { final byte[] val2 = Bytes.fromLong(5L); kvs.add(makekv(qual2, val2)); - final KeyValue kv = compactionq.compact(kvs, annotations); + final KeyValue kv = compactionq.compact(kvs, annotations, null); assertArrayEquals(MockBase.concatByteArrays(qual1, qual2), kv.qualifier()); - assertArrayEquals(MockBase.concatByteArrays(val1, val2, new byte[] { 1 }), + assertArrayEquals(MockBase.concatByteArrays(val1, val2, new byte[] { 1 }), kv.value()); assertEquals(1, annotations.size()); // We had one row to compact, so one put to do. verify(tsdb, times(1)).put(KEY, MockBase.concatByteArrays(qual1, qual2), - MockBase.concatByteArrays(val1, val2, new byte[] { 1 })); + MockBase.concatByteArrays(val1, val2, new byte[] { 1 }), kvCount - 1); // And we had to delete individual cells. verify(tsdb, times(1)).delete(eq(KEY), eqAnyOrder(new byte[][] { qual1, qual2 })); } @@ -475,7 +588,7 @@ public void msSameAsSecond() throws Exception { final byte[] val2 = Bytes.fromLong(5L); kvs.add(makekv(qual2, val2)); - compactionq.compact(kvs, annotations); + compactionq.compact(kvs, annotations, null); } @Test @@ -489,16 +602,16 @@ public void msSameAsSecondFix() throws Exception { final byte[] val2 = Bytes.fromLong(5L); kvs.add(makekv(qual2, val2)); - final KeyValue kv = compactionq.compact(kvs, annotations); + final KeyValue kv = compactionq.compact(kvs, annotations, null); assertArrayEquals(qual2, kv.qualifier()); assertArrayEquals(val2, kv.value()); - + // no compacted row - verify(tsdb, never()).put(KEY, qual2, val2); + verify(tsdb, never()).put(KEY, qual2, val2, kvCount - 1); // And we had to delete the earlier entry. verify(tsdb, times(1)).delete(KEY, new byte[][] {qual1}); } - + @Test public void fixQualifierFlags() throws Exception { ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(2); @@ -513,13 +626,13 @@ public void fixQualifierFlags() throws Exception { final byte[] val2 = Bytes.fromLong(5L); kvs.add(makekv(qual2, val2)); - final KeyValue kv = compactionq.compact(kvs, annotations); + final KeyValue kv = compactionq.compact(kvs, annotations, null); assertArrayEquals(MockBase.concatByteArrays(cqual1, qual2), kv.qualifier()); assertArrayEquals(MockBase.concatByteArrays(val1, val2, ZERO), kv.value()); // We had one row to compact, so one put to do. verify(tsdb, times(1)).put(KEY, MockBase.concatByteArrays(cqual1, qual2), - MockBase.concatByteArrays(val1, val2, ZERO)); + MockBase.concatByteArrays(val1, val2, ZERO), kvCount - 1); // And we had to delete individual cells. verify(tsdb, times(1)).delete(eq(KEY), eqAnyOrder(new byte[][] { qual1, qual2 })); } @@ -540,13 +653,13 @@ public void fixFloatingPoint() throws Exception { final byte[] cval2 = Bytes.fromInt(Float.floatToRawIntBits(4.2F)); kvs.add(makekv(qual2, val2)); - final KeyValue kv = compactionq.compact(kvs, annotations); + final KeyValue kv = compactionq.compact(kvs, annotations, null); assertArrayEquals(MockBase.concatByteArrays(qual1, qual2), kv.qualifier()); assertArrayEquals(MockBase.concatByteArrays(val1, cval2, ZERO), kv.value()); - + // We had one row to compact, so one put to do. verify(tsdb, times(1)).put(KEY, MockBase.concatByteArrays(qual1, qual2), - MockBase.concatByteArrays(val1, cval2, ZERO)); + MockBase.concatByteArrays(val1, cval2, ZERO), kvCount - 1); // And we had to delete individual cells. verify(tsdb, times(1)).delete(eq(KEY), eqAnyOrder(new byte[][] { qual1, qual2, })); } @@ -564,9 +677,9 @@ public void overlappingDataPoints() throws Exception { final byte[] val2 = Bytes.fromInt(4); kvs.add(makekv(qual2, val2)); - compactionq.compact(kvs, annotations); + compactionq.compact(kvs, annotations, null); } - + @Test public void overlappingDataPointsFix() throws Exception { ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(2); @@ -579,12 +692,12 @@ public void overlappingDataPointsFix() throws Exception { final byte[] val2 = Bytes.fromInt(4); kvs.add(makekv(qual2, val2)); - final KeyValue kv = compactionq.compact(kvs, annotations); + final KeyValue kv = compactionq.compact(kvs, annotations, null); assertArrayEquals(qual2, kv.qualifier()); assertArrayEquals(val2, kv.value()); // We didn't have anything to write. - verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes()); + verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes(), anyLong()); // We had to delete the first entry as it was older. verify(tsdb, times(1)).delete(KEY, new byte[][] {qual1}); } @@ -606,12 +719,12 @@ public void failedCompactNoop() throws Exception { final byte[] valcompact = MockBase.concatByteArrays(val1, val2, ZERO); kvs.add(makekv(qualcompact, valcompact)); - final KeyValue kv = compactionq.compact(kvs, annotations); + final KeyValue kv = compactionq.compact(kvs, annotations, null); assertArrayEquals(qualcompact, kv.qualifier()); assertArrayEquals(valcompact, kv.value()); - + // We didn't have anything to write. - verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes()); + verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes(), anyLong()); // We had to delete stuff in 1 row. verify(tsdb, times(1)).delete(eq(KEY), eqAnyOrder(new byte[][] { qual1, qual2 })); } @@ -622,16 +735,16 @@ public void annotationOnly() throws Exception { ArrayList<Annotation> annotations = new ArrayList<Annotation>(1); kvs.add(makekv(note_qual, note)); - final KeyValue kv = compactionq.compact(kvs, annotations); + final KeyValue kv = compactionq.compact(kvs, annotations, null); assertNull(kv); assertEquals(1, annotations.size()); // ... verify there were no put. - verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes()); + verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes(), anyLong()); // ... verify there were no delete. verify(tsdb, never()).delete(anyBytes(), any(byte[][].class)); } - + @Test public void annotationsOnly() throws Exception { // Two annotations in a row only (the value of the second isn't parsed at @@ -641,16 +754,16 @@ public void annotationsOnly() throws Exception { kvs.add(makekv(note_qual, note)); kvs.add(makekv(new byte[] { 1, 0, 1 }, note)); - final KeyValue kv = compactionq.compact(kvs, annotations); + final KeyValue kv = compactionq.compact(kvs, annotations, null); assertNull(kv); assertEquals(2, annotations.size()); // ... verify there were no put. - verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes()); + verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes(), anyLong()); // ... verify there were no delete. verify(tsdb, never()).delete(anyBytes(), any(byte[][].class)); } - + @Test public void secondCompact() throws Exception { // In this test the row has already been compacted, and another data @@ -670,19 +783,19 @@ public void secondCompact() throws Exception { final byte[] val3 = Bytes.fromLong(6L); kvs.add(makekv(qual3, val3)); - final KeyValue kv = compactionq.compact(kvs, annotations); - assertArrayEquals(MockBase.concatByteArrays(qual1, qual3, qual2), + final KeyValue kv = compactionq.compact(kvs, annotations, null); + assertArrayEquals(MockBase.concatByteArrays(qual1, qual3, qual2), kv.qualifier()); - assertArrayEquals(MockBase.concatByteArrays(val1, val3, val2, ZERO), + assertArrayEquals(MockBase.concatByteArrays(val1, val3, val2, ZERO), kv.value()); // We had one row to compact, so one put to do. verify(tsdb, times(1)).put(KEY, MockBase.concatByteArrays(qual1, qual3, qual2), - MockBase.concatByteArrays(val1, val3, val2, ZERO)); + MockBase.concatByteArrays(val1, val3, val2, ZERO), kvCount - 1); // And we had to delete the individual cell + pre-existing compacted cell. verify(tsdb, times(1)).delete(eq(KEY), eqAnyOrder(new byte[][] { qual12, qual3 })); } - + @Test public void secondCompactWAnnotation() throws Exception { // In this test the row has already been compacted, and another data @@ -703,16 +816,16 @@ public void secondCompactWAnnotation() throws Exception { final byte[] val3 = Bytes.fromLong(6L); kvs.add(makekv(qual3, val3)); - final KeyValue kv = compactionq.compact(kvs, annotations); - assertArrayEquals(MockBase.concatByteArrays(qual1, qual3, qual2), + final KeyValue kv = compactionq.compact(kvs, annotations, null); + assertArrayEquals(MockBase.concatByteArrays(qual1, qual3, qual2), kv.qualifier()); - assertArrayEquals(MockBase.concatByteArrays(val1, val3, val2, ZERO), + assertArrayEquals(MockBase.concatByteArrays(val1, val3, val2, ZERO), kv.value()); assertEquals(1, annotations.size()); // We had one row to compact, so one put to do. verify(tsdb, times(1)).put(KEY, MockBase.concatByteArrays(qual1, qual3, qual2), - MockBase.concatByteArrays(val1, val3, val2, ZERO)); + MockBase.concatByteArrays(val1, val3, val2, ZERO), kvCount - 1); // And we had to delete the individual cell + pre-existing compacted cell. verify(tsdb, times(1)).delete(eq(KEY), eqAnyOrder(new byte[][] { qual12, qual3 })); } @@ -736,19 +849,19 @@ public void secondCompactMS() throws Exception { final byte[] val3 = Bytes.fromLong(6L); kvs.add(makekv(qual3, val3)); - final KeyValue kv = compactionq.compact(kvs, annotations); - assertArrayEquals(MockBase.concatByteArrays(qual1, qual3, qual2), + final KeyValue kv = compactionq.compact(kvs, annotations, null); + assertArrayEquals(MockBase.concatByteArrays(qual1, qual3, qual2), kv.qualifier()); - assertArrayEquals(MockBase.concatByteArrays(val1, val3, val2, ZERO), + assertArrayEquals(MockBase.concatByteArrays(val1, val3, val2, ZERO), kv.value()); - + // We had one row to compact, so one put to do. verify(tsdb, times(1)).put(KEY, MockBase.concatByteArrays(qual1, qual3, qual2), - MockBase.concatByteArrays(val1, val3, val2, ZERO)); + MockBase.concatByteArrays(val1, val3, val2, ZERO), kvCount - 1); // And we had to delete the individual cell + pre-existing compacted cell. verify(tsdb, times(1)).delete(eq(KEY), eqAnyOrder(new byte[][] { qual12, qual3 })); } - + @Test public void secondCompactMixedSecond() throws Exception { // In this test the row has already been compacted, and another data @@ -761,7 +874,7 @@ public void secondCompactMixedSecond() throws Exception { final byte[] qual2 = { (byte) 0xF0, 0x0A, 0x41, 0x07 }; final byte[] val2 = Bytes.fromLong(5L); final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); - kvs.add(makekv(qual12, MockBase.concatByteArrays(val1, val2, + kvs.add(makekv(qual12, MockBase.concatByteArrays(val1, val2, new byte[] { 1 }))); // This data point came late. Note that its time delta falls in between // that of the two data points above. @@ -769,20 +882,20 @@ public void secondCompactMixedSecond() throws Exception { final byte[] val3 = Bytes.fromLong(6L); kvs.add(makekv(qual3, val3)); - final KeyValue kv = compactionq.compact(kvs, annotations); - assertArrayEquals(MockBase.concatByteArrays(qual1, qual3, qual2), + final KeyValue kv = compactionq.compact(kvs, annotations, null); + assertArrayEquals(MockBase.concatByteArrays(qual1, qual3, qual2), kv.qualifier()); - assertArrayEquals(MockBase.concatByteArrays(val1, val3, val2, + assertArrayEquals(MockBase.concatByteArrays(val1, val3, val2, new byte[] { 1 }), kv.value()); - + // We had one row to compact, so one put to do. verify(tsdb, times(1)).put(KEY, MockBase.concatByteArrays(qual1, qual3, qual2), - MockBase.concatByteArrays(val1, val3, val2, - new byte[] { 1 })); + MockBase.concatByteArrays(val1, val3, val2, + new byte[] { 1 }), kvCount - 1); // And we had to delete the individual cell + pre-existing compacted cell. verify(tsdb, times(1)).delete(eq(KEY), eqAnyOrder(new byte[][] { qual12, qual3 })); } - + @Test public void secondCompactMixedMS() throws Exception { // In this test the row has already been compacted, and another data @@ -795,7 +908,7 @@ public void secondCompactMixedMS() throws Exception { final byte[] qual2 = { (byte) 0xF0, 0x0A, 0x41, 0x07 }; final byte[] val2 = Bytes.fromLong(5L); final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); - kvs.add(makekv(qual12, MockBase.concatByteArrays(val1, val2, + kvs.add(makekv(qual12, MockBase.concatByteArrays(val1, val2, new byte[] { 1 }))); // This data point came late. Note that its time delta falls in between // that of the two data points above. @@ -803,20 +916,20 @@ public void secondCompactMixedMS() throws Exception { final byte[] val3 = Bytes.fromLong(6L); kvs.add(makekv(qual3, val3)); - final KeyValue kv = compactionq.compact(kvs, annotations); - assertArrayEquals(MockBase.concatByteArrays(qual1, qual3, qual2), + final KeyValue kv = compactionq.compact(kvs, annotations, null); + assertArrayEquals(MockBase.concatByteArrays(qual1, qual3, qual2), kv.qualifier()); - assertArrayEquals(MockBase.concatByteArrays(val1, val3, val2, + assertArrayEquals(MockBase.concatByteArrays(val1, val3, val2, new byte[] { 1 }), kv.value()); - + // We had one row to compact, so one put to do. verify(tsdb, times(1)).put(KEY, MockBase.concatByteArrays(qual1, qual3, qual2), - MockBase.concatByteArrays(val1, val3, val2, - new byte[] { 1 })); + MockBase.concatByteArrays(val1, val3, val2, + new byte[] { 1 }), kvCount - 1); // And we had to delete the individual cell + pre-existing compacted cell. verify(tsdb, times(1)).delete(eq(KEY), eqAnyOrder(new byte[][] { qual12, qual3 })); } - + @Test public void secondCompactMixedMSAndS() throws Exception { // In this test the row has already been compacted with a ms flag as the @@ -830,7 +943,7 @@ public void secondCompactMixedMSAndS() throws Exception { final byte[] qual2 = { 0x00, (byte) 0xF7 }; final byte[] val2 = Bytes.fromLong(5L); final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); - kvs.add(makekv(qual12, MockBase.concatByteArrays(val1, val2, + kvs.add(makekv(qual12, MockBase.concatByteArrays(val1, val2, new byte[] { 1 }))); // This data point came late. Note that its time delta falls in between // that of the two data points above. @@ -838,20 +951,20 @@ public void secondCompactMixedMSAndS() throws Exception { final byte[] val3 = Bytes.fromLong(6L); kvs.add(makekv(qual3, val3)); - final KeyValue kv = compactionq.compact(kvs, annotations); - assertArrayEquals(MockBase.concatByteArrays(qual3, qual1, qual2), + final KeyValue kv = compactionq.compact(kvs, annotations, null); + assertArrayEquals(MockBase.concatByteArrays(qual3, qual1, qual2), kv.qualifier()); - assertArrayEquals(MockBase.concatByteArrays(val3, val1, val2, + assertArrayEquals(MockBase.concatByteArrays(val3, val1, val2, new byte[] { 1 }), kv.value()); - + // We had one row to compact, so one put to do. verify(tsdb, times(1)).put(KEY, MockBase.concatByteArrays(qual3, qual1, qual2), - MockBase.concatByteArrays(val3, val1, val2, - new byte[] { 1 })); + MockBase.concatByteArrays(val3, val1, val2, + new byte[] { 1 }), kvCount - 1); // And we had to delete the individual cell + pre-existing compacted cell. verify(tsdb, times(1)).delete(eq(KEY), eqAnyOrder(new byte[][] { qual12, qual3 })); } - + @Test (expected = IllegalDataException.class) public void secondCompactOverwrite() throws Exception { PowerMockito.when(config.fix_duplicates()).thenReturn(false); @@ -872,9 +985,9 @@ public void secondCompactOverwrite() throws Exception { final byte[] val3 = Bytes.fromLong(6L); kvs.add(makekv(qual3, val3)); - compactionq.compact(kvs, annotations); + compactionq.compact(kvs, annotations, null); } - + @Test public void secondCompactOverwriteFix() throws Exception { // In this test the row has already been compacted, and a new value for an @@ -894,20 +1007,20 @@ public void secondCompactOverwriteFix() throws Exception { final byte[] val3 = Bytes.fromLong(6L); kvs.add(makekv(qual3, val3)); - final KeyValue kv = compactionq.compact(kvs, annotations); - assertArrayEquals(MockBase.concatByteArrays(qual3, qual2), + final KeyValue kv = compactionq.compact(kvs, annotations, null); + assertArrayEquals(MockBase.concatByteArrays(qual3, qual2), kv.qualifier()); - assertArrayEquals(MockBase.concatByteArrays(val3, val2, new byte[] { 0 }), + assertArrayEquals(MockBase.concatByteArrays(val3, val2, new byte[] { 0 }), kv.value()); - + // We had one row to compact, so one put to do. verify(tsdb, times(1)).put(KEY, MockBase.concatByteArrays(qual3, qual2), - MockBase.concatByteArrays(val3, val2, new byte[] { 0 })); + MockBase.concatByteArrays(val3, val2, new byte[] { 0 }), kvCount - 1); // And we had to delete the individual cell, but we overwrite the pre-existing compacted cell // rather than delete it. verify(tsdb, times(1)).delete(eq(KEY), eqAnyOrder(new byte[][] {qual3})); } - + @Test public void doubleFailedCompactNoop() throws Exception { // In this test the row has already been compacted once, but we didn't @@ -934,17 +1047,17 @@ public void doubleFailedCompactNoop() throws Exception { kvs.add(makekv(qual3, val3)); kvs.add(makekv(qual2, val2)); - final KeyValue kv = compactionq.compact(kvs, annotations); + final KeyValue kv = compactionq.compact(kvs, annotations, null); assertArrayEquals(qual132, kv.qualifier()); - assertArrayEquals(MockBase.concatByteArrays(val1, val3, val2, ZERO), + assertArrayEquals(MockBase.concatByteArrays(val1, val3, val2, ZERO), kv.value()); - + // We didn't have anything to write, the last cell is already the correct // compacted version of the row. - verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes()); + verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes(), anyLong()); // And we had to delete the 3 individual cells + the first pre-existing // compacted cell. - verify(tsdb, times(1)).delete(eq(KEY), eqAnyOrder(new byte[][] { qual1, + verify(tsdb, times(1)).delete(eq(KEY), eqAnyOrder(new byte[][] { qual1, qual12, qual3, qual2 })); } @@ -972,15 +1085,15 @@ public void weirdOverlappingCompactedCells() throws Exception { kvs.add(makekv(qual3, val3)); kvs.add(makekv(qual2, val2)); - final KeyValue kv = compactionq.compact(kvs, annotations); - assertArrayEquals(MockBase.concatByteArrays(qual1, qual3, qual2), + final KeyValue kv = compactionq.compact(kvs, annotations, null); + assertArrayEquals(MockBase.concatByteArrays(qual1, qual3, qual2), kv.qualifier()); - assertArrayEquals(MockBase.concatByteArrays(val1, val3, val2, ZERO), + assertArrayEquals(MockBase.concatByteArrays(val1, val3, val2, ZERO), kv.value()); - + // We had one row to compact, so one put to do. verify(tsdb, times(1)).put(KEY, MockBase.concatByteArrays(qual1, qual3, qual2), - MockBase.concatByteArrays(val1, val3, val2, ZERO)); + MockBase.concatByteArrays(val1, val3, val2, ZERO), kvCount - 1); // And we had to delete the 3 individual cells + 2 pre-existing // compacted cells. verify(tsdb, times(1)).delete(eq(KEY), eqAnyOrder(new byte[][] { qual1, qual12, qual13, qual3, qual2 })); @@ -1014,21 +1127,21 @@ public void tripleCompacted() throws Exception { kvs.add(makekv(qual34, MockBase.concatByteArrays(val3, val4, ZERO))); kvs.add(makekv(qual56, MockBase.concatByteArrays(val5, val6, ZERO))); - final KeyValue kv = compactionq.compact(kvs, annotations); + final KeyValue kv = compactionq.compact(kvs, annotations, null); assertArrayEquals( MockBase.concatByteArrays(qual12, qual34, qual56), kv.qualifier()); assertArrayEquals( - MockBase.concatByteArrays(val1, val2, val3, val4, val5, val6, ZERO), + MockBase.concatByteArrays(val1, val2, val3, val4, val5, val6, ZERO), kv.value()); // We wrote only the combined column. - verify(tsdb, times(1)).put(KEY, + verify(tsdb, times(1)).put(KEY, MockBase.concatByteArrays(qual1, qual2, qual3, qual4, qual5, qual6), - MockBase.concatByteArrays(val1, val2, val3, val4, val5, val6, ZERO)); + MockBase.concatByteArrays(val1, val2, val3, val4, val5, val6, ZERO), kvCount - 1); // And we had to delete the 3 partially compacted columns. verify(tsdb, times(1)).delete(eq(KEY), eqAnyOrder(new byte[][] { qual12, qual34, qual56 })); } - + @Test public void tripleCompactedOutOfOrder() throws Exception { // Here we have a row with #kvs > scanner.maxNumKeyValues and the result @@ -1057,21 +1170,21 @@ public void tripleCompactedOutOfOrder() throws Exception { kvs.add(makekv(qual56, MockBase.concatByteArrays(val5, val6, ZERO))); kvs.add(makekv(qual34, MockBase.concatByteArrays(val3, val4, ZERO))); - final KeyValue kv = compactionq.compact(kvs, annotations); + final KeyValue kv = compactionq.compact(kvs, annotations, null); assertArrayEquals( MockBase.concatByteArrays(qual12, qual34, qual56), kv.qualifier()); assertArrayEquals( - MockBase.concatByteArrays(val1, val2, val3, val4, val5, val6, ZERO), - kv.value()); - + MockBase.concatByteArrays(val1, val2, val3, val4, val5, val6, ZERO), + kv.value()); + // We wrote only the combined column. - verify(tsdb, times(1)).put(KEY, + verify(tsdb, times(1)).put(KEY, MockBase.concatByteArrays(qual1, qual2, qual3, qual4, qual5, qual6), - MockBase.concatByteArrays(val1, val2, val3, val4, val5, val6, ZERO)); + MockBase.concatByteArrays(val1, val2, val3, val4, val5, val6, ZERO), kvCount - 1); // And we had to delete the 3 partially compacted columns. verify(tsdb, times(1)).delete(eq(KEY), eqAnyOrder(new byte[][] { qual12, qual56, qual34 })); } - + @Test public void tripleCompactedSecondsAndMs() throws Exception { // Here we have a row with #kvs > scanner.maxNumKeyValues and the result @@ -1101,7 +1214,7 @@ public void tripleCompactedSecondsAndMs() throws Exception { kvs.add(makekv(qual34, MockBase.concatByteArrays(val3, val4, ZERO))); kvs.add(makekv(qual56, MockBase.concatByteArrays(val5, val6, ZERO))); - final KeyValue kv = compactionq.compact(kvs, annotations); + final KeyValue kv = compactionq.compact(kvs, annotations, null); assertArrayEquals( MockBase.concatByteArrays(qual12, qual34, qual56), kv.qualifier()); // TODO(jat): metadata byte should be 0x01? @@ -1110,13 +1223,309 @@ public void tripleCompactedSecondsAndMs() throws Exception { kv.value()); // We wrote only the combined column. - verify(tsdb, times(1)).put(KEY, + verify(tsdb, times(1)).put(KEY, MockBase.concatByteArrays(qual1, qual2, qual3, qual4, qual5, qual6), - MockBase.concatByteArrays(val1, val2, val3, val4, val5, val6, MIXED_FLAG)); + MockBase.concatByteArrays(val1, val2, val3, val4, val5, val6, MIXED_FLAG), kvCount - 1); // And we had to delete the 3 partially compacted columns. verify(tsdb, times(1)).delete(eq(KEY), eqAnyOrder(new byte[][] { qual12, qual34, qual56 })); } - + + @Test + public void appendsAndLaterPuts() throws Exception { + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); + ArrayList<Annotation> annotations = new ArrayList<Annotation>(0); + final byte[] qual = { 0x00, 0x07 }; + final byte[] val = Bytes.fromLong(42L); + final byte[] qual2 = { 0x00, 0x17 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual3 = { 0x00, 0x27 }; + final byte[] val3 = Bytes.fromLong(3L); + final byte[] qual4 = { 0x00, 0x37 }; + final byte[] val4 = Bytes.fromLong(2L); + kvs.add(makekv(AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(qual, val, qual2, val2))); + kvs.add(makekv(qual3, val3)); + kvs.add(makekv(qual4, val4)); + final KeyValue kv = compactionq.compact(kvs, annotations, null); + assertArrayEquals(MockBase.concatByteArrays(qual, qual2, qual3, qual4), + kv.qualifier()); + assertArrayEquals(MockBase.concatByteArrays(val, val2, val3, val4, ZERO), + kv.value()); + + // We had nothing to do so... + // ... verify there were no put. + verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes(), anyLong()); + // ... verify there were no delete. + verify(tsdb, never()).delete(anyBytes(), any(byte[][].class)); + } + + @Test + public void appendsAndEarlierPuts() throws Exception { + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); + ArrayList<Annotation> annotations = new ArrayList<Annotation>(0); + final byte[] qual = { 0x00, 0x07 }; + final byte[] val = Bytes.fromLong(42L); + final byte[] qual2 = { 0x00, 0x17 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual3 = { 0x00, 0x27 }; + final byte[] val3 = Bytes.fromLong(3L); + final byte[] qual4 = { 0x00, 0x37 }; + final byte[] val4 = Bytes.fromLong(2L); + kvs.add(makekv(qual, val)); + kvs.add(makekv(qual2, val2)); + kvs.add(makekv(AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(qual3, val3, qual4, val4))); + final KeyValue kv = compactionq.compact(kvs, annotations, null); + assertArrayEquals(MockBase.concatByteArrays(qual, qual2, qual3, qual4), + kv.qualifier()); + assertArrayEquals(MockBase.concatByteArrays(val, val2, val3, val4, ZERO), + kv.value()); + + // We had nothing to do so... + // ... verify there were no put. + verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes(), anyLong()); + // ... verify there were no delete. + verify(tsdb, never()).delete(anyBytes(), any(byte[][].class)); + } + + @Test + public void appendsAndInterspersedPuts() throws Exception { + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); + ArrayList<Annotation> annotations = new ArrayList<Annotation>(0); + final byte[] qual = { 0x00, 0x07 }; + final byte[] val = Bytes.fromLong(42L); + final byte[] qual2 = { 0x00, 0x17 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual3 = { 0x00, 0x27 }; + final byte[] val3 = Bytes.fromLong(3L); + final byte[] qual4 = { 0x00, 0x37 }; + final byte[] val4 = Bytes.fromLong(2L); + kvs.add(makekv(qual, val)); + kvs.add(makekv(qual3, val3)); + kvs.add(makekv(AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(qual2, val2, qual4, val4))); + final KeyValue kv = compactionq.compact(kvs, annotations, null); + assertArrayEquals(MockBase.concatByteArrays(qual, qual2, qual3, qual4), + kv.qualifier()); + assertArrayEquals(MockBase.concatByteArrays(val, val2, val3, val4, ZERO), + kv.value()); + + // We had nothing to do so... + // ... verify there were no put. + verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes(), anyLong()); + // ... verify there were no delete. + verify(tsdb, never()).delete(anyBytes(), any(byte[][].class)); + } + + @Test + public void doubleAppends() throws Exception { + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); + ArrayList<Annotation> annotations = new ArrayList<Annotation>(0); + final byte[] qual = { 0x00, 0x07 }; + final byte[] val = Bytes.fromLong(42L); + final byte[] qual2 = { 0x00, 0x17 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual3 = { 0x00, 0x27 }; + final byte[] val3 = Bytes.fromLong(3L); + final byte[] qual4 = { 0x00, 0x37 }; + final byte[] val4 = Bytes.fromLong(2L); + kvs.add(makekv(AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(qual, val, qual2, val2))); + kvs.add(makekv(AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(qual3, val3, qual4, val4))); + final KeyValue kv = compactionq.compact(kvs, annotations, null); + assertArrayEquals(MockBase.concatByteArrays(qual, qual2, qual3, qual4), + kv.qualifier()); + assertArrayEquals(MockBase.concatByteArrays(val, val2, val3, val4, ZERO), + kv.value()); + + // We had nothing to do so... + // ... verify there were no put. + verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes(), anyLong()); + // ... verify there were no delete. + verify(tsdb, never()).delete(anyBytes(), any(byte[][].class)); + } + + @Test + public void tripleAppends() throws Exception { + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); + ArrayList<Annotation> annotations = new ArrayList<Annotation>(0); + final byte[] qual = { 0x00, 0x07 }; + final byte[] val = Bytes.fromLong(42L); + final byte[] qual2 = { 0x00, 0x17 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual3 = { 0x00, 0x27 }; + final byte[] val3 = Bytes.fromLong(3L); + final byte[] qual4 = { 0x00, 0x37 }; + final byte[] val4 = Bytes.fromLong(2L); + final byte[] qual5 = { 0x00, 0x47 }; + final byte[] val5 = Bytes.fromLong(1L); + final byte[] qual6 = { 0x00, 0x57 }; + final byte[] val6 = Bytes.fromLong(0L); + kvs.add(makekv(AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(qual, val, qual2, val2))); + kvs.add(makekv(AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(qual3, val3, qual4, val4))); + kvs.add(makekv(AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(qual5, val5, qual6, val6))); + final KeyValue kv = compactionq.compact(kvs, annotations, null); + assertArrayEquals(MockBase.concatByteArrays( + qual, qual2, qual3, qual4, qual5, qual6), kv.qualifier()); + assertArrayEquals(MockBase.concatByteArrays( + val, val2, val3, val4, val5, val6, ZERO), kv.value()); + + // We had nothing to do so... + // ... verify there were no put. + verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes(), anyLong()); + // ... verify there were no delete. + verify(tsdb, never()).delete(anyBytes(), any(byte[][].class)); + } + + @Test + public void doubleAppendsAndPuts() throws Exception { + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); + ArrayList<Annotation> annotations = new ArrayList<Annotation>(0); + final byte[] qual = { 0x00, 0x07 }; + final byte[] val = Bytes.fromLong(42L); + final byte[] qual2 = { 0x00, 0x17 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual3 = { 0x00, 0x27 }; + final byte[] val3 = Bytes.fromLong(3L); + final byte[] qual4 = { 0x00, 0x37 }; + final byte[] val4 = Bytes.fromLong(2L); + final byte[] qual5 = { 0x00, 0x47 }; + final byte[] val5 = Bytes.fromLong(1L); + final byte[] qual6 = { 0x00, 0x57 }; + final byte[] val6 = Bytes.fromLong(0L); + kvs.add(makekv(AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(qual, val, qual2, val2))); + kvs.add(makekv(qual3, val3)); + kvs.add(makekv(qual4, val4)); + kvs.add(makekv(AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(qual5, val5, qual6, val6))); + final KeyValue kv = compactionq.compact(kvs, annotations, null); + assertArrayEquals(MockBase.concatByteArrays( + qual, qual2, qual3, qual4, qual5, qual6), kv.qualifier()); + assertArrayEquals(MockBase.concatByteArrays( + val, val2, val3, val4, val5, val6, ZERO), kv.value()); + + // We had nothing to do so... + // ... verify there were no put. + verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes(), anyLong()); + // ... verify there were no delete. + verify(tsdb, never()).delete(anyBytes(), any(byte[][].class)); + } + + @Test + public void appendsAndCompacted() throws Exception { + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); + ArrayList<Annotation> annotations = new ArrayList<Annotation>(0); + final byte[] qual = { 0x00, 0x07 }; + final byte[] val = Bytes.fromLong(42L); + final byte[] qual2 = { 0x00, 0x17 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual3 = { 0x00, 0x27 }; + final byte[] val3 = Bytes.fromLong(3L); + final byte[] qual4 = { 0x00, 0x37 }; + final byte[] val4 = Bytes.fromLong(2L); + kvs.add(makekv(AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(qual, val, qual2, val2))); + kvs.add(makekv(MockBase.concatByteArrays(qual3, qual4), + MockBase.concatByteArrays(val3, val4, ZERO))); + final KeyValue kv = compactionq.compact(kvs, annotations, null); + assertArrayEquals(MockBase.concatByteArrays(qual, qual2, qual3, qual4), + kv.qualifier()); + assertArrayEquals(MockBase.concatByteArrays(val, val2, val3, val4, ZERO), + kv.value()); + + // We had nothing to do so... + // ... verify there were no put. + verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes(), anyLong()); + // ... verify there were no delete. + verify(tsdb, never()).delete(anyBytes(), any(byte[][].class)); + } + + @Test + public void appendsAndCompactedAndPuts() throws Exception { + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); + ArrayList<Annotation> annotations = new ArrayList<Annotation>(0); + final byte[] qual = { 0x00, 0x07 }; + final byte[] val = Bytes.fromLong(42L); + final byte[] qual2 = { 0x00, 0x17 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual3 = { 0x00, 0x27 }; + final byte[] val3 = Bytes.fromLong(3L); + final byte[] qual4 = { 0x00, 0x37 }; + final byte[] val4 = Bytes.fromLong(2L); + final byte[] qual5 = { 0x00, 0x47 }; + final byte[] val5 = Bytes.fromLong(1L); + final byte[] qual6 = { 0x00, 0x57 }; + final byte[] val6 = Bytes.fromLong(0L); + kvs.add(makekv(AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(qual, val, qual2, val2))); + kvs.add(makekv(MockBase.concatByteArrays(qual3, qual4), + MockBase.concatByteArrays(val3, val4, ZERO))); + kvs.add(makekv(qual5, val5)); + kvs.add(makekv(qual6, val6)); + final KeyValue kv = compactionq.compact(kvs, annotations, null); + assertArrayEquals(MockBase.concatByteArrays( + qual, qual2, qual3, qual4, qual5, qual6), kv.qualifier()); + assertArrayEquals(MockBase.concatByteArrays( + val, val2, val3, val4, val5, val6, ZERO), kv.value()); + + // We had nothing to do so... + // ... verify there were no put. + verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes(), anyLong()); + // ... verify there were no delete. + verify(tsdb, never()).delete(anyBytes(), any(byte[][].class)); + } + + @Test + public void appendsDuplicatePuts() throws Exception { + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); + ArrayList<Annotation> annotations = new ArrayList<Annotation>(0); + final byte[] qual = { 0x00, 0x07 }; + final byte[] val = Bytes.fromLong(42L); + final byte[] qual2 = { 0x00, 0x17 }; + final byte[] val2 = Bytes.fromLong(5L); + kvs.add(makekv(AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(qual, val, qual2, val2))); + kvs.add(makekv(qual, val)); + kvs.add(makekv(qual2, val2)); + final KeyValue kv = compactionq.compact(kvs, annotations, null); + assertArrayEquals(MockBase.concatByteArrays(qual, qual2), kv.qualifier()); + assertArrayEquals(MockBase.concatByteArrays(val, val2, ZERO), kv.value()); + + // We had nothing to do so... + // ... verify there were no put. + verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes(), anyLong()); + // ... verify there were no delete. + verify(tsdb, never()).delete(anyBytes(), any(byte[][].class)); + } + + @Test + public void appendsDuplicateCompacted() throws Exception { + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); + ArrayList<Annotation> annotations = new ArrayList<Annotation>(0); + final byte[] qual = { 0x00, 0x07 }; + final byte[] val = Bytes.fromLong(42L); + final byte[] qual2 = { 0x00, 0x17 }; + final byte[] val2 = Bytes.fromLong(5L); + kvs.add(makekv(AppendDataPoints.APPEND_COLUMN_QUALIFIER, + MockBase.concatByteArrays(qual, val, qual2, val2))); + kvs.add(makekv(MockBase.concatByteArrays(qual, qual2), + MockBase.concatByteArrays(val, val2, ZERO))); + final KeyValue kv = compactionq.compact(kvs, annotations, null); + assertArrayEquals(MockBase.concatByteArrays(qual, qual2), kv.qualifier()); + assertArrayEquals(MockBase.concatByteArrays(val, val2, ZERO), kv.value()); + + // We had nothing to do so... + // ... verify there were no put. + verify(tsdb, never()).put(anyBytes(), anyBytes(), anyBytes(), anyLong()); + // ... verify there were no delete. + verify(tsdb, never()).delete(anyBytes(), any(byte[][].class)); + } + // ----------------- // // Helper functions. // // ----------------- // @@ -1129,6 +1538,10 @@ private static KeyValue makekv(final byte[] qualifier, final byte[] value) { return new KeyValue(KEY, FAMILY, qualifier, kvCount++, value); } + private static KeyValue makekvWithTs(final byte[] qualifier, long ts, final byte[] value) { + return new KeyValue(KEY, FAMILY, qualifier, ts, value); + } + private static byte[] anyBytes() { return any(byte[].class); } diff --git a/test/core/TestDownsampler.java b/test/core/TestDownsampler.java index 32e94c8d2a..5432b4c48c 100644 --- a/test/core/TestDownsampler.java +++ b/test/core/TestDownsampler.java @@ -12,7 +12,6 @@ // see <http://www.gnu.org/licenses/>. package net.opentsdb.core; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -20,17 +19,23 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; +import java.util.Calendar; import java.util.List; +import java.util.Locale; +import java.util.TimeZone; import com.google.common.collect.Lists; +import net.opentsdb.core.SeekableViewsForTest.MockSeekableView; +import net.opentsdb.rollup.RollupInterval; +import net.opentsdb.rollup.RollupQuery; import net.opentsdb.utils.DateTime; import org.junit.Before; import org.junit.Test; - /** Tests {@link Downsampler}. */ +@SuppressWarnings("deprecation") public class TestDownsampler { private static final long BASE_TIME = 1356998400000L; @@ -54,9 +59,19 @@ public class TestDownsampler { (int)DateTime.parseDuration("10s"); private static final Aggregator AVG = Aggregators.get("avg"); private static final Aggregator SUM = Aggregators.get("sum"); - + private static final TimeZone EST_TIME_ZONE = DateTime.timezones.get("EST"); + //30 minute offset + final static TimeZone AF = DateTime.timezones.get("Asia/Kabul"); + // 12h offset w/o DST + final static TimeZone TV = DateTime.timezones.get("Pacific/Funafuti"); + // 12h offset w DST + final static TimeZone FJ = DateTime.timezones.get("Pacific/Fiji"); + // Tue, 15 Dec 2015 04:02:25.123 UTC + final static long DST_TS = 1450137600000L; + private SeekableView source; private Downsampler downsampler; + private DownsamplingSpecification specification; @Before public void before() { @@ -65,6 +80,33 @@ public void before() { @Test public void testDownsampler() { + specification = new DownsamplingSpecification("1000s-avg"); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + verify(source, never()).next(); + List<Double> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + values.add(dp.doubleValue()); + timestamps_in_millis.add(dp.timestamp()); + } + + assertEquals(5, values.size()); + assertEquals(40, values.get(0), 0.0000001); + assertEquals(BASE_TIME - 400000L, timestamps_in_millis.get(0).longValue()); + assertEquals(50, values.get(1), 0.0000001); + assertEquals(BASE_TIME + 1600000, timestamps_in_millis.get(1).longValue()); + assertEquals(45, values.get(2), 0.0000001); + assertEquals(BASE_TIME + 3600000L, timestamps_in_millis.get(2).longValue()); + assertEquals(40, values.get(3), 0.0000001); + assertEquals(BASE_TIME + 6600000L, timestamps_in_millis.get(3).longValue()); + assertEquals(50, values.get(4), 0.0000001); + assertEquals(BASE_TIME + 8600000L, timestamps_in_millis.get(4).longValue()); + } + + @Test + public void testDownsamplerDeprecated() { downsampler = new Downsampler(source, THOUSAND_SEC_INTERVAL, AVG); verify(source, never()).next(); List<Double> values = Lists.newArrayList(); @@ -88,6 +130,47 @@ public void testDownsampler() { assertEquals(50, values.get(4), 0.0000001); assertEquals(BASE_TIME + 8600000L, timestamps_in_millis.get(4).longValue()); } + + @Test + public void testDownsamplerDeprecated_10seconds() { + source = spy(SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 0, 1), + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 1, 2), + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 2, 4), + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 3, 8), + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 4, 16), + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 5, 32), + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 6, 64), + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 7, 128), + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 8, 256), + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 9, 512), + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 10, 1024) + })); + downsampler = new Downsampler(source, 10000, SUM); + verify(source, never()).next(); + List<Double> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + values.add(dp.doubleValue()); + timestamps_in_millis.add(dp.timestamp()); + } + + assertEquals(6, values.size()); + assertEquals(3, values.get(0), 0.0000001); + assertEquals(BASE_TIME + 00000L, timestamps_in_millis.get(0).longValue()); + assertEquals(12, values.get(1), 0.0000001); + assertEquals(BASE_TIME + 10000L, timestamps_in_millis.get(1).longValue()); + assertEquals(48, values.get(2), 0.0000001); + assertEquals(BASE_TIME + 20000L, timestamps_in_millis.get(2).longValue()); + assertEquals(192, values.get(3), 0.0000001); + assertEquals(BASE_TIME + 30000L, timestamps_in_millis.get(3).longValue()); + assertEquals(768, values.get(4), 0.0000001); + assertEquals(BASE_TIME + 40000L, timestamps_in_millis.get(4).longValue()); + assertEquals(1024, values.get(5), 0.0000001); + assertEquals(BASE_TIME + 50000L, timestamps_in_millis.get(5).longValue()); + } @Test public void testDownsampler_10seconds() { @@ -104,7 +187,8 @@ public void testDownsampler_10seconds() { MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 9, 512), MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 10, 1024) })); - downsampler = new Downsampler(source, 10000, SUM); + specification = new DownsamplingSpecification("10s-sum"); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); verify(source, never()).next(); List<Double> values = Lists.newArrayList(); List<Long> timestamps_in_millis = Lists.newArrayList(); @@ -129,6 +213,38 @@ public void testDownsampler_10seconds() { assertEquals(1024, values.get(5), 0.0000001); assertEquals(BASE_TIME + 50000L, timestamps_in_millis.get(5).longValue()); } + + @Test + public void testDownsamplerDeprecated_15seconds() { + source = spy(SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(BASE_TIME + 5000L, 1), + MutableDataPoint.ofLongValue(BASE_TIME + 15000L, 2), + MutableDataPoint.ofLongValue(BASE_TIME + 25000L, 4), + MutableDataPoint.ofLongValue(BASE_TIME + 35000L, 8), + MutableDataPoint.ofLongValue(BASE_TIME + 45000L, 16), + MutableDataPoint.ofLongValue(BASE_TIME + 55000L, 32) + })); + downsampler = new Downsampler(source, 15000, SUM); + verify(source, never()).next(); + List<Double> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + values.add(dp.doubleValue()); + timestamps_in_millis.add(dp.timestamp()); + } + + assertEquals(4, values.size()); + assertEquals(1, values.get(0), 0.0000001); + assertEquals(BASE_TIME + 00000L, timestamps_in_millis.get(0).longValue()); + assertEquals(6, values.get(1), 0.0000001); + assertEquals(BASE_TIME + 15000L, timestamps_in_millis.get(1).longValue()); + assertEquals(8, values.get(2), 0.0000001); + assertEquals(BASE_TIME + 30000L, timestamps_in_millis.get(2).longValue()); + assertEquals(48, values.get(3), 0.0000001); + assertEquals(BASE_TIME + 45000L, timestamps_in_millis.get(3).longValue()); + } @Test public void testDownsampler_15seconds() { @@ -140,7 +256,8 @@ public void testDownsampler_15seconds() { MutableDataPoint.ofLongValue(BASE_TIME + 45000L, 16), MutableDataPoint.ofLongValue(BASE_TIME + 55000L, 32) })); - downsampler = new Downsampler(source, 15000, SUM); + specification = new DownsamplingSpecification("15s-sum"); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); verify(source, never()).next(); List<Double> values = Lists.newArrayList(); List<Long> timestamps_in_millis = Lists.newArrayList(); @@ -161,7 +278,1064 @@ public void testDownsampler_15seconds() { assertEquals(48, values.get(3), 0.0000001); assertEquals(BASE_TIME + 45000L, timestamps_in_millis.get(3).longValue()); } + + @Test + public void testDownsampler_allFullRange() { + source = spy(SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(BASE_TIME + 5000L, 1), + MutableDataPoint.ofLongValue(BASE_TIME + 15000L, 2), + MutableDataPoint.ofLongValue(BASE_TIME + 25000L, 4), + MutableDataPoint.ofLongValue(BASE_TIME + 35000L, 8), + MutableDataPoint.ofLongValue(BASE_TIME + 45000L, 16), + MutableDataPoint.ofLongValue(BASE_TIME + 55000L, 32) + })); + specification = new DownsamplingSpecification("0all-sum"); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + + verify(source, never()).next(); + List<Double> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + values.add(dp.doubleValue()); + timestamps_in_millis.add(dp.timestamp()); + } + + assertEquals(1, values.size()); + assertEquals(63, values.get(0), 0.0000001); + assertEquals(0L, timestamps_in_millis.get(0).longValue()); + } + + @Test + public void testDownsampler_allFilterOnQuery() { + source = spy(SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(BASE_TIME + 5000L, 1), + MutableDataPoint.ofLongValue(BASE_TIME + 15000L, 2), + MutableDataPoint.ofLongValue(BASE_TIME + 25000L, 4), + MutableDataPoint.ofLongValue(BASE_TIME + 35000L, 8), + MutableDataPoint.ofLongValue(BASE_TIME + 45000L, 16), + MutableDataPoint.ofLongValue(BASE_TIME + 55000L, 32) + })); + specification = new DownsamplingSpecification("0all-sum"); + downsampler = new Downsampler(source, specification, + BASE_TIME + 15000L, BASE_TIME + 45000L); + verify(source, never()).next(); + List<Double> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + values.add(dp.doubleValue()); + timestamps_in_millis.add(dp.timestamp()); + } + + assertEquals(1, values.size()); + assertEquals(14, values.get(0), 0.0000001); + assertEquals(BASE_TIME + 15000L, timestamps_in_millis.get(0).longValue()); + } + + @Test + public void testDownsampler_allFilterOnQueryOutOfRangeEarly() { + source = spy(SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(BASE_TIME + 5000L, 1), + MutableDataPoint.ofLongValue(BASE_TIME + 15000L, 2), + MutableDataPoint.ofLongValue(BASE_TIME + 25000L, 4), + MutableDataPoint.ofLongValue(BASE_TIME + 35000L, 8), + MutableDataPoint.ofLongValue(BASE_TIME + 45000L, 16), + MutableDataPoint.ofLongValue(BASE_TIME + 55000L, 32) + })); + specification = new DownsamplingSpecification("0all-sum"); + downsampler = new Downsampler(source, specification, + BASE_TIME + 65000L, BASE_TIME + 75000L); + verify(source, never()).next(); + List<Double> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + values.add(dp.doubleValue()); + timestamps_in_millis.add(dp.timestamp()); + } + + assertEquals(0, values.size()); + } + + @Test + public void testDownsampler_allFilterOnQueryOutOfRangeLate() { + source = spy(SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(BASE_TIME + 5000L, 1), + MutableDataPoint.ofLongValue(BASE_TIME + 15000L, 2), + MutableDataPoint.ofLongValue(BASE_TIME + 25000L, 4), + MutableDataPoint.ofLongValue(BASE_TIME + 35000L, 8), + MutableDataPoint.ofLongValue(BASE_TIME + 45000L, 16), + MutableDataPoint.ofLongValue(BASE_TIME + 55000L, 32) + })); + specification = new DownsamplingSpecification("0all-sum"); + downsampler = new Downsampler(source, specification, + BASE_TIME - 15000L, BASE_TIME - 5000L); + verify(source, never()).next(); + List<Double> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + values.add(dp.doubleValue()); + timestamps_in_millis.add(dp.timestamp()); + } + + assertEquals(0, values.size()); + } + + @Test + public void testDownsampler_calendar() { + source = spy(SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(BASE_TIME + 5000L, 1), + MutableDataPoint.ofLongValue(BASE_TIME + 15000L, 2), + MutableDataPoint.ofLongValue(BASE_TIME + 25000L, 4), + MutableDataPoint.ofLongValue(BASE_TIME + 35000L, 8), + MutableDataPoint.ofLongValue(BASE_TIME + 45000L, 16), + MutableDataPoint.ofLongValue(BASE_TIME + 55000L, 32) + })); + specification = new DownsamplingSpecification("1dc-sum"); + specification.setTimezone(DateTime.timezones.get("America/Denver")); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + verify(source, never()).next(); + List<Double> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + values.add(dp.doubleValue()); + timestamps_in_millis.add(dp.timestamp()); + } + + assertEquals(1, values.size()); + assertEquals(63, values.get(0), 0.0000001); + assertEquals(1356937200000L, timestamps_in_millis.get(0).longValue()); + } + + @Test + public void testDownsampler_calendarHour() { + source = spy(SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(BASE_TIME, 1), + MutableDataPoint.ofLongValue(BASE_TIME + 1800000, 2), + MutableDataPoint.ofLongValue(BASE_TIME + 3599000L, 3), + MutableDataPoint.ofLongValue(BASE_TIME + 3600000L, 4), + MutableDataPoint.ofLongValue(BASE_TIME + 5400000L, 5), + MutableDataPoint.ofLongValue(BASE_TIME + 7199000L, 6) + })); + specification = new DownsamplingSpecification("1hc-sum"); + specification.setTimezone(TV); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + + long ts = BASE_TIME; + double value = 6; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + ts += 3600000; + value = 15; + } + + // hour offset by 30m + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("1hc-sum"); + specification.setTimezone(AF); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1356996600000L; + value = 1; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + ts += 3600000; + if (value == 1) { + value = 9; + } else { + value = 11; + } + } + + // multiple hours + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("4hc-sum"); + specification.setTimezone(AF); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1356996600000L; + value = 21; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + } + } + + @Test + public void testDownsampler_calendarDay() { + source = spy(SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(DST_TS, 1), + MutableDataPoint.ofLongValue(DST_TS + 86399000, 2), + MutableDataPoint.ofLongValue(DST_TS + 126001000L, 3), // falls to the next in FJ + MutableDataPoint.ofLongValue(DST_TS + 172799000L, 4), + MutableDataPoint.ofLongValue(DST_TS + 172800000L, 5), + MutableDataPoint.ofLongValue(DST_TS + 242999000L, 6) // falls within 30m offset + })); + + // control + specification = new DownsamplingSpecification("1dc-sum"); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + + long ts = DST_TS; + double value = 3; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + ts += 86400000; + if (value == 3) { + value = 7; + } else if (value == 7) { + value = 11; + } + } + + // 12 hour offset from UTC + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("1dc-sum"); + specification.setTimezone(TV); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1450094400000L; + value = 1; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + ts += 86400000; + if (value == 1) { + value = 5; + } else if (value == 5) { + value = 9; + } else { + value = 6; + } + } + + // 11 hour offset from UTC + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("1dc-sum"); + specification.setTimezone(FJ); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1450090800000L; + value = 1; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + ts += 86400000; + if (value == 1) { + value = 2; + } else if (value == 2) { + value = 12; + } else { + value = 6; + } + } + + // 30m offset + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("1dc-sum"); + specification.setTimezone(AF); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1450121400000L; + value = 1; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + ts += 86400000; + if (value == 1) { + value = 5; + } else if (value == 5) { + value = 15; + } + } + + // multiple days + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("3dc-sum"); + specification.setTimezone(AF); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1450121400000L; + value = 21; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + } + } + + @Test + public void testDownsampler_calendarWeek() { + // Test assumes Sunday is first day of week. + Locale.setDefault(Locale.US); + + source = SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(DST_TS, 1), // a Tuesday in UTC land + MutableDataPoint.ofLongValue(DST_TS + (86400000L * 7), 2), + MutableDataPoint.ofLongValue(1451129400000L, 3), // falls to the next in FJ + MutableDataPoint.ofLongValue(DST_TS + (86400000L * 21), 4), + MutableDataPoint.ofLongValue(1452367799000L, 5) // falls within 30m offset + }); + // control + specification = new DownsamplingSpecification("1wc-sum"); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + + long ts = 1449964800000L; + double value = 1; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + if (ts == 1450569600000L) { + ts = 1451779200000L; // skips a week + } else { + ts += 86400000L * 7; + } + if (value == 1) { + value = 5; + } else { + value = 9; + } + } + + // 12 hour offset from UTC + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("1wc-sum"); + specification.setTimezone(TV); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + ts = 1449921600000L; + value = 1; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + if (ts == 1450526400000L) { + ts = 1451736000000L; // skip a week + } else { + ts += 86400000L * 7; + } + if (value == 1) { + value = 5; + } else if (value == 5) { + value = 4; + } else { + value = 5; + } + } + + // 11 hour offset from UTC + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("1wc-sum"); + specification.setTimezone(FJ); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1449918000000L; + value = 1; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + ts += 86400000L * 7; + value++; + } + + // 30m offset + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("1wc-sum"); + specification.setTimezone(AF); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1449948600000L; + value = 1; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + if (ts == 1449948600000L) { + ts = 1450553400000L; + } else { + ts = 1451763000000L; + } + if (value == 1) { + value = 5; + } else { + value = 9; + } + } + + // multiple weeks + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("2wc-sum"); + specification.setTimezone(AF); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1449948600000L; + value = 6; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + ts = 1451158200000L; + value = 9; + } + } + + @Test + public void testDownsampler_calendarMonth() { + final long dec_1st = 1448928000000L; + source = spy(SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(dec_1st, 1), + MutableDataPoint.ofLongValue(1451559600000L, 2), // falls to the next in FJ + MutableDataPoint.ofLongValue(1451606400000L, 3), // jan 1st + MutableDataPoint.ofLongValue(1454284800000L, 4), // feb 1st + MutableDataPoint.ofLongValue(1456704000000L, 5), // feb 29th (leap year) + MutableDataPoint.ofLongValue(1456772400000L, 6) // falls within 30m offset AF + })); + + // control + specification = new DownsamplingSpecification("1nc-sum"); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + + long ts = dec_1st; + double value = 3; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + if (ts == 1448928000000L) { + ts = 1451606400000L; + } else { + ts = 1454284800000L; + value = 15; + } + } + + // 12h offset + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("1nc-sum"); + specification.setTimezone(TV); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1448884800000L; + value = 3; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + if (ts == 1448884800000L) { + ts = 1451563200000L; + } else if (ts == 1451563200000L) { + value = 9; + ts = 1454241600000L; + } else { + ts = 1456747200000L; + value = 6; + } + } + + // 11h offset + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("1nc-sum"); + specification.setTimezone(FJ); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1448881200000L; + value = 1; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + if (ts == 1448881200000L) { + ts = 1451559600000L; + value = 5; + } else if (ts == 1451559600000L) { + ts = 1454241600000L; + value = 9; + } else { + ts = 1456747200000L; + value = 6; + } + } + + // 30m offset + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("1nc-sum"); + specification.setTimezone(AF); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1448911800000L; + value = 3; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + if (ts == 1448911800000L) { + ts = 1451590200000L; + } else { + ts = 1454268600000L; + value = 15; + } + } + + // multiple months + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("3nc-sum"); + specification.setTimezone(TV); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1443614400000L; + value = 3; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + ts = 1451563200000L; + value = 18; + } + } + + @Test + public void testDownsampler_noData() { + source = spy(SeekableViewsForTest.fromArray(new DataPoint[] { })); + specification = new DownsamplingSpecification("1d-sum"); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + verify(source, never()).next(); + assertFalse(downsampler.hasNext()); + } + + @Test + public void testDownsampler_noDataCalendar() { + source = spy(SeekableViewsForTest.fromArray(new DataPoint[] { })); + specification = new DownsamplingSpecification("1mc-sum"); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + verify(source, never()).next(); + assertFalse(downsampler.hasNext()); + } + + @Test + public void testDownsampler_1day() { + source = spy(SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(BASE_TIME, 1), + MutableDataPoint.ofLongValue(BASE_TIME + 43200000L, 2), + MutableDataPoint.ofLongValue(BASE_TIME + 86400000L, 4), + MutableDataPoint.ofLongValue(BASE_TIME + 129600000L, 8) + })); + + downsampler = new Downsampler(source, 86400000, SUM); + verify(source, never()).next(); + long timestamp = BASE_TIME; + double value = 3; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(timestamp, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.000001); + timestamp = 1357084800000L; + value = 12; + } + } + + @Test + public void testDownsampler_1day_timezone() { + source = spy(SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(1357016400000L, 1), + MutableDataPoint.ofLongValue(1357059600000L, 2), + MutableDataPoint.ofLongValue(1357102800000L, 4), + MutableDataPoint.ofLongValue(1357146000000L, 8) + })); + + specification = new DownsamplingSpecification("1dc-sum"); + specification.setTimezone(EST_TIME_ZONE); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + verify(source, never()).next(); + + long timestamp = 1357016400000L; + double value = 3; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(timestamp, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.000001); + timestamp = 1357102800000L; + value = 12; + } + } + + @Test + public void testDownsampler_1week() { + source = spy(SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(1356825600000L, 1), + MutableDataPoint.ofLongValue(1357128000000L, 2), + MutableDataPoint.ofLongValue(1357430400000L, 4), + MutableDataPoint.ofLongValue(1357732800000L, 8) + })); + + // Test assumes Sunday is first day of week. + Locale.setDefault(Locale.US); + specification = new DownsamplingSpecification("1wc-sum"); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + verify(source, never()).next(); + long timestamp = 1356825600000L; + double value = 3; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(timestamp, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.000001); + timestamp = 1357430400000L; + value = 12; + } + } + + @Test + public void testDownsampler_1week_timezone() { + source = spy(SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(1356843600000L, 1), + MutableDataPoint.ofLongValue(1357146000000L, 2), + MutableDataPoint.ofLongValue(1357448400000L, 4), + MutableDataPoint.ofLongValue(1357750800000L, 8) + })); + + // Test assumes Sunday is first day of week. + Locale.setDefault(Locale.US); + specification = new DownsamplingSpecification("1wc-sum"); + specification.setTimezone(EST_TIME_ZONE); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + verify(source, never()).next(); + long timestamp = 1356843600000L; + double value = 3; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(timestamp, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.000001); + timestamp = 1357448400000L; + value = 12; + } + } + + @Test + public void testDownsampler_1month() { + final int field = DateTime.unitsToCalendarType("n"); + final DataPoint [] data_points = new DataPoint[24]; + Calendar c = DateTime.previousInterval(BASE_TIME, 1, field); + //long timestamp = DateTime.toStartOfMonth(BASE_TIME, UTC_TIME_ZONE); + long timestamp = c.getTimeInMillis(); + for (int i = 0; i < data_points.length; i++) { + long value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + + i += 1; + c.add(field, 1); + long startOfNextInterval = c.getTimeInMillis() + 1; + timestamp = timestamp + (startOfNextInterval - timestamp) / 2; + value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + timestamp = startOfNextInterval; + } + + source = spy(SeekableViewsForTest.fromArray(data_points)); + + specification = new DownsamplingSpecification("1nc-sum"); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + verify(source, never()).next(); + c = DateTime.previousInterval(BASE_TIME, 1, field); + int j = 0; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals((1 << j++) + (1 << j++), dp.doubleValue(), 0.0000001); + assertEquals(c.getTimeInMillis(), dp.timestamp()); + c.add(field, 1); + } + } + + @Test + public void testDownsampler_1month_alt() { + /* + 1380600000 -> 2013-10-01T04:00:00Z + 1383278400 -> 2013-11-01T04:00:00Z + 1385874000 -> 2013-12-01T05:00:00Z + 1388552400 -> 2014-01-01T05:00:00Z + 1391230800 -> 2014-02-01T05:00:00Z + 1393650000 -> 2014-03-01T05:00:00Z + 1396324800 -> 2014-04-01T04:00:00Z + 1398916800 -> 2014-05-01T04:00:00Z + 1401595200 -> 2014-06-01T04:00:00Z + 1404187200 -> 2014-07-01T04:00:00Z + 1406865600 -> 2014-08-01T04:00:00Z + 1409544000 -> 2014-09-01T04:00:00Z + */ + + int value = 1; + final DataPoint [] data_points = new DataPoint[] { + MutableDataPoint.ofLongValue(1380600000000L, value), + MutableDataPoint.ofLongValue(1383278400000L, value), + MutableDataPoint.ofLongValue(1385874000000L, value), + MutableDataPoint.ofLongValue(1388552400000L, value), + MutableDataPoint.ofLongValue(1391230800000L, value), + MutableDataPoint.ofLongValue(1393650000000L, value), + MutableDataPoint.ofLongValue(1396324800000L, value), + MutableDataPoint.ofLongValue(1398916800000L, value), + MutableDataPoint.ofLongValue(1401595200000L, value), + MutableDataPoint.ofLongValue(1404187200000L, value), + MutableDataPoint.ofLongValue(1406865600000L, value), + MutableDataPoint.ofLongValue(1409544000000L, value), + }; + + source = spy(SeekableViewsForTest.fromArray(data_points)); + + specification = new DownsamplingSpecification("1dc-sum"); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + verify(source, never()).next(); + final int field = DateTime.unitsToCalendarType("n"); + final Calendar c = DateTime.previousInterval(1380585600000L, 1, field); + long timestamp = c.getTimeInMillis(); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(1, dp.doubleValue(), 0.0000001); + assertEquals(timestamp, dp.timestamp()); + c.add(field, 1); + timestamp = c.getTimeInMillis(); + } + } + + @Test + public void testDownsampler_2months() { + final int field = DateTime.unitsToCalendarType("n"); + final DataPoint [] data_points = new DataPoint[24]; + Calendar c = DateTime.previousInterval(BASE_TIME, 1, field); + long timestamp = c.getTimeInMillis(); + for (int i = 0; i < data_points.length; i++) { + long value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + + i += 1; + c.add(field, 1); + long startOfNextInterval = c.getTimeInMillis(); + timestamp = timestamp + (startOfNextInterval - timestamp) / 2; + value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + timestamp = startOfNextInterval; + } + + source = spy(SeekableViewsForTest.fromArray(data_points)); + + specification = new DownsamplingSpecification("2nc-sum"); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + verify(source, never()).next(); + int j = 0; + c = DateTime.previousInterval(BASE_TIME, 1, field); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + long value = 0; + for (int k = 0; k < 4; k++) { + value += (1 << j++); + } + assertEquals(value, dp.doubleValue(), 0.0000001); + assertEquals(c.getTimeInMillis(), dp.timestamp()); + c.add(field, 2); + } + } + + @Test + public void testDownsampler_1month_timezone() { + final int field = DateTime.unitsToCalendarType("n"); + final DataPoint [] data_points = new DataPoint[24]; + Calendar c = DateTime.previousInterval(1357016400000L, 1, field, EST_TIME_ZONE); + long timestamp = c.getTimeInMillis(); + for (int i = 0; i < data_points.length; i++) { + long value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + + i += 1; + c.add(field, 1); + long startOfNextInterval = c.getTimeInMillis(); + timestamp = timestamp + (startOfNextInterval - timestamp) / 2; + value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + timestamp = startOfNextInterval; + } + + source = spy(SeekableViewsForTest.fromArray(data_points)); + + specification = new DownsamplingSpecification("1nc-sum"); + specification.setTimezone(EST_TIME_ZONE); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + verify(source, never()).next(); + int j = 0; + c = DateTime.previousInterval(1357016400000L, 1, field, EST_TIME_ZONE); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals((1 << j++) + (1 << j++), dp.doubleValue(), 0.0000001); + assertEquals(c.getTimeInMillis(), dp.timestamp()); + c.add(field, 1); + } + } + + @Test + public void testDownsampler_1year() { + final int field = DateTime.unitsToCalendarType("y"); + final DataPoint [] data_points = new DataPoint[4]; + Calendar c = DateTime.previousInterval(BASE_TIME, 1, field); + long timestamp = c.getTimeInMillis(); + for (int i = 0; i < data_points.length; i++) { + long value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + + i += 1; + c.add(field, 1); + long startOfNextInterval = c.getTimeInMillis(); + timestamp = timestamp + (startOfNextInterval - timestamp) / 2; + value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + timestamp = startOfNextInterval; + } + + source = spy(SeekableViewsForTest.fromArray(data_points)); + + specification = new DownsamplingSpecification("1yc-sum"); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + verify(source, never()).next(); + int j = 0; + c = DateTime.previousInterval(BASE_TIME, 1, field); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals((1 << j++) + (1 << j++), dp.doubleValue(), 0.0000001); + assertEquals(c.getTimeInMillis(), dp.timestamp()); + c.add(field, 1); + } + } + + @Test + public void testDownsampler_1year_timezone() { + final int field = DateTime.unitsToCalendarType("y"); + final DataPoint [] data_points = new DataPoint[4]; + Calendar c = DateTime.previousInterval(1357016400000L, 1, field, + EST_TIME_ZONE); + long timestamp = c.getTimeInMillis(); + for (int i = 0; i < data_points.length; i++) { + long value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + + i += 1; + c.add(field, 1); + long startOfNextInterval = c.getTimeInMillis(); + timestamp = timestamp + (startOfNextInterval - timestamp) / 2; + value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + timestamp = startOfNextInterval; + } + + source = spy(SeekableViewsForTest.fromArray(data_points)); + + specification = new DownsamplingSpecification("1yc-sum"); + specification.setTimezone(EST_TIME_ZONE); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + verify(source, never()).next(); + int j = 0; + c = DateTime.previousInterval(1357016400000L, 1, field, EST_TIME_ZONE); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals((1 << j++) + (1 << j++), dp.doubleValue(), 0.0000001); + assertEquals(c.getTimeInMillis(), dp.timestamp()); + c.add(field, 1); + } + } + + @Test + public void testDownsampler_rollupSum() { + final RollupInterval interval = RollupInterval.builder() + .setTable("tsdb-rollup-1h") + .setPreAggregationTable("tsdb-agg-rollup-1h") + .setInterval("1h") + .setRowSpan("1d") + .build(); + final RollupQuery rollup_query = new RollupQuery(interval, Aggregators.SUM, + 3600000, Aggregators.SUM); + source = spy(SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 0, 1), + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 1, 2), + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 2, 4), + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 3, 8), + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 4, 16), + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 5, 32), + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 6, 64), + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 7, 128), + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 8, 256), + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 9, 512), + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 10, 1024) + })); + specification = new DownsamplingSpecification("10s-sum"); + downsampler = new Downsampler(source, specification, 0, 0, rollup_query); + verify(source, never()).next(); + List<Double> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + values.add(dp.doubleValue()); + timestamps_in_millis.add(dp.timestamp()); + } + + assertEquals(6, values.size()); + assertEquals(3, values.get(0), 0.0000001); + assertEquals(BASE_TIME + 00000L, timestamps_in_millis.get(0).longValue()); + assertEquals(12, values.get(1), 0.0000001); + assertEquals(BASE_TIME + 10000L, timestamps_in_millis.get(1).longValue()); + assertEquals(48, values.get(2), 0.0000001); + assertEquals(BASE_TIME + 20000L, timestamps_in_millis.get(2).longValue()); + assertEquals(192, values.get(3), 0.0000001); + assertEquals(BASE_TIME + 30000L, timestamps_in_millis.get(3).longValue()); + assertEquals(768, values.get(4), 0.0000001); + assertEquals(BASE_TIME + 40000L, timestamps_in_millis.get(4).longValue()); + assertEquals(1024, values.get(5), 0.0000001); + assertEquals(BASE_TIME + 50000L, timestamps_in_millis.get(5).longValue()); + } + + @Test + public void testDownsampler_rollupAvg() { + final RollupInterval interval = RollupInterval.builder() + .setTable("tsdb-rollup-1h") + .setPreAggregationTable("tsdb-agg-rollup-1h") + .setInterval("1h") + .setRowSpan("1d") + .build(); + final RollupQuery rollup_query = new RollupQuery(interval, Aggregators.AVG, + 3600000, Aggregators.SUM); + source = spy(SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 0, 1), + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 1, 2), + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 2, 4), + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 3, 8) + })); + specification = new DownsamplingSpecification("10s-avg"); + downsampler = new Downsampler(source, specification, 0, 0, rollup_query); + verify(source, never()).next(); + List<Double> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + values.add(dp.doubleValue()); + timestamps_in_millis.add(dp.timestamp()); + } + + assertEquals(2, values.size()); + assertEquals(1.5, values.get(0), 0.0000001); + assertEquals(BASE_TIME + 00000L, timestamps_in_millis.get(0).longValue()); + assertEquals(6, values.get(1), 0.0000001); + assertEquals(BASE_TIME + 10000L, timestamps_in_millis.get(1).longValue()); + } + + @Test + public void testDownsampler_rollupCount() { + final RollupInterval interval = RollupInterval.builder() + .setTable("tsdb-rollup-1h") + .setPreAggregationTable("tsdb-agg-rollup-1h") + .setInterval("1h") + .setRowSpan("1d") + .build(); + + // This query, in combination with the configuration/interval above, is asking for rolled up COUNTs (i.e. already + // downsampled). These COUNTs should then be SUMmed over some interval we'll define later in a DownsamplingSpecification + final RollupQuery rollup_query = new RollupQuery(interval, Aggregators.COUNT, + 3600000, Aggregators.SUM); + + // These points represent rolled up COUNTs that would be stored in the rollup table (e.g. tsdb-rollup-1h) and as + // such we don't expect these to be COUNTed again on retrieval. These points are at 5s intervals + source = spy(SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 0, 1), + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 1, 2), + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 2, 4), + MutableDataPoint.ofDoubleValue(BASE_TIME + 5000L * 3, 8) + })); + + // The rolled up points above are at 5s intervals but we want to downsample these further so that we only get + // points at 10s intervals + specification = new DownsamplingSpecification("10s-count"); + downsampler = new Downsampler(source, specification, 0, 0, rollup_query); + verify(source, never()).next(); + List<Double> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + values.add(dp.doubleValue()); + timestamps_in_millis.add(dp.timestamp()); + } + + // Asserts here different to upstream as there is a bug upstream and the original test was written to pass with the bug. + // 2 points are expected because the 4 points at 5s intervals will be downsampled to 2 points at 10s intervals + assertEquals(2, values.size()); + + // Expect the SUM of points 1 and 2 + assertEquals(3, values.get(0), 0.0000001); + assertEquals(BASE_TIME + 00000L, timestamps_in_millis.get(0).longValue()); + + // Expect the SUM of points 3 and 4 + assertEquals(12, values.get(1), 0.0000001); + assertEquals(BASE_TIME + 10000L, timestamps_in_millis.get(1).longValue()); + } + + @Test (expected = UnsupportedOperationException.class) + public void testDownsampler_rollupDev() { + final RollupInterval interval = RollupInterval.builder() + .setTable("tsdb-rollup-1h") + .setPreAggregationTable("tsdb-agg-rollup-1h") + .setInterval("1h") + .setRowSpan("1d") + .build(); + final RollupQuery rollup_query = new RollupQuery(interval, Aggregators.DEV, + 3600000, Aggregators.SUM); + specification = new DownsamplingSpecification("10s-dev"); + downsampler = new Downsampler(source, specification, 0, 0, rollup_query); + while (downsampler.hasNext()) { + downsampler.next(); // <-- throws here + } + } + @Test(expected = UnsupportedOperationException.class) public void testRemove() { new Downsampler(source, THOUSAND_SEC_INTERVAL, AVG).remove(); @@ -190,6 +1364,46 @@ public void testSeek() { assertEquals(BASE_TIME + 8600000L, timestamps_in_millis.get(2).longValue()); } + @Test + public void testSeek_useCalendar() { + source = spy(SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(1356998400000L, 1), + MutableDataPoint.ofLongValue(1388534400000L, 2), + MutableDataPoint.ofLongValue(1420070400000L, 4), + MutableDataPoint.ofLongValue(1451606400000L, 8) + })); + + specification = new DownsamplingSpecification("1y-sum"); + specification.setUseCalendar(true); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + + downsampler.seek(1420070400000L); + verify(source, never()).next(); + + long timestamp = 1420070400000L; + double value = 4; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(timestamp, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.0000001); + timestamp = 1451606400000L; + value = 8; + } + + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("1yc-sum"); + downsampler = new Downsampler(source, specification, 0, Long.MAX_VALUE); + downsampler.seek(1420070400001L); + + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(timestamp, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.0000001); + } + } + @Test public void testSeek_skipPartialInterval() { downsampler = new Downsampler(source, THOUSAND_SEC_INTERVAL, AVG); @@ -284,7 +1498,6 @@ public void testSeek_abandoningIncompleteInterval() { public void testToString() { downsampler = new Downsampler(source, THOUSAND_SEC_INTERVAL, AVG); DataPoint dp = downsampler.next(); - System.out.println(downsampler.toString()); assertTrue(downsampler.toString().contains(dp.toString())); } } diff --git a/test/core/TestDownsamplingSpecification.java b/test/core/TestDownsamplingSpecification.java new file mode 100644 index 0000000000..9e541f55ec --- /dev/null +++ b/test/core/TestDownsamplingSpecification.java @@ -0,0 +1,184 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import org.junit.Test; + +import net.opentsdb.utils.DateTime; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.TimeZone; + +public class TestDownsamplingSpecification { + final long interval = 60000L; + final FillPolicy fill_policy = FillPolicy.ZERO; + final Aggregator function = Aggregators.SUM; + final TimeZone timezone = DateTime.timezones.get(DateTime.UTC_ID); + + @Test + public void testCtor() { + final long interval = 1234567L; + final Aggregator function = Aggregators.SUM; + final FillPolicy fill_policy = FillPolicy.ZERO; + + final DownsamplingSpecification ds = new DownsamplingSpecification( + interval, function, fill_policy); + + assertEquals(interval, ds.getInterval()); + assertEquals(function, ds.getFunction()); + assertEquals(fill_policy, ds.getFillPolicy()); + } + + @Test + public void testStringCtor() { + final DownsamplingSpecification ds = new DownsamplingSpecification( + "15m-avg-nan"); + + assertEquals(900000L, ds.getInterval()); + assertEquals(Aggregators.AVG, ds.getFunction()); + assertEquals(FillPolicy.NOT_A_NUMBER, ds.getFillPolicy()); + } + + @Test(expected = RuntimeException.class) + public void testBadInterval() { + new DownsamplingSpecification("blah-avg-lerp"); + } + + @Test(expected = RuntimeException.class) + public void testBadFunction() { + new DownsamplingSpecification("1m-hurp-lerp"); + } + + @Test(expected = RuntimeException.class) + public void testBadFillPolicy() { + new DownsamplingSpecification("10m-avg-max"); + } + + @Test (expected = IllegalArgumentException.class) + public void testNoneAgg() { + new DownsamplingSpecification("1m-none-lerp"); + } + + @Test (expected = IllegalArgumentException.class) + public void testCtorNegativeInterval() { + new DownsamplingSpecification(-1, function, fill_policy); + } + + @Test (expected = IllegalArgumentException.class) + public void testCtorZeroInterval() { + new DownsamplingSpecification(DownsamplingSpecification.NO_INTERVAL, + function, fill_policy); + } + + @Test (expected = IllegalArgumentException.class) + public void testCtorNullFunction() { + new DownsamplingSpecification(interval, null, fill_policy); + } + + @Test(expected = IllegalArgumentException.class) + public void testStringCtorNull() { + new DownsamplingSpecification(null); + } + + @Test(expected = IllegalArgumentException.class) + public void testStringCtorEmpty() { + new DownsamplingSpecification(""); + } + + @Test(expected = IllegalArgumentException.class) + public void testStringCtorNoIntervalString() { + new DownsamplingSpecification("blah-avg-lerp"); + } + + @Test(expected = IllegalArgumentException.class) + public void testStringCtorZeroInterval() { + new DownsamplingSpecification("0m-avg-lerp"); + } + + @Test(expected = IllegalArgumentException.class) + public void testStringCtorNegativeInterval() { + new DownsamplingSpecification("-60m-avg-lerp"); + } + + @Test(expected = IllegalArgumentException.class) + public void testStringCtorUnknownUnits() { + new DownsamplingSpecification("1j-avg-lerp"); + } + + @Test(expected = IllegalArgumentException.class) + public void testStringCtorMissingUnits() { + new DownsamplingSpecification("1-avg-lerp"); + } + + @Test(expected = IllegalArgumentException.class) + public void testStringCtorBadFunction() { + new DownsamplingSpecification("1m-hurp-lerp"); + } + + @Test(expected = IllegalArgumentException.class) + public void testStringCtorMissingFunction() { + new DownsamplingSpecification("1m"); + } + + @Test(expected = IllegalArgumentException.class) + public void testStringCtorBadFillPolicy() { + new DownsamplingSpecification("10m-avg-max"); + } + + @Test + public void testSetCalendar() { + DownsamplingSpecification ds = new DownsamplingSpecification("15m-avg"); + assertFalse(ds.useCalendar()); + + ds.setUseCalendar(true); + assertTrue(ds.useCalendar()); + + ds.setUseCalendar(false); + assertFalse(ds.useCalendar()); + } + + @Test + public void setTimezone() { + DownsamplingSpecification ds = new DownsamplingSpecification("15m-avg"); + assertEquals(timezone, ds.getTimezone()); + + final TimeZone tz = DateTime.timezones.get("America/Denver"); + ds.setTimezone(tz); + assertEquals(tz, ds.getTimezone()); + } + + @Test (expected = IllegalArgumentException.class) + public void setTimezoneNull() { + DownsamplingSpecification ds = new DownsamplingSpecification("15m-avg"); + ds.setTimezone(null); + } + + @Test + public void testToString() { + final String string = new DownsamplingSpecification( + 4532019L, + Aggregators.ZIMSUM, + FillPolicy.NOT_A_NUMBER).toString(); + + assertTrue(string.contains("interval=4532019")); + assertTrue(string.contains("function=zimsum")); + assertTrue(string.contains("fillPolicy=NOT_A_NUMBER")); + assertTrue(string.contains("useCalendar=false")); + assertTrue(string.contains("timeZone=UTC")); + } + +} + diff --git a/test/core/TestFillingDownsampler.java b/test/core/TestFillingDownsampler.java new file mode 100644 index 0000000000..dbcc056cf5 --- /dev/null +++ b/test/core/TestFillingDownsampler.java @@ -0,0 +1,1119 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import org.junit.Test; + +import net.opentsdb.core.SeekableViewsForTest.MockSeekableView; +import net.opentsdb.rollup.RollupInterval; +import net.opentsdb.rollup.RollupQuery; +import net.opentsdb.utils.DateTime; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.spy; + +import java.util.TimeZone; + +/** Tests {@link FillingDownsampler}. */ +public class TestFillingDownsampler { + private static final long BASE_TIME = 1356998400000L; + //30 minute offset + final static TimeZone AF = DateTime.timezones.get("Asia/Kabul"); + // 12h offset w/o DST + final static TimeZone TV = DateTime.timezones.get("Pacific/Funafuti"); + // 12h offset w DST + final static TimeZone FJ = DateTime.timezones.get("Pacific/Fiji"); + // Tue, 15 Dec 2015 04:02:25.123 UTC + final static long DST_TS = 1450137600000L; + + private SeekableView source; + private Downsampler downsampler; + private DownsamplingSpecification specification; + + /** Data with gaps: before, during, and after. */ + @Test + public void testNaNMissingInterval() { + final long baseTime = 500L; + final SeekableView source = + SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofDoubleValue(baseTime + 25L * 4L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 5L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 7L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 12L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 15L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 24L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 25L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 26L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 27L, 1.), + }); + + specification = new DownsamplingSpecification("100ms-sum-nan"); + final Downsampler downsampler = new FillingDownsampler(source, baseTime, + baseTime + 36 * 25L, specification, 0, 0); + + long timestamp = baseTime; + step(downsampler, timestamp, Double.NaN); + step(downsampler, timestamp += 100, 3.); + step(downsampler, timestamp += 100, Double.NaN); + step(downsampler, timestamp += 100, 2.); + step(downsampler, timestamp += 100, Double.NaN); + step(downsampler, timestamp += 100, Double.NaN); + step(downsampler, timestamp += 100, 4.); + step(downsampler, timestamp += 100, Double.NaN); + step(downsampler, timestamp += 100, Double.NaN); + assertFalse(downsampler.hasNext()); + } + + @Test + public void testZeroMissingInterval() { + final long baseTime = 500L; + final SeekableView source = + SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofDoubleValue(baseTime + 25L * 4L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 5L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 7L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 12L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 15L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 24L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 25L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 26L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 27L, 1.), + }); + + specification = new DownsamplingSpecification("100ms-sum-zero"); + final Downsampler downsampler = new FillingDownsampler(source, baseTime, + baseTime + 36 * 25L, specification, 0, 0); + + long timestamp = baseTime; + step(downsampler, timestamp, 0.); + step(downsampler, timestamp += 100, 3.); + step(downsampler, timestamp += 100, 0.); + step(downsampler, timestamp += 100, 2.); + step(downsampler, timestamp += 100, 0.); + step(downsampler, timestamp += 100, 0.); + step(downsampler, timestamp += 100, 4.); + step(downsampler, timestamp += 100, 0.); + step(downsampler, timestamp += 100, 0.); + assertFalse(downsampler.hasNext()); + } + + /** Contiguous data, i.e., nothing missing. */ + @Test + public void testWithoutMissingIntervals() { + final long baseTime = 1000L; + final SeekableView source = + SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofDoubleValue(baseTime + 25L * 0L, 12.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 1L, 11.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 2L, 10.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 3L, 9.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 4L, 8.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 5L, 7.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 6L, 6.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 7L, 5.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 8L, 4.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 9L, 3.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 10L, 2.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 11L, 1.), + }); + + specification = new DownsamplingSpecification("100ms-sum-nan"); + final Downsampler downsampler = new FillingDownsampler(source, baseTime, + baseTime + 12L * 25L, specification, 0, 0); + + long timestamp = baseTime; + step(downsampler, timestamp, 42.); + step(downsampler, timestamp += 100, 26.); + step(downsampler, timestamp += 100, 10.); + assertFalse(downsampler.hasNext()); + } + + /** Data up to five minutes out of query time bounds. */ + @Test + public void testWithOutOfBoundsData() { + final long baseTime = 1425335895000L; + final SeekableView source = + SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofDoubleValue(baseTime - 60000L * 5L + 320L, 53.), + MutableDataPoint.ofDoubleValue(baseTime - 60000L * 2L + 8839L, 16.), + + // start query + MutableDataPoint.ofDoubleValue(baseTime + 60000L * 0L + 849L, 9.), + MutableDataPoint.ofDoubleValue(baseTime + 60000L * 0L + 3849L, 8.), + MutableDataPoint.ofDoubleValue(baseTime + 60000L * 0L + 6210L, 7.), + MutableDataPoint.ofDoubleValue(baseTime + 60000L * 0L + 42216L, 6.), + MutableDataPoint.ofDoubleValue(baseTime + 60000L * 1L + 167L, 5.), + MutableDataPoint.ofDoubleValue(baseTime + 60000L * 1L + 28593L, 4.), + // end query + + MutableDataPoint.ofDoubleValue(baseTime + 60000L * 2L + 30384L, 37.), + MutableDataPoint.ofDoubleValue(baseTime + 60000L * 4L + 1530L, 86.) + }); + + specification = new DownsamplingSpecification("1m-sum-nan"); + final Downsampler downsampler = new FillingDownsampler(source, baseTime, + baseTime + 60000L * 2L, specification, 0, 0); + + long timestamp = 1425335880000L; + step(downsampler, timestamp, 30.); + step(downsampler, timestamp += 60000, 9.); + assertFalse(downsampler.hasNext()); + } + + @Test + public void testWithOutOfBoundsDataEarly() { + final long baseTime = 1425335895000L; + final SeekableView source = + SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofDoubleValue(baseTime - 60000L * 5L + 320L, 53.), + MutableDataPoint.ofDoubleValue(baseTime - 60000L * 2L + 8839L, 16.) + }); + + specification = new DownsamplingSpecification("1m-sum-nan"); + final Downsampler downsampler = new FillingDownsampler(source, baseTime, + baseTime + 60000L * 2L, specification, 0, 0); + + long timestamp = 1425335880000L; + step(downsampler, timestamp, Double.NaN); + step(downsampler, timestamp += 60000, Double.NaN); + assertFalse(downsampler.hasNext()); + } + + @Test + public void testWithOutOfBoundsDataLate() { + final long baseTime = 1425335895000L; + final SeekableView source = + SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofDoubleValue(baseTime + 60000L * 2L + 30384L, 37.), + MutableDataPoint.ofDoubleValue(baseTime + 60000L * 4L + 1530L, 86.) + }); + + specification = new DownsamplingSpecification("1m-sum-nan"); + final Downsampler downsampler = new FillingDownsampler(source, baseTime, + baseTime + 60000L * 2L, specification, 0, 0); + + long timestamp = 1425335880000L; + step(downsampler, timestamp, Double.NaN); + step(downsampler, timestamp += 60000, Double.NaN); + assertFalse(downsampler.hasNext()); + } + + @Test + public void testDownsampler_allFullRange() { + final SeekableView source = SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(BASE_TIME + 5000L, 1), + MutableDataPoint.ofLongValue(BASE_TIME + 15000L, 2), + MutableDataPoint.ofLongValue(BASE_TIME + 25000L, 4), + MutableDataPoint.ofLongValue(BASE_TIME + 35000L, 8), + MutableDataPoint.ofLongValue(BASE_TIME + 45000L, 16), + MutableDataPoint.ofLongValue(BASE_TIME + 55000L, 32) + }); + + specification = new DownsamplingSpecification("0all-sum-nan"); + final Downsampler downsampler = new FillingDownsampler(source, + BASE_TIME + 5000L,BASE_TIME + 55000L, specification, 0, + Long.MAX_VALUE); + + step(downsampler, 0, 63); + assertFalse(downsampler.hasNext()); + } + + @Test + public void testDownsampler_allFilterOnQuery() { + final SeekableView source = SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(BASE_TIME + 5000L, 1), + MutableDataPoint.ofLongValue(BASE_TIME + 15000L, 2), + MutableDataPoint.ofLongValue(BASE_TIME + 25000L, 4), + MutableDataPoint.ofLongValue(BASE_TIME + 35000L, 8), + MutableDataPoint.ofLongValue(BASE_TIME + 45000L, 16), + MutableDataPoint.ofLongValue(BASE_TIME + 55000L, 32) + }); + + specification = new DownsamplingSpecification("0all-sum-nan"); + final Downsampler downsampler = new FillingDownsampler(source, + BASE_TIME + 5000L,BASE_TIME + 55000L, specification, + BASE_TIME + 15000L, BASE_TIME + 45000L); + + step(downsampler, BASE_TIME + 15000L, 14); + assertFalse(downsampler.hasNext()); + } + + @Test + public void testDownsampler_allFilterOnQueryOutOfRangeEarly() { + final SeekableView source = SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(BASE_TIME + 5000L, 1), + MutableDataPoint.ofLongValue(BASE_TIME + 15000L, 2), + MutableDataPoint.ofLongValue(BASE_TIME + 25000L, 4), + MutableDataPoint.ofLongValue(BASE_TIME + 35000L, 8), + MutableDataPoint.ofLongValue(BASE_TIME + 45000L, 16), + MutableDataPoint.ofLongValue(BASE_TIME + 55000L, 32) + }); + + specification = new DownsamplingSpecification("0all-sum-nan"); + final Downsampler downsampler = new FillingDownsampler(source, + BASE_TIME + 5000L,BASE_TIME + 55000L, specification, + BASE_TIME + 65000L, BASE_TIME + 75000L); + + assertFalse(downsampler.hasNext()); + } + + @Test + public void testDownsampler_allFilterOnQueryOutOfRangeLate() { + final SeekableView source = SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(BASE_TIME + 5000L, 1), + MutableDataPoint.ofLongValue(BASE_TIME + 15000L, 2), + MutableDataPoint.ofLongValue(BASE_TIME + 25000L, 4), + MutableDataPoint.ofLongValue(BASE_TIME + 35000L, 8), + MutableDataPoint.ofLongValue(BASE_TIME + 45000L, 16), + MutableDataPoint.ofLongValue(BASE_TIME + 55000L, 32) + }); + + specification = new DownsamplingSpecification("0all-sum-nan"); + final Downsampler downsampler = new FillingDownsampler(source, + BASE_TIME + 5000L,BASE_TIME + 55000L, specification, + BASE_TIME - 15000L, BASE_TIME - 5000L); + + assertFalse(downsampler.hasNext()); + } + + @Test + public void testDownsampler_calendarHour() { + source = SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(BASE_TIME, 1), + MutableDataPoint.ofLongValue(BASE_TIME + 1800000, 2), + MutableDataPoint.ofLongValue(BASE_TIME + 3599000L, 3), + MutableDataPoint.ofLongValue(BASE_TIME + 3600000L, 4), + MutableDataPoint.ofLongValue(BASE_TIME + 5400000L, 5), + MutableDataPoint.ofLongValue(BASE_TIME + 7199000L, 6) + }); + specification = new DownsamplingSpecification("1hc-sum-nan"); + specification.setTimezone(TV); + downsampler = new FillingDownsampler(source, + BASE_TIME, BASE_TIME + (3600000 * 3), specification, 0, Long.MAX_VALUE); + + long ts = BASE_TIME; + double value = 6; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + ts += 3600000; + if (value == 6) { + value = 15; + } else { + value = Double.NaN; + } + } + + // hour offset by 30m + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("1hc-sum-nan"); + specification.setTimezone(AF); + downsampler = new FillingDownsampler(source, 1356996600000L, + 1356996600000L + (3600000 * 4), specification, 0, Long.MAX_VALUE); + + ts = 1356996600000L; + value = 1; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + ts += 3600000; + if (value == 1) { + value = 9; + } else if (value == 9) { + value = 11; + } else { + value = Double.NaN; + } + } + + // multiple hours + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("4hc-sum-nan"); + specification.setTimezone(AF); + downsampler = new FillingDownsampler(source, 1356996600000L, + 1356996600000L + (3600000 * 8), specification, 0, Long.MAX_VALUE); + + ts = 1356996600000L; + value = 21; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + ts = 1357011000000L; + value = Double.NaN; + } + } + + @Test + public void testDownsampler_calendarDay() { + source = SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(DST_TS, 1), + MutableDataPoint.ofLongValue(DST_TS + 86399000, 2), + MutableDataPoint.ofLongValue(DST_TS + 126001000L, 3), // falls to the next in FJ + MutableDataPoint.ofLongValue(DST_TS + 172799000L, 4), + MutableDataPoint.ofLongValue(DST_TS + 172800000L, 5), + MutableDataPoint.ofLongValue(DST_TS + 242999000L, 6) // falls within 30m offset + }); + + // control + specification = new DownsamplingSpecification("1d-sum-nan"); + downsampler = new FillingDownsampler(source, DST_TS, + DST_TS + (86400000 * 4), specification, 0, Long.MAX_VALUE); + + long ts = DST_TS; + double value = 3; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + ts += 86400000; + if (value == 3) { + value = 7; + } else if (value == 7) { + value = 11; + } else { + value = Double.NaN; + } + } + + // 12 hour offset from UTC + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("1dc-sum-nan"); + specification.setTimezone(TV); + downsampler = new FillingDownsampler(source, 1450094400000L - 86400000, + DST_TS + (86400000 * 5), specification, 0, Long.MAX_VALUE); + + ts = 1450094400000L - 86400000; // make sure we front-fill too + value = Double.NaN; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + ts += 86400000; + if (Double.isNaN(value)) { + value = 1; + } else if (value == 1) { + value = 5; + } else if (value == 5) { + value = 9; + } else if (value == 9) { + value = 6; + } else { + value = Double.NaN; + } + } + + // 11 hour offset from UTC + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("1dc-sum-nan"); + specification.setTimezone(FJ); + downsampler = new FillingDownsampler(source, 1450094400000L, + DST_TS + (86400000 * 5), specification, 0, Long.MAX_VALUE); + + ts = 1450090800000L; + value = 1; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + ts += 86400000; + if (value == 1) { + value = 2; + } else if (value == 2) { + value = 12; + } else if (value == 12) { + value = 6; + } else { + value = Double.NaN; + } + } + + // 30m offset + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("1dc-sum-nan"); + specification.setTimezone(AF); + downsampler = new FillingDownsampler(source, 1450121400000L, + DST_TS + (86400000 * 4), specification, 0, Long.MAX_VALUE); + + ts = 1450121400000L; + value = 1; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + ts += 86400000; + if (value == 1) { + value = 5; + } else if (value == 5) { + value = 15; + } else { + value = Double.NaN; + } + } + + // multiple days + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("3dc-sum-nan"); + specification.setTimezone(AF); + downsampler = new FillingDownsampler(source, 1450121400000L, + DST_TS + (86400000 * 6), specification, 0, Long.MAX_VALUE); + + ts = 1450121400000L; + value = 21; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + ts += 86400000L * 3; + value = Double.NaN; + } + } + + @Test + public void testDownsampler_calendarWeek() { + source = SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(DST_TS, 1), // a Tuesday in UTC land + MutableDataPoint.ofLongValue(DST_TS + (86400000L * 7), 2), + MutableDataPoint.ofLongValue(1451129400000L, 3), // falls to the next in FJ + MutableDataPoint.ofLongValue(DST_TS + (86400000L * 21), 4), + MutableDataPoint.ofLongValue(1452367799000L, 5) // falls within 30m offset + }); + // control + specification = new DownsamplingSpecification("1wc-sum-nan"); + downsampler = new FillingDownsampler(source, 1449964800000L, + DST_TS + (86400000L * 35), specification, 0, Long.MAX_VALUE); + + long ts = 1449964800000L; + double value = 1; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + ts += 86400000L * 7; + if (value == 1) { + value = 5; + } else if (value == 5) { + value = Double.NaN; + } else if (Double.isNaN(value)) { + value = 9; + } else { + value = Double.NaN; + } + } + + // 12 hour offset from UTC + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("1wc-sum-nan"); + specification.setTimezone(TV); + downsampler = new FillingDownsampler(source, 1449964800000L, + DST_TS + (86400000L * 35), specification, 0, Long.MAX_VALUE); + + ts = 1449921600000L; + value = 1; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + ts += 86400000L * 7; + if (value == 1) { + value = 5; + } else if (value == 5) { + value = Double.NaN; + } else if (Double.isNaN(value)) { + value = 4; + } else { + value = 5; + } + } + + // 11 hour offset from UTC + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("1wc-sum-nan"); + specification.setTimezone(FJ); + downsampler = new FillingDownsampler(source, 1449964800000L, + DST_TS + (86400000L * 35), specification, 0, Long.MAX_VALUE); + + ts = 1449918000000L; + value = 1; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + ts += 86400000L * 7; + value++; + } + + // 30m offset + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("1wc-sum-nan"); + specification.setTimezone(AF); + downsampler = new FillingDownsampler(source, 1449964800000L, + DST_TS + (86400000L * 35), specification, 0, Long.MAX_VALUE); + + ts = 1449948600000L; + value = 1; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + ts += 86400000L * 7; + if (value == 1) { + value = 5; + } else if (value == 5) { + value = Double.NaN; + } else if (Double.isNaN(value)) { + value = 9; + } else { + value = Double.NaN; + } + } + + // multiple weeks + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("2wc-sum-nan"); + specification.setTimezone(AF); + downsampler = new FillingDownsampler(source, 1449964800000L, + DST_TS + (86400000L * 35), specification, 0, Long.MAX_VALUE); + + ts = 1449948600000L; + value = 6; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + ts += 86400000L * 14; + if (value == 6) { + value = 9; + } else { + value = Double.NaN; + } + } + } + + @Test + public void testDownsampler_calendarMonth() { + final long dec_1st = 1448928000000L; + source = spy(SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(dec_1st, 1), + MutableDataPoint.ofLongValue(1451559600000L, 2), // falls to the next in FJ + MutableDataPoint.ofLongValue(1451606400000L, 3), // jan 1st + MutableDataPoint.ofLongValue(1454284800000L, 4), // feb 1st + MutableDataPoint.ofLongValue(1456704000000L, 5), // feb 29th (leap year) + MutableDataPoint.ofLongValue(1456772400000L, 6) // falls within 30m offset AF + })); + + // control + specification = new DownsamplingSpecification("1n-sum-nan"); + downsampler = new FillingDownsampler(source, dec_1st, + dec_1st + (2592000000L * 5), specification, 0, Long.MAX_VALUE); + + long ts = dec_1st; + double value = 1; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + ts += 2592000000L; + if (value == 1) { + value = 5; + } else if (value == 5) { + value = 4; + } else if (value == 4) { + value = 11; + } else { + value = Double.NaN; + } + } + + // 12h offset + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("1nc-sum-nan"); + specification.setTimezone(TV); + downsampler = new FillingDownsampler(source, dec_1st, + dec_1st + (2592000000L * 6), specification, 0, Long.MAX_VALUE); + + ts = 1448884800000L; + value = 3; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + if (ts == 1448884800000L) { + ts = 1451563200000L; + } else if (ts == 1451563200000L) { + ts = 1454241600000L; + value = 9; + } else if (ts == 1454241600000L) { + ts = 1456747200000L; + value = 6; + } else { + ts = 1459425600000L; + value = Double.NaN; + } + } + + // 11h offset + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("1nc-sum-nan"); + specification.setTimezone(FJ); + downsampler = new FillingDownsampler(source, dec_1st, + dec_1st + (2592000000L * 6), specification, 0, Long.MAX_VALUE); + + ts = 1448881200000L; + value = 1; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + if (ts == 1448881200000L) { + ts = 1451559600000L; + value = 5; + } else if (ts == 1451559600000L) { + ts = 1454241600000L; + value = 9; + } else if (ts == 1454241600000L) { + ts = 1456747200000L; + value = 6; + } else { + ts = 1459425600000L; + value = Double.NaN; + } + } + + // 30m offset + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("1nc-sum-nan"); + specification.setTimezone(AF); + downsampler = new FillingDownsampler(source, dec_1st, + dec_1st + (2592000000L * 5), specification, 0, Long.MAX_VALUE); + + ts = 1448911800000L; + value = 3; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + if (ts == 1448911800000L) { + ts = 1451590200000L; + } else if (ts == 1451590200000L) { + ts = 1454268600000L; + value = 15; + } else { + ts = 1456774200000L; + value = Double.NaN; + } + } + + // multiple months + ((MockSeekableView)source).resetIndex(); + specification = new DownsamplingSpecification("3nc-sum-nan"); + specification.setTimezone(TV); + downsampler = new FillingDownsampler(source, dec_1st, + dec_1st + (2592000000L * 9), specification, 0, Long.MAX_VALUE); + + ts = 1443614400000L; + value = 3; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + if (ts == 1443614400000L) { + ts = 1451563200000L; + value = 18; + } else { + ts = 1459425600000L; + value = Double.NaN; + } + } + } + + @Test + public void testDownsampler_calendarSkipSomePoints() { + source = SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(BASE_TIME, 1), + MutableDataPoint.ofLongValue(BASE_TIME + 1800000, 2), + // skip an hour + MutableDataPoint.ofLongValue(BASE_TIME + 7200000, 6) + }); + specification = new DownsamplingSpecification("1hc-sum-nan"); + specification.setTimezone(TV); + downsampler = new FillingDownsampler(source, 1356998400000L, 1357009200000L, + specification, 0, Long.MAX_VALUE); + + long ts = BASE_TIME; + double value = 3; + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.001); + ts += 3600000; + if (value == 3) { + value = Double.NaN; + } else { + value = 6; + } + } + } + + @Test + public void testDownsampler_noData() { + final SeekableView source = + SeekableViewsForTest.fromArray(new DataPoint[] { }); + specification = new DownsamplingSpecification("1m-sum-nan"); + final Downsampler downsampler = new FillingDownsampler(source, BASE_TIME, + BASE_TIME + 60000L * 2L, specification, 0, 0); + + long timestamp = 1356998400000L; + step(downsampler, timestamp, Double.NaN); + step(downsampler, timestamp += 60000, Double.NaN); + assertFalse(downsampler.hasNext()); + } + + @Test + public void testDownsampler_noDataCalendar() { + final SeekableView source = + SeekableViewsForTest.fromArray(new DataPoint[] { }); + specification = new DownsamplingSpecification("1mc-sum-nan"); + final Downsampler downsampler = new FillingDownsampler(source, BASE_TIME, + BASE_TIME + 60000L * 2L, specification, 0, 0); + + long timestamp = 1356998400000L; + step(downsampler, timestamp, Double.NaN); + step(downsampler, timestamp += 60000, Double.NaN); + assertFalse(downsampler.hasNext()); + } + + @Test + public void testDownsampler_rollup() { + final RollupInterval interval = RollupInterval.builder() + .setTable("tsdb-rollup-1h") + .setPreAggregationTable("tsdb-agg-rollup-1h") + .setInterval("1h") + .setRowSpan("1d") + .build(); + final RollupQuery rollup_query = new RollupQuery(interval, Aggregators.SUM, + 3600000, Aggregators.SUM); + final long baseTime = 1000L; + final SeekableView source = + SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofDoubleValue(baseTime + 25L * 0L, 12.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 1L, 11.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 2L, 10.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 3L, 9.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 4L, 8.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 5L, 7.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 6L, 6.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 7L, 5.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 8L, 4.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 9L, 3.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 10L, 2.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 11L, 1.), + }); + + specification = new DownsamplingSpecification("100ms-sum-nan"); + final Downsampler downsampler = new FillingDownsampler(source, baseTime, + baseTime + 12L * 25L, specification, 0, 0, rollup_query); + + long timestamp = baseTime; + step(downsampler, timestamp, 42.); + step(downsampler, timestamp += 100, 26.); + step(downsampler, timestamp += 100, 10.); + assertFalse(downsampler.hasNext()); + } + + @Test + public void testDownsampler_rollupMissing() { + final RollupInterval interval = RollupInterval.builder() + .setTable("tsdb-rollup-1h") + .setPreAggregationTable("tsdb-agg-rollup-1h") + .setInterval("1h") + .setRowSpan("1d") + .build(); + final RollupQuery rollup_query = new RollupQuery(interval, Aggregators.SUM, + 3600000, Aggregators.SUM); + final long baseTime = 500L; + final SeekableView source = + SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofDoubleValue(baseTime + 25L * 4L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 5L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 7L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 12L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 15L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 24L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 25L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 26L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 27L, 1.), + }); + + specification = new DownsamplingSpecification("100ms-sum-nan"); + final Downsampler downsampler = new FillingDownsampler(source, baseTime, + baseTime + 36 * 25L, specification, 0, 0, rollup_query); + + long timestamp = baseTime; + step(downsampler, timestamp, Double.NaN); + step(downsampler, timestamp += 100, 3.); + step(downsampler, timestamp += 100, Double.NaN); + step(downsampler, timestamp += 100, 2.); + step(downsampler, timestamp += 100, Double.NaN); + step(downsampler, timestamp += 100, Double.NaN); + step(downsampler, timestamp += 100, 4.); + step(downsampler, timestamp += 100, Double.NaN); + step(downsampler, timestamp += 100, Double.NaN); + assertFalse(downsampler.hasNext()); + } + + @Test + public void testDownsampler_rollupAvg() { + final RollupInterval interval = RollupInterval.builder() + .setTable("tsdb-rollup-1h") + .setPreAggregationTable("tsdb-agg-rollup-1h") + .setInterval("1h") + .setRowSpan("1d") + .build(); + final RollupQuery rollup_query = new RollupQuery(interval, Aggregators.AVG, + 3600000, Aggregators.SUM); + final long baseTime = 1000L; + final SeekableView source = + SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofDoubleValue(baseTime + 25L * 0L, 12.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 1L, 11.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 2L, 10.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 3L, 9.), + + MutableDataPoint.ofDoubleValue(baseTime + 25L * 4L, 8.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 5L, 7.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 6L, 6.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 7L, 5.), + + MutableDataPoint.ofDoubleValue(baseTime + 25L * 8L, 4.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 9L, 3.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 10L, 2.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 11L, 1.), + }); + + specification = new DownsamplingSpecification("100ms-avg-nan"); + final Downsampler downsampler = new FillingDownsampler(source, baseTime, + baseTime + 12L * 25L, specification, 0, 0, rollup_query); + + long timestamp = baseTime; + step(downsampler, timestamp, 10.5); + step(downsampler, timestamp += 100, 6.5); + step(downsampler, timestamp += 100, 2.5); + assertFalse(downsampler.hasNext()); + } + + @Test + public void testDownsampler_rollupAvgMissing() { + final RollupInterval interval = RollupInterval.builder() + .setTable("tsdb-rollup-1h") + .setPreAggregationTable("tsdb-agg-rollup-1h") + .setInterval("1h") + .setRowSpan("1d") + .build(); + final RollupQuery rollup_query = new RollupQuery(interval, Aggregators.SUM, + 3600000, Aggregators.SUM); + final long baseTime = 500L; + final SeekableView source = + SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofDoubleValue(baseTime + 25L * 4L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 5L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 7L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 12L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 15L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 24L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 25L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 26L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 27L, 1.), + }); + + specification = new DownsamplingSpecification("100ms-avg-nan"); + final Downsampler downsampler = new FillingDownsampler(source, baseTime, + baseTime + 36 * 25L, specification, 0, 0, rollup_query); + + long timestamp = baseTime; + step(downsampler, timestamp, Double.NaN); + step(downsampler, timestamp += 100, 1.); + step(downsampler, timestamp += 100, Double.NaN); + step(downsampler, timestamp += 100, 1); + step(downsampler, timestamp += 100, Double.NaN); + step(downsampler, timestamp += 100, Double.NaN); + step(downsampler, timestamp += 100, 1.); + step(downsampler, timestamp += 100, Double.NaN); + step(downsampler, timestamp += 100, Double.NaN); + assertFalse(downsampler.hasNext()); + } + + @Test + public void testDownsampler_rollupCount() { + final RollupInterval interval = RollupInterval.builder() + .setTable("tsdb-rollup-1h") + .setPreAggregationTable("tsdb-agg-rollup-1h") + .setInterval("1h") + .setRowSpan("1d") + .build(); + + // This query, in combination with the configuration/interval above, is asking for rolled up COUNTs (i.e. already + // downsampled). These COUNTs should then be SUMmed over some interval we'll define later in a DownsamplingSpecification + final RollupQuery rollup_query = new RollupQuery(interval, Aggregators.COUNT, + 3600000, Aggregators.SUM); + final long baseTime = 1000L; + + // These points represent rolled up COUNTs that would be stored in the rollup table (e.g. tsdb-rollup-1h) and as + // such we don't expect these to be COUNTed again on retrieval. These points are at 25ms intervals + final SeekableView source = + SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofDoubleValue(baseTime + 25L * 0L, 12.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 1L, 11.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 2L, 10.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 3L, 9.), + + MutableDataPoint.ofDoubleValue(baseTime + 25L * 4L, 8.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 5L, 7.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 6L, 6.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 7L, 5.), + + MutableDataPoint.ofDoubleValue(baseTime + 25L * 8L, 4.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 9L, 3.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 10L, 2.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 11L, 1.), + }); + + // The rolled up points above are at 25ms intervals but we want to downsample these further so that we only get + // points at 100ms intervals + specification = new DownsamplingSpecification("100ms-count-nan"); + final Downsampler downsampler = new FillingDownsampler(source, baseTime, + baseTime + 12L * 25L, specification, 0, 0, rollup_query); + + long timestamp = baseTime; + + // Expect the SUM of points 1 to 4 + step(downsampler, timestamp, 42); + + // Expect the SUM of points 5 to 8 + step(downsampler, timestamp += 100, 26); + + // Expect the SUM of points 9 to 12 + step(downsampler, timestamp += 100, 10); + assertFalse(downsampler.hasNext()); + } + + @Test + public void testDownsampler_rollupCountMissing() { + final RollupInterval interval = RollupInterval.builder() + .setTable("tsdb-rollup-1h") + .setPreAggregationTable("tsdb-agg-rollup-1h") + .setInterval("1h") + .setRowSpan("1d") + .build(); + final RollupQuery rollup_query = new RollupQuery(interval, Aggregators.SUM, + 3600000, Aggregators.SUM); + final long baseTime = 500L; + final SeekableView source = + SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofDoubleValue(baseTime + 25L * 4L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 5L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 7L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 12L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 15L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 24L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 25L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 26L, 1.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 27L, 1.), + }); + + specification = new DownsamplingSpecification("100ms-count-nan"); + final Downsampler downsampler = new FillingDownsampler(source, baseTime, + baseTime + 36 * 25L, specification, 0, 0, rollup_query); + + long timestamp = baseTime; + step(downsampler, timestamp, Double.NaN); + step(downsampler, timestamp += 100, 3.); + step(downsampler, timestamp += 100, Double.NaN); + step(downsampler, timestamp += 100, 2.); + step(downsampler, timestamp += 100, Double.NaN); + step(downsampler, timestamp += 100, Double.NaN); + step(downsampler, timestamp += 100, 4.); + step(downsampler, timestamp += 100, Double.NaN); + step(downsampler, timestamp += 100, Double.NaN); + assertFalse(downsampler.hasNext()); + } + + @Test (expected = UnsupportedOperationException.class) + public void testDownsampler_rollupDev() { + final RollupInterval interval = RollupInterval.builder() + .setTable("tsdb-rollup-1h") + .setPreAggregationTable("tsdb-agg-rollup-1h") + .setInterval("1h") + .setRowSpan("1d") + .build(); + final RollupQuery rollup_query = new RollupQuery(interval, Aggregators.DEV, + 3600000, Aggregators.SUM); + final long baseTime = 1000L; + final SeekableView source = + SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofDoubleValue(baseTime + 25L * 0L, 12.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 1L, 11.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 2L, 10.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 3L, 9.), + + MutableDataPoint.ofDoubleValue(baseTime + 25L * 4L, 8.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 5L, 7.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 6L, 6.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 7L, 5.), + + MutableDataPoint.ofDoubleValue(baseTime + 25L * 8L, 4.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 9L, 3.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 10L, 2.), + MutableDataPoint.ofDoubleValue(baseTime + 25L * 11L, 1.), + }); + + specification = new DownsamplingSpecification("100ms-dev-nan"); + final Downsampler downsampler = new FillingDownsampler(source, baseTime, + baseTime + 12L * 25L, specification, 0, 0, rollup_query); + while (downsampler.hasNext()) { + downsampler.next(); // <-- throws here + } + } + + private void step(final Downsampler downsampler, final long expected_timestamp, + final double expected_value) { + assertTrue(downsampler.hasNext()); + final DataPoint point = downsampler.next(); + assertEquals(expected_timestamp, point.timestamp()); + assertEquals(expected_value, point.doubleValue(), 0.01); + } +} + diff --git a/test/core/TestHistogramAggregationIterator.java b/test/core/TestHistogramAggregationIterator.java new file mode 100644 index 0000000000..7ae07c8649 --- /dev/null +++ b/test/core/TestHistogramAggregationIterator.java @@ -0,0 +1,560 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. + +package net.opentsdb.core; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.powermock.api.mockito.PowerMockito.mock; + +import java.util.ArrayList; +import java.util.List; + +import org.hbase.async.Bytes; +import org.hbase.async.KeyValue; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import net.opentsdb.uid.UniqueId; +import net.opentsdb.utils.Config; + +@RunWith(PowerMockRunner.class) +//"Classloader hell"... It's real. Tell PowerMock to ignore these classes +//because they fiddle with the class loader. We don't test them anyway. +@PowerMockIgnore({ "javax.management.*", "javax.xml.*", "ch.qos.*", + "org.slf4j.*", "com.sum.*", "org.xml.*" }) +@PrepareForTest({ HistogramSpan.class, HistogramRowSeq.class, TSDB.class, + UniqueId.class, KeyValue.class, Config.class, + RowKey.class }) + +public class TestHistogramAggregationIterator { + private TSDB tsdb = mock(TSDB.class); + private static final long BASE_TIME = 1356998400000L; + public static final byte[] KEY = + new byte[] { 0, 0, 0, 0, 1, 0x50, (byte)0xE2, 0x27, 0, 0, 0, 0, 1, 0, 0, 0, 2 }; + + @Test + public void testOneHistogramSpanWithNoDownsampler() { + List<HistogramDataPoint> row = new ArrayList<HistogramDataPoint>(); + for (int i = 0; i < 10; ++i) { + row.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, i), BASE_TIME + 5000L * i)); + } + + final HistogramSpan hspan = new HistogramSpan(tsdb); + hspan.addRow(KEY, row); + + List<HistogramSpan> spans = new ArrayList<HistogramSpan>(); + spans.add(hspan); + + HistogramAggregationIterator histAggIt = + HistogramAggregationIterator.create(spans, BASE_TIME, + BASE_TIME + 5000L * 10, HistogramAggregation.SUM, + DownsamplingSpecification.NO_DOWNSAMPLER, 0, 0, false); + + List<Long> values = new ArrayList<Long>(); + List<Long> timestamps = new ArrayList<Long>(); + while (histAggIt.hasNext()) { + HistogramDataPoint hdp = histAggIt.next(); + values.add(Bytes.getLong(hdp.getRawData(false))); + timestamps.add(hdp.timestamp()); + } // end while + + assertEquals(10, values.size()); + for (int i = 0; i < 10; ++i) { + assertEquals(i, values.get(i).longValue()); + assertEquals(BASE_TIME + 5000L * i, timestamps.get(i).longValue()); + } // end for + } // end testOneHistogramSpanWithNoDownsampler() + + @Test + public void testOneHistogramSpanWithDownsampler_10secs() { + List<HistogramDataPoint> row = new ArrayList<HistogramDataPoint>(); + for (int i = 0; i < 10; ++i) { + row.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, i), BASE_TIME + 5000L * i)); + } + + final HistogramSpan hspan = new HistogramSpan(tsdb); + hspan.addRow(KEY, row); + + List<HistogramSpan> spans = new ArrayList<HistogramSpan>(); + spans.add(hspan); + + DownsamplingSpecification specification = + new DownsamplingSpecification("10s-sum"); + HistogramAggregationIterator histAggIt = + HistogramAggregationIterator.create(spans, BASE_TIME, + BASE_TIME + 5000L * 10, HistogramAggregation.SUM, specification, 0, 0, + false); + + List<Long> values = new ArrayList<Long>(); + List<Long> timestamps_in_millis = new ArrayList<Long>(); + while (histAggIt.hasNext()) { + HistogramDataPoint hdp = histAggIt.next(); + values.add(Bytes.getLong(hdp.getRawData(false))); + timestamps_in_millis.add(hdp.timestamp()); + } // end while + + assertEquals(5, values.size()); + // 0 + 1 + assertEquals(1, values.get(0).longValue()); + assertEquals(BASE_TIME + 00000L, timestamps_in_millis.get(0).longValue()); + + // 2 + 3 + assertEquals(5, values.get(1).longValue()); + assertEquals(BASE_TIME + 10000L, timestamps_in_millis.get(1).longValue()); + + // 4 + 5 + assertEquals(9, values.get(2).longValue()); + assertEquals(BASE_TIME + 20000L, timestamps_in_millis.get(2).longValue()); + + // 6 + 7 + assertEquals(13, values.get(3).longValue()); + assertEquals(BASE_TIME + 30000L, timestamps_in_millis.get(3).longValue()); + + // 8 + 9 + assertEquals(17, values.get(4).longValue()); + assertEquals(BASE_TIME + 40000L, timestamps_in_millis.get(4).longValue()); + } // end testOneHistogramSpanWithDownsampler_10secs() + + @Test + public void testOneHistogramSpanNoDownSamplerSkipEarlyDataPoints() { + List<HistogramDataPoint> row = new ArrayList<HistogramDataPoint>(); + for (int i = 0; i < 10; ++i) { + row.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, i), BASE_TIME + 5000L * i)); + } + + final HistogramSpan hspan = new HistogramSpan(tsdb); + hspan.addRow(KEY, row); + + List<HistogramSpan> spans = new ArrayList<HistogramSpan>(); + spans.add(hspan); + + HistogramAggregationIterator histAggIt = + HistogramAggregationIterator.create(spans, BASE_TIME + 5000L, + BASE_TIME + 5000L * 10, HistogramAggregation.SUM, + DownsamplingSpecification.NO_DOWNSAMPLER, 0, 0, false); + + List<Long> values = new ArrayList<Long>(); + List<Long> timestamps_in_millis = new ArrayList<Long>(); + while (histAggIt.hasNext()) { + HistogramDataPoint hdp = histAggIt.next(); + values.add(Bytes.getLong(hdp.getRawData(false))); + timestamps_in_millis.add(hdp.timestamp()); + } // end while + + assertEquals(9, values.size()); + for (int i = 0; i < values.size(); ++i) { + assertEquals(i + 1, values.get(i).longValue()); + assertEquals(BASE_TIME + 5000L * (i + 1), + timestamps_in_millis.get(i).longValue()); + } + } // end testOneHistogramSpanNoDownSamplerSkipEarlyDataPoints() + + @Test + public void testOneHistogramSpanNoDownSamplerOutofRange() { + List<HistogramDataPoint> row = new ArrayList<HistogramDataPoint>(); + for (int i = 0; i < 10; ++i) { + row.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, i), BASE_TIME + 5000L * i)); + } + + final HistogramSpan hspan = new HistogramSpan(tsdb); + hspan.addRow(KEY, row); + + List<HistogramSpan> spans = new ArrayList<HistogramSpan>(); + spans.add(hspan); + + HistogramAggregationIterator histAggIt = + HistogramAggregationIterator.create(spans, BASE_TIME + 5000L * 10, + BASE_TIME + 5000L * 20, HistogramAggregation.SUM, + DownsamplingSpecification.NO_DOWNSAMPLER, 0, 0, false); + assertFalse(histAggIt.hasNext()); + } // end testOneHistogramSpanNoDownSamplerOutofRange() + + @Test + public void testOneHistogramSpanNoDownSamplerLaterDataPoints() { + List<HistogramDataPoint> row = new ArrayList<HistogramDataPoint>(); + for (int i = 0; i < 10; ++i) { + row.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, i), BASE_TIME + 5000L * i)); + } + + final HistogramSpan hspan = new HistogramSpan(tsdb); + hspan.addRow(KEY, row); + + List<HistogramSpan> spans = new ArrayList<HistogramSpan>(); + spans.add(hspan); + + HistogramAggregationIterator histAggIt = + HistogramAggregationIterator.create(spans, BASE_TIME, + BASE_TIME + 5000L * 5, HistogramAggregation.SUM, + DownsamplingSpecification.NO_DOWNSAMPLER, 0, 0, false); + + List<Long> values = new ArrayList<Long>(); + List<Long> timestamps_in_millis = new ArrayList<Long>(); + while (histAggIt.hasNext()) { + HistogramDataPoint hdp = histAggIt.next(); + values.add(Bytes.getLong(hdp.getRawData(false))); + timestamps_in_millis.add(hdp.timestamp()); + } // end while + + assertEquals(6, values.size()); + for (int i = 0; i < values.size(); ++i) { + assertEquals(i, values.get(i).longValue()); + assertEquals(BASE_TIME + 5000L * i, timestamps_in_millis.get(i).longValue()); + } + } // end testOneHistogramSpanNoDownSamplerLaterDataPoints() + + @Test + public void testOneHistogramSpanDownSamplerLaterDataPoints() { + List<HistogramDataPoint> row = new ArrayList<HistogramDataPoint>(); + for (int i = 0; i < 10; ++i) { + row.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, i), BASE_TIME + 5000L * i)); + } + + final HistogramSpan hspan = new HistogramSpan(tsdb); + hspan.addRow(KEY, row); + + List<HistogramSpan> spans = new ArrayList<HistogramSpan>(); + spans.add(hspan); + + DownsamplingSpecification specification = + new DownsamplingSpecification("10s-sum"); + HistogramAggregationIterator histAggIt = + HistogramAggregationIterator.create(spans, BASE_TIME, + BASE_TIME + 5000L * 5, HistogramAggregation.SUM, specification, 0, 0, false); + + List<Long> values = new ArrayList<Long>(); + List<Long> timestamps_in_millis = new ArrayList<Long>(); + while (histAggIt.hasNext()) { + HistogramDataPoint hdp = histAggIt.next(); + values.add(Bytes.getLong(hdp.getRawData(false))); + timestamps_in_millis.add(hdp.timestamp()); + } // end while + + assertEquals(3, values.size()); + // 0 + 1 + assertEquals(1, values.get(0).longValue()); + assertEquals(BASE_TIME + 00000L, timestamps_in_millis.get(0).longValue()); + + // 2 + 3 + assertEquals(5, values.get(1).longValue()); + assertEquals(BASE_TIME + 10000L, timestamps_in_millis.get(1).longValue()); + + // 4 + 5 + assertEquals(9, values.get(2).longValue()); + assertEquals(BASE_TIME + 20000L, timestamps_in_millis.get(2).longValue()); + } // end testOneHistogramSpanWithLaterDataPoints() + + @Test + public void testTwoHistogramSpanNoDownSamplerSameTimestamp() { + List<HistogramDataPoint> row = new ArrayList<HistogramDataPoint>(); + for (int i = 0; i < 10; ++i) { + row.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, i), BASE_TIME + 5000L * i)); + } + + final HistogramSpan hspan = new HistogramSpan(tsdb); + hspan.addRow(KEY, row); + + List<HistogramSpan> spans = new ArrayList<HistogramSpan>(); + spans.add(hspan); + + List<HistogramDataPoint> row2 = new ArrayList<HistogramDataPoint>(); + for (int i = 0; i < 10; ++i) { + row2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, i), BASE_TIME + 5000L * i)); + } + + final HistogramSpan hspan2 = new HistogramSpan(tsdb); + hspan2.addRow(KEY, row2); + spans.add(hspan2); + + HistogramAggregationIterator histAggIt = + HistogramAggregationIterator.create(spans, BASE_TIME, + BASE_TIME + 5000L * 10, HistogramAggregation.SUM, + DownsamplingSpecification.NO_DOWNSAMPLER, 0, 0, false); + + List<Long> values = new ArrayList<Long>(); + List<Long> timestamps_in_millis = new ArrayList<Long>(); + while (histAggIt.hasNext()) { + HistogramDataPoint hdp = histAggIt.next(); + values.add(Bytes.getLong(hdp.getRawData(false))); + timestamps_in_millis.add(hdp.timestamp()); + } // end while + + assertEquals(10, values.size()); + for (int i = 0; i < 10; ++i) { + assertEquals(i * 2, values.get(i).longValue()); + assertEquals(BASE_TIME + 5000L * i, timestamps_in_millis.get(i).longValue()); + } + } // end testTwoHistogramSpanNoDownSamplerSameTimestamp() + + @Test + public void testTwoHistogramSpanDownSamplerSameTimestamp() { + List<HistogramDataPoint> row = new ArrayList<HistogramDataPoint>(); + for (int i = 0; i < 10; ++i) { + row.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, i), BASE_TIME + 5000L * i)); + } + + final HistogramSpan hspan = new HistogramSpan(tsdb); + hspan.addRow(KEY, row); + + List<HistogramSpan> spans = new ArrayList<HistogramSpan>(); + spans.add(hspan); + + List<HistogramDataPoint> row2 = new ArrayList<HistogramDataPoint>(); + for (int i = 0; i < 10; ++i) { + row2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, i), BASE_TIME + 5000L * i)); + } + + final HistogramSpan hspan2 = new HistogramSpan(tsdb); + hspan2.addRow(KEY, row2); + spans.add(hspan2); + + DownsamplingSpecification specification = + new DownsamplingSpecification("10s-sum"); + HistogramAggregationIterator histAggIt = + HistogramAggregationIterator.create(spans, BASE_TIME, + BASE_TIME + 5000L * 10, HistogramAggregation.SUM, specification, 0, 0, false); + + List<Long> values = new ArrayList<Long>(); + List<Long> timestamps_in_millis = new ArrayList<Long>(); + while (histAggIt.hasNext()) { + HistogramDataPoint hdp = histAggIt.next(); + values.add(Bytes.getLong(hdp.getRawData(false))); + timestamps_in_millis.add(hdp.timestamp()); + } // end while + + assertEquals(5, values.size()); + for (int i = 0; i < values.size(); ++i) { + assertEquals(2 * (i * 2 + i * 2 + 1), values.get(i).longValue()); + assertEquals(BASE_TIME + 5000L * 2 * i, timestamps_in_millis.get(i).longValue()); + } + } // end testTwoHistogramSpanDownSamplerSameTimestamp() + + @Test + public void testTwoHistogramSpanNoDownSamplerDiffTimestamp() { + List<HistogramDataPoint> row = new ArrayList<HistogramDataPoint>(); + // 0, 2, 4... + for (int i = 0; i < 10; ) { + row.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, i), BASE_TIME + 5000L * i)); + i += 2; + } + + final HistogramSpan hspan = new HistogramSpan(tsdb); + hspan.addRow(KEY, row); + + List<HistogramSpan> spans = new ArrayList<HistogramSpan>(); + spans.add(hspan); + + List<HistogramDataPoint> row2 = new ArrayList<HistogramDataPoint>(); + // 1, 3, 5... + for (int i = 1; i < 10; ) { + row2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, i), BASE_TIME + 5000L * i)); + i += 2; + } + + final HistogramSpan hspan2 = new HistogramSpan(tsdb); + hspan2.addRow(KEY, row2); + spans.add(hspan2); + + HistogramAggregationIterator histAggIt = + HistogramAggregationIterator.create(spans, BASE_TIME, + BASE_TIME + 5000L * 10, HistogramAggregation.SUM, + DownsamplingSpecification.NO_DOWNSAMPLER, 0, 0, false); + + List<Long> values = new ArrayList<Long>(); + List<Long> timestamps_in_millis = new ArrayList<Long>(); + while (histAggIt.hasNext()) { + HistogramDataPoint hdp = histAggIt.next(); + values.add(Bytes.getLong(hdp.getRawData(false))); + timestamps_in_millis.add(hdp.timestamp()); + } // end while + + assertEquals(10, values.size()); + for (int i = 0; i < values.size(); ++i) { + assertEquals(i, values.get(i).longValue()); + assertEquals(BASE_TIME + 5000L * i, timestamps_in_millis.get(i).longValue()); + } + } // end testTwoHistogramSpanNoDownSamplerDiffTimestamp() + + @Test + public void testTwoHistogramSpanNoDownSamplerMergeSome() { + List<HistogramDataPoint> row = new ArrayList<HistogramDataPoint>(); + for (int i = 0; i < 10; ++i) { + row.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, i), BASE_TIME + 5000L * i)); + } + + final HistogramSpan hspan = new HistogramSpan(tsdb); + hspan.addRow(KEY, row); + + List<HistogramSpan> spans = new ArrayList<HistogramSpan>(); + spans.add(hspan); + + List<HistogramDataPoint> row2 = new ArrayList<HistogramDataPoint>(); + for (int i = 1; i < 5; ++i) { + row2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, i), BASE_TIME + 5000L * i)); + } + + for (int i = 5; i < 10; ++i) { + row2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 5 + i), BASE_TIME + 5000L * (5 + i))); + } + + final HistogramSpan hspan2 = new HistogramSpan(tsdb); + hspan2.addRow(KEY, row2); + spans.add(hspan2); + + + HistogramAggregationIterator histAggIt = + HistogramAggregationIterator.create(spans, BASE_TIME, + BASE_TIME + 5000L * 20, HistogramAggregation.SUM, + DownsamplingSpecification.NO_DOWNSAMPLER, 0, 0, false); + + List<Long> values = new ArrayList<Long>(); + List<Long> timestamps_in_millis = new ArrayList<Long>(); + while (histAggIt.hasNext()) { + HistogramDataPoint hdp = histAggIt.next(); + values.add(Bytes.getLong(hdp.getRawData(false))); + timestamps_in_millis.add(hdp.timestamp()); + } // end while + + assertEquals(15, values.size()); + for (int i = 0; i < 5; ++i) { + assertEquals(2 * i, values.get(i).longValue()); + assertEquals(BASE_TIME + 5000L * i, timestamps_in_millis.get(i).longValue()); + } + + for (int i = 5; i < 10; ++i) { + assertEquals(i, values.get(i).longValue()); + assertEquals(BASE_TIME + 5000L * i, timestamps_in_millis.get(i).longValue()); + } + + for (int i = 10; i < 15; ++i) { + assertEquals(i, values.get(i).longValue()); + assertEquals(BASE_TIME + 5000L * i, timestamps_in_millis.get(i).longValue()); + } + } // end testTwoHistogramSpanNoDownSamplerOverlap() + + @Test + public void testTwoHistogramSpanNoDownSamplerOneHasMore() { + // span 1 has 10 data points + List<HistogramDataPoint> row = new ArrayList<HistogramDataPoint>(); + for (int i = 0; i < 10; ++i) { + row.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, i), BASE_TIME + 5000L * i)); + } + + final HistogramSpan hspan = new HistogramSpan(tsdb); + hspan.addRow(KEY, row); + + List<HistogramSpan> spans = new ArrayList<HistogramSpan>(); + spans.add(hspan); + + // span 2 has 5 data points + List<HistogramDataPoint> row2 = new ArrayList<HistogramDataPoint>(); + for (int i = 1; i < 5; ++i) { + row2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, i), BASE_TIME + 5000L * i)); + } + + final HistogramSpan hspan2 = new HistogramSpan(tsdb); + hspan2.addRow(KEY, row2); + spans.add(hspan2); + + HistogramAggregationIterator histAggIt = + HistogramAggregationIterator.create(spans, BASE_TIME, + BASE_TIME + 5000L * 20, HistogramAggregation.SUM, + DownsamplingSpecification.NO_DOWNSAMPLER, 0, 0, false); + + List<Long> values = new ArrayList<Long>(); + List<Long> timestamps_in_millis = new ArrayList<Long>(); + while (histAggIt.hasNext()) { + HistogramDataPoint hdp = histAggIt.next(); + values.add(Bytes.getLong(hdp.getRawData(false))); + timestamps_in_millis.add(hdp.timestamp()); + } // end while + + assertEquals(10, values.size()); + for (int i = 0; i < 5; ++i) { + assertEquals(2 * i, values.get(i).longValue()); + assertEquals(BASE_TIME + 5000L * i, timestamps_in_millis.get(i).longValue()); + } // end for + + for (int i = 5; i < 10; ++i) { + assertEquals(i, values.get(i).longValue()); + assertEquals(BASE_TIME + 5000L * i, timestamps_in_millis.get(i).longValue()); + } // end for + } // end testTwoHistogramSpanNoDownSamplerOneHasMore() + + @Test + public void testTwoHistogramSpanNoDownSamplerOneOutofRange() { + // span1 has 10 data points + List<HistogramDataPoint> row = new ArrayList<HistogramDataPoint>(); + for (int i = 0; i < 10; ++i) { + row.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, i), BASE_TIME + 5000L * i)); + } + + final HistogramSpan hspan = new HistogramSpan(tsdb); + hspan.addRow(KEY, row); + + List<HistogramSpan> spans = new ArrayList<HistogramSpan>(); + spans.add(hspan); + + // span 2 has 5 data points + List<HistogramDataPoint> row2 = new ArrayList<HistogramDataPoint>(); + for (int i = 1; i < 5; ++i) { + row2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, i), BASE_TIME + 5000L * i)); + } + + final HistogramSpan hspan2 = new HistogramSpan(tsdb); + hspan2.addRow(KEY, row2); + spans.add(hspan2); + + HistogramAggregationIterator histAggIt = + HistogramAggregationIterator.create(spans, BASE_TIME + 5000L * 5, + BASE_TIME + 5000L * 10, HistogramAggregation.SUM, + DownsamplingSpecification.NO_DOWNSAMPLER, 0, 0, false); + + List<Long> values = new ArrayList<Long>(); + List<Long> timestamps_in_millis = new ArrayList<Long>(); + while (histAggIt.hasNext()) { + HistogramDataPoint hdp = histAggIt.next(); + values.add(Bytes.getLong(hdp.getRawData(false))); + timestamps_in_millis.add(hdp.timestamp()); + } // end while + + assertEquals(5, values.size()); + for (int i = 0; i < values.size(); ++i) { + assertEquals(5 + i, values.get(i).longValue()); + assertEquals(BASE_TIME + 5000L * (5 + i), + timestamps_in_millis.get(i).longValue()); + } // end for + } // end testTwoHistogramSpanNoDownSamplerOneOutofRange() +} diff --git a/test/core/TestHistogramCodecManager.java b/test/core/TestHistogramCodecManager.java new file mode 100644 index 0000000000..453d7a88cb --- /dev/null +++ b/test/core/TestHistogramCodecManager.java @@ -0,0 +1,192 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.io.IOException; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.google.common.io.Files; + +import net.opentsdb.utils.Config; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({ TSDB.class, HistogramCodecManager.class, + Files.class }) +public class TestHistogramCodecManager { + + private TSDB tsdb; + private Config config; + + @Before + public void before() throws Exception { + tsdb = PowerMockito.mock(TSDB.class); + config = new Config(false); + + config.overrideConfig("tsd.core.histograms.config", + "{\"net.opentsdb.core.SimpleHistogramDecoder\": 0," + + "\"net.opentsdb.core.TestHistogramCodecManager$MockDecoder\":1}"); + when(tsdb.getConfig()).thenReturn(config); + PowerMockito.mockStatic(Files.class); + } + + @Test + public void ctor() throws Exception { + HistogramCodecManager manager = + new HistogramCodecManager(tsdb); + assertEquals(0, manager.getCodec(SimpleHistogramDecoder.class)); + assertEquals(1, manager.getCodec(MockDecoder.class)); + HistogramDataPointCodec codec = manager.getCodec(0); + assertEquals(0, codec.getId()); + assertTrue(codec instanceof SimpleHistogramDecoder); + codec = manager.getCodec(1); + assertEquals(1, codec.getId()); + assertTrue(codec instanceof MockDecoder); + + // bad JSON + config.overrideConfig("tsd.core.histograms.config", + "{\"net.opentsdb.core.SimpleHistogramDecoder\": "); + try { + new HistogramCodecManager(tsdb); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + + // id too small + config.overrideConfig("tsd.core.histograms.config", + "{\"net.opentsdb.core.SimpleHistogramDecoder\":-1}s"); + try { + new HistogramCodecManager(tsdb); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + + // id too big + config.overrideConfig("tsd.core.histograms.config", + "{\"net.opentsdb.core.SimpleHistogramDecoder\":256}s"); + try { + new HistogramCodecManager(tsdb); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + + // duplicate ID + config.overrideConfig("tsd.core.histograms.config", + "{\"net.opentsdb.core.SimpleHistogramDecoder\": 42," + + "\"net.opentsdb.core.TestHistogramCodecManager$MockDecoder\":42}"); + try { + new HistogramCodecManager(tsdb); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + + config.overrideConfig("tsd.core.histograms.config", "nosuchfile.json"); + when(Files.toString(any(File.class), eq(Const.UTF8_CHARSET))) + .thenReturn("{\"net.opentsdb.core.SimpleHistogramDecoder\": 0}"); + manager = new HistogramCodecManager(tsdb); + assertEquals(0, manager.getCodec(SimpleHistogramDecoder.class)); + assertTrue(manager.getCodec(0) instanceof SimpleHistogramDecoder); + + when(Files.toString(any(File.class), eq(Const.UTF8_CHARSET))) + .thenThrow(new IOException("Boo!")); + try { + new HistogramCodecManager(tsdb); + fail("Expected RuntimeException"); + } catch (RuntimeException e) { } + + // no such plugin + config.overrideConfig("tsd.core.histograms.config", + "{\"net.opentsdb.core.NoSuchPlugin\":0}"); + try { + new HistogramCodecManager(tsdb); + fail("Expected IllegalStateException"); + } catch (IllegalStateException e) { } + + // bad plugin + config.overrideConfig("tsd.core.histograms.config", + "{\"net.opentsdb.core.TestHistogramCodecManager$MockDecoderBadly\":0}"); + try { + new HistogramCodecManager(tsdb); + fail("Expected IllegalStateException"); + } catch (IllegalStateException e) { } + } + + @Test + public void getDecoder() throws Exception { + final HistogramCodecManager manager = + new HistogramCodecManager(tsdb); + assertTrue(manager.getCodec(0) instanceof SimpleHistogramDecoder); + + try { + manager.getCodec(43); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + } + + @Test + public void getDecoderClass() throws Exception { + final HistogramCodecManager manager = + new HistogramCodecManager(tsdb); + assertEquals(0, manager.getCodec(SimpleHistogramDecoder.class)); + assertEquals(1, manager.getCodec(MockDecoder.class)); + + try { + manager.getCodec(MockDecoderBadly.class); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + } + + public static class MockDecoder extends HistogramDataPointCodec { + + @Override + public Histogram decode(byte[] raw_data, boolean includes_id) { + return null; + } + + + @Override + public byte[] encode(Histogram data_point, boolean include_id) { + // TODO Auto-generated method stub + return null; + } + + } + + static class MockDecoderBadly extends HistogramDataPointCodec { + + // not allowed! + public MockDecoderBadly(final long unwanted_param) { } + + @Override + public Histogram decode(byte[] raw_data, boolean includes_id) { + return null; + } + + + @Override + public byte[] encode(Histogram data_point, boolean include_id) { + // TODO Auto-generated method stub + return null; + } + + } +} diff --git a/test/core/TestHistogramDataPointsToDataPointsAdaptor.java b/test/core/TestHistogramDataPointsToDataPointsAdaptor.java new file mode 100644 index 0000000000..c3fad80dbe --- /dev/null +++ b/test/core/TestHistogramDataPointsToDataPointsAdaptor.java @@ -0,0 +1,447 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. + +package net.opentsdb.core; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.List; + +import org.hbase.async.Bytes; +import org.hbase.async.KeyValue; +import org.hbase.async.Bytes.ByteMap; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +import net.opentsdb.uid.UniqueId; +import net.opentsdb.utils.ByteSet; +import net.opentsdb.utils.Config; + +@RunWith(PowerMockRunner.class) +//"Classloader hell"... It's real. Tell PowerMock to ignore these classes +//because they fiddle with the class loader. We don't test them anyway. +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ HistogramSpanGroup.class, HistogramSpan.class, + HistogramRowSeq.class, TSDB.class, UniqueId.class, KeyValue.class, + Config.class, RowKey.class }) +public final class TestHistogramDataPointsToDataPointsAdaptor { + private static final long BASE_TIME = 1356998400000L; + public static final byte[] KEY = + new byte[] { 0, 0, 0, 0, 1, 0x50, (byte)0xE2, 0x27, 0, 0, 0, 0, 1, 0, 0, 0, 2 }; + + private static long start_ts = 1356998400L; + private static long end_ts = 1356998600L; + + private TSDB tsdb; + + @Before + public void before() { + tsdb = PowerMockito.mock(TSDB.class); + } + + @Test + public void getTagUids() throws Exception { + final ByteMap<byte[]> uids = new ByteMap<byte[]>(); + uids.put(new byte[] { 0, 0, 0, 1 }, new byte[] { 0, 0, 0, 2 }); + final HistogramSpan span = mock(HistogramSpan.class); + when(span.getTagUids()).thenReturn(uids); + + DownsamplingSpecification specification = + new DownsamplingSpecification("1dc-sum"); + final HistogramSpanGroup group = PowerMockito.spy( + new HistogramSpanGroup(tsdb, start_ts, + end_ts, null, HistogramAggregation.SUM, specification, 0, 0, 0, false, null)); + + final ArrayList<HistogramSpan> spans = Whitebox.getInternalState(group, "spans"); + spans.add(span); + + HistogramDataPointsToDataPointsAdaptor dps_ada = + new HistogramDataPointsToDataPointsAdaptor(group, 0.98f); + final ByteMap<byte[]> uids_read = dps_ada.getTagUids(); + + assertEquals(1, uids_read.size()); + assertEquals(0, Bytes.memcmp(new byte[] {0, 0, 0, 1 }, uids_read.firstKey())); + assertEquals(0, Bytes.memcmp(new byte[] {0, 0, 0, 2 }, + uids_read.firstEntry().getValue())); + } + + @Test + public void getTagUidsAggedOut() throws Exception { + final ByteMap<byte[]> uids = new ByteMap<byte[]>(); + uids.put(new byte[] { 0, 0, 0, 1 }, new byte[] { 0, 0, 0, 2 }); + final HistogramSpan span = mock(HistogramSpan.class); + when(span.getTagUids()).thenReturn(uids); + + final ByteMap<byte[]> uids2 = new ByteMap<byte[]>(); + uids2.put(new byte[] { 0, 0, 0, 1 }, new byte[] { 0, 0, 0, 3 }); + final HistogramSpan span2 = mock(HistogramSpan.class); + when(span2.getTagUids()).thenReturn(uids2); + + DownsamplingSpecification specification = + new DownsamplingSpecification("1dc-sum"); + final HistogramSpanGroup group = PowerMockito.spy( + new HistogramSpanGroup(tsdb, start_ts, + end_ts, null, HistogramAggregation.SUM, specification, 0, 0, 0, false, null)); + + final ArrayList<HistogramSpan> spans = Whitebox.getInternalState(group, "spans"); + spans.add(span); + spans.add(span2); + + HistogramDataPointsToDataPointsAdaptor dps_ada = + new HistogramDataPointsToDataPointsAdaptor(group, 0.98f); + final ByteMap<byte[]> uids_read = dps_ada.getTagUids(); + + assertEquals(0, uids_read.size()); + } + + @Test + public void getTagUidsNoSpans() throws Exception { + DownsamplingSpecification specification = + new DownsamplingSpecification("1dc-sum"); + final HistogramSpanGroup group = PowerMockito.spy( + new HistogramSpanGroup(tsdb, start_ts, + end_ts, null, HistogramAggregation.SUM, specification, 0, 0, 0, false, null)); + + HistogramDataPointsToDataPointsAdaptor dps_ada = + new HistogramDataPointsToDataPointsAdaptor(group, 0.98f); + final ByteMap<byte[]> uids_read = dps_ada.getTagUids(); + assertEquals(0, uids_read.size()); + } + + @Test + public void getAggregatedTagUidsNotAgged() throws Exception { + final ByteMap<byte[]> uids = new ByteMap<byte[]>(); + uids.put(new byte[] { 0, 0, 0, 1 }, new byte[] { 0, 0, 0, 2 }); + final HistogramSpan span = mock(HistogramSpan.class); + when(span.getTagUids()).thenReturn(uids); + + DownsamplingSpecification specification = + new DownsamplingSpecification("1dc-sum"); + final HistogramSpanGroup group = PowerMockito.spy( + new HistogramSpanGroup(tsdb, start_ts, + end_ts, null, HistogramAggregation.SUM, specification, 0, 0, 0, false, null)); + + final ArrayList<HistogramSpan> spans = + Whitebox.getInternalState(group, "spans"); + spans.add(span); + + HistogramDataPointsToDataPointsAdaptor dps_ada = + new HistogramDataPointsToDataPointsAdaptor(group, 0.98f); + final List<byte[]> uids_read = dps_ada.getAggregatedTagUids(); + + assertEquals(0, uids_read.size()); + } + + @Test + public void getAggregatedTagUids() throws Exception { + final ByteMap<byte[]> uids = new ByteMap<byte[]>(); + uids.put(new byte[] { 0, 0, 0, 1 }, new byte[] { 0, 0, 0, 2 }); + final HistogramSpan span = mock(HistogramSpan.class); + when(span.getTagUids()).thenReturn(uids); + + final ByteMap<byte[]> uids2 = new ByteMap<byte[]>(); + uids2.put(new byte[] { 0, 0, 0, 1 }, new byte[] { 0, 0, 0, 3 }); + final HistogramSpan span2 = mock(HistogramSpan.class); + when(span2.getTagUids()).thenReturn(uids2); + + DownsamplingSpecification specification = + new DownsamplingSpecification("1dc-sum"); + final HistogramSpanGroup group = PowerMockito.spy( + new HistogramSpanGroup(tsdb, start_ts, + end_ts, null, HistogramAggregation.SUM, specification, 0, 0, 0, false, null)); + + final ArrayList<HistogramSpan> spans = Whitebox.getInternalState(group, "spans"); + spans.add(span); + spans.add(span2); + + HistogramDataPointsToDataPointsAdaptor dps_ada = + new HistogramDataPointsToDataPointsAdaptor(group, 0.98f); + final List<byte[]> uids_read = dps_ada.getAggregatedTagUids(); + + assertEquals(1, uids_read.size()); + assertArrayEquals(new byte[] { 0, 0, 0, 1 }, uids_read.get(0)); + } + + @Test + public void getAggregatedTagUidsNoSpans() throws Exception { + DownsamplingSpecification specification = new + DownsamplingSpecification("1dc-sum"); + final HistogramSpanGroup group = PowerMockito.spy( + new HistogramSpanGroup(tsdb, start_ts, + end_ts, null, HistogramAggregation.SUM, specification, 0, 0, 0, false, null)); + + HistogramDataPointsToDataPointsAdaptor dps_ada = + new HistogramDataPointsToDataPointsAdaptor(group, 0.98f); + final List<byte[]> uids_read = dps_ada.getAggregatedTagUids(); + + assertEquals(0, uids_read.size()); + } + + @Test + public void getTagUidsAggedNotInQuery() throws Exception { + final ByteSet query_tags = new ByteSet(); + query_tags.add(new byte[] { 0, 0, 0, 3 }); + + final ByteMap<byte[]> uids = new ByteMap<byte[]>(); + uids.put(new byte[] { 0, 0, 0, 1 }, new byte[] { 0, 0, 0, 2 }); + final HistogramSpan span = mock(HistogramSpan.class); + when(span.getTagUids()).thenReturn(uids); + + DownsamplingSpecification specification = new + DownsamplingSpecification("1dc-sum"); + final HistogramSpanGroup group = PowerMockito.spy( + new HistogramSpanGroup(tsdb, start_ts, + end_ts, null, HistogramAggregation.SUM, specification, 0, 0, 0, false, + query_tags)); + + final ArrayList<HistogramSpan> spans = + Whitebox.getInternalState(group, "spans"); + spans.add(span); + + HistogramDataPointsToDataPointsAdaptor dps_ada = + new HistogramDataPointsToDataPointsAdaptor(group, 0.98f); + final ByteMap<byte[]> uids_read = dps_ada.getTagUids(); + assertEquals(0, uids_read.size()); + + final List<byte[]> agg_tags = dps_ada.getAggregatedTagUids(); + assertEquals(1, agg_tags.size()); + assertArrayEquals(new byte[] { 0, 0, 0, 1 }, agg_tags.get(0)); + } + + @Test + public void getTagUidsInQueryTags() throws Exception { + final ByteSet query_tags = new ByteSet(); + query_tags.add(new byte[] { 0, 0, 0, 1 }); + + final ByteMap<byte[]> uids = new ByteMap<byte[]>(); + uids.put(new byte[] { 0, 0, 0, 1 }, new byte[] { 0, 0, 0, 2 }); + final HistogramSpan span = mock(HistogramSpan.class); + when(span.getTagUids()).thenReturn(uids); + + DownsamplingSpecification specification = + new DownsamplingSpecification("1dc-sum"); + final HistogramSpanGroup group = PowerMockito.spy( + new HistogramSpanGroup(tsdb, start_ts, + end_ts, null, HistogramAggregation.SUM, specification, 0, 0, 0, false, + query_tags)); + + final ArrayList<HistogramSpan> spans = + Whitebox.getInternalState(group, "spans"); + spans.add(span); + + HistogramDataPointsToDataPointsAdaptor dps_ada = + new HistogramDataPointsToDataPointsAdaptor(group, 0.98f); + final ByteMap<byte[]> uids_read = dps_ada.getTagUids(); + assertEquals(1, uids_read.size()); + assertEquals(0, Bytes.memcmp(new byte[] {0, 0, 0, 1 }, uids_read.firstKey())); + assertEquals(0, Bytes.memcmp(new byte[] {0, 0, 0, 2 }, + uids_read.firstEntry().getValue())); + assertEquals(0, dps_ada.getAggregatedTagUids().size()); + } + + @Test + public void getTagUidsNullQueryTags() throws Exception { + final ByteMap<byte[]> uids = new ByteMap<byte[]>(); + uids.put(new byte[] { 0, 0, 0, 1 }, new byte[] { 0, 0, 0, 2 }); + final HistogramSpan span = mock(HistogramSpan.class); + when(span.getTagUids()).thenReturn(uids); + + DownsamplingSpecification specification = + new DownsamplingSpecification("1dc-sum"); + final HistogramSpanGroup group = PowerMockito.spy( + new HistogramSpanGroup(tsdb, start_ts, + end_ts, null, HistogramAggregation.SUM, specification, 0, 0, 0, false, null)); + + final ArrayList<HistogramSpan> spans = + Whitebox.getInternalState(group, "spans"); + spans.add(span); + + HistogramDataPointsToDataPointsAdaptor dps_ada = + new HistogramDataPointsToDataPointsAdaptor(group, 0.98f); + final ByteMap<byte[]> uids_read = dps_ada.getTagUids(); + assertEquals(1, uids_read.size()); + assertEquals(0, Bytes.memcmp(new byte[] {0, 0, 0, 1 }, uids_read.firstKey())); + assertEquals(0, Bytes.memcmp(new byte[] {0, 0, 0, 2 }, + uids_read.firstEntry().getValue())); + assertEquals(0, dps_ada.getAggregatedTagUids().size()); + } + + @Test + public void iteratorAllItems() { + List<HistogramDataPoint> row = new ArrayList<HistogramDataPoint>(); + for (int i = 0; i < 10; ++i) { + row.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, i), BASE_TIME + 5000L * i)); + } + + final HistogramSpan hspan = new HistogramSpan(tsdb); + hspan.addRow(KEY, row); + + List<HistogramSpan> spans = new ArrayList<HistogramSpan>(); + spans.add(hspan); + + final ByteSet query_tags = new ByteSet(); + query_tags.add(new byte[] { 0, 0, 0, 1 }); + HistogramSpanGroup hist_span_group = + new HistogramSpanGroup(tsdb, BASE_TIME, BASE_TIME + 5000L * 10, spans, + HistogramAggregation.SUM, + DownsamplingSpecification.NO_DOWNSAMPLER, 0, 0, 0, false, query_tags); + + HistogramDataPointsToDataPointsAdaptor dps_ada = + new HistogramDataPointsToDataPointsAdaptor(hist_span_group, 0.98f); + List<Double> values = new ArrayList<Double>(); + List<Long> timestamp_in_ms = new ArrayList<Long>(); + for (DataPoint dp : dps_ada) { + assertFalse(dp.isInteger()); + values.add(dp.doubleValue()); + timestamp_in_ms.add(dp.timestamp()); + } // end for + + assertTrue(dps_ada.isPercentile()); + List<Double> to_checks = new ArrayList<Double>(); + for (int i = 0; i < 10; ++i) { + to_checks.add(i * 0.98); + } // end for + + assertEquals(values.size(), to_checks.size()); + for (int i = 0; i < values.size(); ++i) { + assertEquals(values.get(i).doubleValue(), to_checks.get(i).doubleValue(), + 0.0001); + assertEquals(timestamp_in_ms.get(i).longValue(), BASE_TIME + 5000L * i); + } // end for + } + + @Test + public void doubleIteratorAllItems() { + List<HistogramDataPoint> row = new ArrayList<HistogramDataPoint>(); + for (int i = 0; i < 10; ++i) { + row.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, i), BASE_TIME + 5000L * i)); + } + + final HistogramSpan hspan = new HistogramSpan(tsdb); + hspan.addRow(KEY, row); + + List<HistogramSpan> spans = new ArrayList<HistogramSpan>(); + spans.add(hspan); + + final ByteSet query_tags = new ByteSet(); + query_tags.add(new byte[] { 0, 0, 0, 1 }); + HistogramSpanGroup hist_span_group = + new HistogramSpanGroup(tsdb, BASE_TIME, BASE_TIME + 5000L * 10, spans, + HistogramAggregation.SUM, DownsamplingSpecification.NO_DOWNSAMPLER, + 0, 0, 0, false, query_tags); + + HistogramDataPointsToDataPointsAdaptor dps_ada = + new HistogramDataPointsToDataPointsAdaptor(hist_span_group, 0.98f); + List<Double> values = new ArrayList<Double>(); + for (DataPoint dp : dps_ada) { + assertFalse(dp.isInteger()); + values.add(dp.doubleValue()); + } // end for + + List<Double> values2 = new ArrayList<Double>(); + for (DataPoint dp : dps_ada) { + values2.add(dp.doubleValue()); + } // end for + + assertTrue(dps_ada.isPercentile()); + List<Double> to_checks = new ArrayList<Double>(); + for (int i = 0; i < 10; ++i) { + to_checks.add(i * 0.98); + } // end for + + assertEquals(values.size(), to_checks.size()); + for (int i = 0; i < values.size(); ++i) { + assertEquals(values.get(i).doubleValue(), to_checks.get(i).doubleValue(), + 0.0001); + assertEquals(values.get(i).doubleValue(), values2.get(i).doubleValue(), + 0.0001); + } // end for + } + + @Test + public void iteratorAllItemsWithDiffPercentile() { + List<HistogramDataPoint> row = new ArrayList<HistogramDataPoint>(); + for (int i = 0; i < 10; ++i) { + row.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, i), BASE_TIME + 5000L * i)); + } + + final HistogramSpan hspan = new HistogramSpan(tsdb); + hspan.addRow(KEY, row); + + List<HistogramSpan> spans = new ArrayList<HistogramSpan>(); + spans.add(hspan); + + final ByteSet query_tags = new ByteSet(); + query_tags.add(new byte[] { 0, 0, 0, 1 }); + HistogramSpanGroup hist_span_group = + new HistogramSpanGroup(tsdb, BASE_TIME, BASE_TIME + 5000L * 10, spans, + HistogramAggregation.SUM, DownsamplingSpecification.NO_DOWNSAMPLER, + 0, 0, 0, false, query_tags); + + // 98 percentile + HistogramDataPointsToDataPointsAdaptor dps_ada_98 = + new HistogramDataPointsToDataPointsAdaptor(hist_span_group, 0.98f); + List<Double> values = new ArrayList<Double>(); + for (DataPoint dp : dps_ada_98) { + assertFalse(dp.isInteger()); + values.add(dp.doubleValue()); + } // end for + + assertTrue(dps_ada_98.isPercentile()); + List<Double> to_checks = new ArrayList<Double>(); + for (int i = 0; i < 10; ++i) { + to_checks.add(i * 0.98); + } // end for + + assertEquals(values.size(), to_checks.size()); + for (int i = 0; i < values.size(); ++i) { + assertEquals(values.get(i).doubleValue(), to_checks.get(i).doubleValue(), + 0.0001); + } // end for + + // 95 percentile + HistogramDataPointsToDataPointsAdaptor dps_ada_95 = + new HistogramDataPointsToDataPointsAdaptor(hist_span_group, 0.95f); + List<Double> values_95 = new ArrayList<Double>(); + for (DataPoint dp : dps_ada_95) { + assertFalse(dp.isInteger()); + values_95.add(dp.doubleValue()); + } // end for + + assertTrue(dps_ada_95.isPercentile()); + List<Double> to_checks_95 = new ArrayList<Double>(); + for (int i = 0; i < 10; ++i) { + to_checks_95.add(i * 0.95); + } // end for + + assertEquals(values_95.size(), to_checks_95.size()); + for (int i = 0; i < values.size(); ++i) { + assertEquals(values_95.get(i).doubleValue(), + to_checks_95.get(i).doubleValue(), 0.0001); + } // end for + } +} diff --git a/test/core/TestHistogramDownsampler.java b/test/core/TestHistogramDownsampler.java new file mode 100644 index 0000000000..c91a2ed39f --- /dev/null +++ b/test/core/TestHistogramDownsampler.java @@ -0,0 +1,1482 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.powermock.api.mockito.PowerMockito.mock; + +import java.util.Arrays; +import java.util.List; +import java.util.TimeZone; + +import com.google.common.collect.Lists; + +import net.opentsdb.uid.UniqueId; +import net.opentsdb.core.HistogramSeekableViewForTest.MockHistogramSeekableView; +import net.opentsdb.utils.Config; +import net.opentsdb.utils.DateTime; + +import org.hbase.async.Bytes; +import org.hbase.async.KeyValue; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +@RunWith(PowerMockRunner.class) +// "Classloader hell"... It's real. Tell PowerMock to ignore these classes +// because they fiddle with the class loader. We don't test them anyway. +@PowerMockIgnore({ "javax.management.*", "javax.xml.*", "ch.qos.*", + "org.slf4j.*", "com.sum.*", "org.xml.*" }) +@PrepareForTest({ HistogramSpan.class, HistogramRowSeq.class, TSDB.class, + UniqueId.class, KeyValue.class, Config.class, + RowKey.class }) +public class TestHistogramDownsampler { + private TSDB tsdb = mock(TSDB.class); + private Config config = mock(Config.class); + private UniqueId metrics = mock(UniqueId.class); + + private static final long BASE_TIME = 1356998400000L; + + private static final HistogramDataPoint[] HIST_DATA_POINTS = + new HistogramDataPoint[] { + // timestamp = 1,356,998,400,000 ms + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 40L), BASE_TIME), + // timestamp = 1,357,000,400,000 ms + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 50L), BASE_TIME + 2000000), + // timestamp = 1,357,002,000,000 ms + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 40L), BASE_TIME + 3600000), + // timestamp = 1,357,002,005,000 ms + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 50L), BASE_TIME + 3605000), + // timestamp = 1,357,005,600,000 ms + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 40L), BASE_TIME + 7200000), + // timestamp = 1,357,007,600,000 ms + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 50L), BASE_TIME + 9200000) + }; + + public static final byte[] KEY = + new byte[] { 0, 0, 0, 0, 1, 0x50, (byte)0xE2, 0x27, 0, 0, 0, 0, 1, 0, 0, 0, 2 }; + + // 30 minute offset + final static TimeZone AF = DateTime.timezones.get("Asia/Kabul"); + + // 12h offset w/o DST + final static TimeZone TV = DateTime.timezones.get("Pacific/Funafuti"); + + // 12h offset w DST + final static TimeZone FJ = DateTime.timezones.get("Pacific/Fiji"); + + // Tue, 15 Dec 2015 04:02:25.123 UTC + final static long DST_TS = 1450137600000L; + + private HistogramSeekableView source; + private HistogramDownsampler downsampler; + private DownsamplingSpecification specification; + + @Before + public void before() { + // Inject the attributes we need into the "tsdb" object. + Whitebox.setInternalState(tsdb, "metrics", metrics); + Whitebox.setInternalState(tsdb, "config", config); + when(tsdb.getConfig()).thenReturn(config); + when(tsdb.metrics.width()).thenReturn((short)4); + + source = spy(HistogramSeekableViewForTest.fromArray(HIST_DATA_POINTS)); + } + + @Test + public void testDownsampler() { + specification = new DownsamplingSpecification("1000s-sum"); + downsampler = new HistogramDownsampler(source, specification, 0, 0); + verify(source, never()).next(); + List<Long> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + values.add(Bytes.getLong(hdp.getRawData(false))); + timestamps_in_millis.add(hdp.timestamp()); + } + + assertEquals(5, values.size()); + assertEquals(40L, values.get(0).longValue()); + assertEquals(BASE_TIME - 400000L, timestamps_in_millis.get(0).longValue()); + assertEquals(50, values.get(1).longValue()); + assertEquals(BASE_TIME + 1600000, timestamps_in_millis.get(1).longValue()); + assertEquals(90, values.get(2).longValue()); + assertEquals(BASE_TIME + 3600000L, timestamps_in_millis.get(2).longValue()); + assertEquals(40, values.get(3).longValue()); + assertEquals(BASE_TIME + 6600000L, timestamps_in_millis.get(3).longValue()); + assertEquals(50, values.get(4).longValue()); + assertEquals(BASE_TIME + 8600000L, timestamps_in_millis.get(4).longValue()); + } + + @Test + public void testDownsampler_10seconds() { + source = spy(HistogramSeekableViewForTest + .fromArray(new HistogramDataPoint[] { + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1L), BASE_TIME + 5000L * 0), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2L), BASE_TIME + 5000L * 1), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 4L), BASE_TIME + 5000L * 2), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 8L), BASE_TIME + 5000L * 3), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 16L), BASE_TIME + 5000L * 4), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 32L), BASE_TIME + 5000L * 5), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 64L), BASE_TIME + 5000L * 6), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 128L), BASE_TIME + 5000L * 7), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 256L), BASE_TIME + 5000L * 8), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 512L), BASE_TIME + 5000L * 9), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1024L), BASE_TIME + 5000L * 10) + })); + + specification = new DownsamplingSpecification("10s-sum"); + downsampler = new HistogramDownsampler(source, specification, 0, 0); + verify(source, never()).next(); + List<Long> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + values.add(Bytes.getLong(hdp.getRawData(false))); + timestamps_in_millis.add(hdp.timestamp()); + } + + assertEquals(6, values.size()); + assertEquals(3, values.get(0).longValue()); + assertEquals(BASE_TIME + 00000L, timestamps_in_millis.get(0).longValue()); + assertEquals(12, values.get(1).longValue()); + assertEquals(BASE_TIME + 10000L, timestamps_in_millis.get(1).longValue()); + assertEquals(48, values.get(2).longValue()); + assertEquals(BASE_TIME + 20000L, timestamps_in_millis.get(2).longValue()); + assertEquals(192, values.get(3).longValue()); + assertEquals(BASE_TIME + 30000L, timestamps_in_millis.get(3).longValue()); + assertEquals(768, values.get(4).longValue()); + assertEquals(BASE_TIME + 40000L, timestamps_in_millis.get(4).longValue()); + assertEquals(1024, values.get(5).longValue()); + assertEquals(BASE_TIME + 50000L, timestamps_in_millis.get(5).longValue()); + } + + @Test + public void testDownsampler_15seconds() { + source = spy(HistogramSeekableViewForTest + .fromArray(new HistogramDataPoint[] { + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1L), BASE_TIME + 5000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2L), BASE_TIME + 15000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 4L), BASE_TIME + 25000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 8L), BASE_TIME + 35000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 16L), BASE_TIME + 45000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 32L), BASE_TIME + 55000L) })); + specification = new DownsamplingSpecification("15s-sum"); + downsampler = new HistogramDownsampler(source, specification, 0, 0); + verify(source, never()).next(); + List<Long> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + values.add(Bytes.getLong(hdp.getRawData(false))); + timestamps_in_millis.add(hdp.timestamp()); + } + + assertEquals(4, values.size()); + assertEquals(1, values.get(0).longValue()); + assertEquals(BASE_TIME + 00000L, timestamps_in_millis.get(0).longValue()); + assertEquals(6, values.get(1).longValue()); + assertEquals(BASE_TIME + 15000L, timestamps_in_millis.get(1).longValue()); + assertEquals(8, values.get(2).longValue()); + assertEquals(BASE_TIME + 30000L, timestamps_in_millis.get(2).longValue()); + assertEquals(48, values.get(3).longValue()); + assertEquals(BASE_TIME + 45000L, timestamps_in_millis.get(3).longValue()); + } + + @Test + public void testDownsampler_allFullRange() { + source = spy(HistogramSeekableViewForTest + .fromArray(new HistogramDataPoint[] { + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1L), BASE_TIME + 5000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2L), BASE_TIME + 15000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 4L), BASE_TIME + 25000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 8L), BASE_TIME + 35000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 16L), BASE_TIME + 45000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 32L), BASE_TIME + 55000L) })); + specification = new DownsamplingSpecification("0all-sum"); + downsampler = new HistogramDownsampler(source, specification, 0, Long.MAX_VALUE); + verify(source, never()).next(); + List<Long> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + values.add(Bytes.getLong(hdp.getRawData(false))); + timestamps_in_millis.add(hdp.timestamp()); + } + + assertEquals(1, values.size()); + assertEquals(63, values.get(0).longValue()); + assertEquals(0L, timestamps_in_millis.get(0).longValue()); + } + + @Test + public void testDownsampler_allFilterOnQuery() { + source = spy(HistogramSeekableViewForTest + .fromArray(new HistogramDataPoint[] { + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1L), BASE_TIME + 5000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2L), BASE_TIME + 15000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 4L), BASE_TIME + 25000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 8L), BASE_TIME + 35000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 16L), BASE_TIME + 45000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 32L), BASE_TIME + 55000L) })); + specification = new DownsamplingSpecification("0all-sum"); + downsampler = new HistogramDownsampler(source, specification, + BASE_TIME + 15000L, BASE_TIME + 45000L); + verify(source, never()).next(); + List<Long> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + values.add(Bytes.getLong(hdp.getRawData(false))); + timestamps_in_millis.add(hdp.timestamp()); + } + + assertEquals(1, values.size()); + assertEquals(14, values.get(0).longValue()); + assertEquals(BASE_TIME + 15000L, timestamps_in_millis.get(0).longValue()); + } + + @Test + public void testDownsampler_allFilterOnQueryOutOfRangeEarly() { + source = spy(HistogramSeekableViewForTest + .fromArray(new HistogramDataPoint[] { + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1L), BASE_TIME + 5000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2L), BASE_TIME + 15000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 4L), BASE_TIME + 25000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 8L), BASE_TIME + 35000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 16L), BASE_TIME + 45000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 32L), BASE_TIME + 55000L) })); + specification = new DownsamplingSpecification("0all-sum"); + downsampler = new HistogramDownsampler(source, specification, + BASE_TIME + 65000L, BASE_TIME + 75000L); + verify(source, never()).next(); + List<Long> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + values.add(Bytes.getLong(hdp.getRawData(false))); + timestamps_in_millis.add(hdp.timestamp()); + } + + assertEquals(0, values.size()); + } + + @Test + public void testDownsampler_allFilterOnQueryOutOfRangeLate() { + source = spy(HistogramSeekableViewForTest + .fromArray(new HistogramDataPoint[] { + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1L), BASE_TIME + 5000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2L), BASE_TIME + 15000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 4L), BASE_TIME + 25000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 8L), BASE_TIME + 35000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 16L), BASE_TIME + 45000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 32L), BASE_TIME + 55000L) })); + specification = new DownsamplingSpecification("0all-sum"); + downsampler = new HistogramDownsampler(source, specification, + BASE_TIME - 15000L, BASE_TIME - 5000L); + verify(source, never()).next(); + List<Long> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + values.add(Bytes.getLong(hdp.getRawData(false))); + timestamps_in_millis.add(hdp.timestamp()); + } + + assertEquals(0, values.size()); + } + + @Test + public void testDownsampler_calendarHour() { + source = spy(HistogramSeekableViewForTest + .fromArray(new HistogramDataPoint[] { + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1L), BASE_TIME), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2L), BASE_TIME + 1800000), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 3L), BASE_TIME + 3599000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 4L), BASE_TIME + 3600000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 5L), BASE_TIME + 5400000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 6L), BASE_TIME + 7199000L) })); + specification = new DownsamplingSpecification("1hc-sum"); + specification.setTimezone(TV); + downsampler = new HistogramDownsampler(source, specification, 0, Long.MAX_VALUE); + + long ts = BASE_TIME; + long value = 6; + while (downsampler.hasNext()) { + HistogramDataPoint dp = downsampler.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(value, Bytes.getLong(dp.getRawData(false))); + ts += 3600000; + value = 15; + } + + // hour offset by 30m + ((MockHistogramSeekableView) source).resetIndex(); + specification = new DownsamplingSpecification("1hc-sum"); + specification.setTimezone(AF); + downsampler = new HistogramDownsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1356996600000L; + value = 1; + while (downsampler.hasNext()) { + HistogramDataPoint dp = downsampler.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(value, Bytes.getLong(dp.getRawData(false))); + ts += 3600000; + if (value == 1) { + value = 9; + } else { + value = 11; + } + } + + // multiple hours + ((MockHistogramSeekableView) source).resetIndex(); + specification = new DownsamplingSpecification("4hc-sum"); + specification.setTimezone(AF); + downsampler = new HistogramDownsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1356996600000L; + value = 21; + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + assertEquals(ts, hdp.timestamp()); + assertEquals(value, Bytes.getLong(hdp.getRawData(false))); + } + } + + @Test + public void testDownsampler_calendarDay() { + // UTC + source = spy(HistogramSeekableViewForTest + .fromArray(new HistogramDataPoint[] { + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1L), DST_TS), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2L), DST_TS + 86399000), + // falls to the next in FJ + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 3L), DST_TS + 126001000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 4L), DST_TS + 172799000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 5L), DST_TS + 172800000L), + // falls within 30m offset + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 6L), DST_TS + 242999000L) })); + + // control + specification = new DownsamplingSpecification("1d-sum"); + downsampler = new HistogramDownsampler(source, specification, 0, Long.MAX_VALUE); + + long ts = DST_TS; + long value = 3; + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + assertEquals(ts, hdp.timestamp()); + assertEquals(value, Bytes.getLong(hdp.getRawData(false))); + ts += 86400000; + if (value == 3) { + value = 7; + } else if (value == 7) { + value = 11; + } + } + + + // 12 hour offset from UTC + ((MockHistogramSeekableView) source).resetIndex(); + specification = new DownsamplingSpecification("1dc-sum"); + specification.setTimezone(TV); + downsampler = new HistogramDownsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1450094400000L; + value = 1; + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + assertEquals(ts, hdp.timestamp()); + assertEquals(value, Bytes.getLong(hdp.getRawData(false))); + ts += 86400000; + if (value == 1) { + value = 5; + } else if (value == 5) { + value = 9; + } else { + value = 6; + } + } + + + // 11 hour offset from UTC + ((MockHistogramSeekableView) source).resetIndex(); + specification = new DownsamplingSpecification("1dc-sum"); + specification.setTimezone(FJ); + downsampler = new HistogramDownsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1450090800000L; + value = 1; + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + assertEquals(ts, hdp.timestamp()); + assertEquals(value, Bytes.getLong(hdp.getRawData(false))); + ts += 86400000; + if (value == 1) { + value = 2; + } else if (value == 2) { + value = 12; + } else { + value = 6; + } + } + + + // 30m offset + ((MockHistogramSeekableView) source).resetIndex(); + specification = new DownsamplingSpecification("1dc-sum"); + specification.setTimezone(AF); + downsampler = new HistogramDownsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1450121400000L; + value = 1; + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + assertEquals(ts, hdp.timestamp()); + assertEquals(value, Bytes.getLong(hdp.getRawData(false))); + ts += 86400000; + if (value == 1) { + value = 5; + } else if (value == 5) { + value = 15; + } + } + + // multiple days + ((MockHistogramSeekableView) source).resetIndex(); + specification = new DownsamplingSpecification("3dc-sum"); + specification.setTimezone(AF); + downsampler = new HistogramDownsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1450121400000L; + value = 21; + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + assertEquals(ts, hdp.timestamp()); + assertEquals(value, Bytes.getLong(hdp.getRawData(false))); + } + } + + @Test + public void testDownsampler_calendarWeek() { + source = HistogramSeekableViewForTest + .fromArray(new HistogramDataPoint[] { + // a Tuesday in UTC land + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1L), DST_TS), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2L), DST_TS + (86400000L * 7)), + // falls to the next in FJ + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 3L), 1451129400000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 4L), DST_TS + (86400000L * 21)), + // falls within 30m offset + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 5L), 1452367799000L) + }); + // control + specification = new DownsamplingSpecification("1wc-sum"); + downsampler = new HistogramDownsampler(source, specification, 0, Long.MAX_VALUE); + + long ts = 1449964800000L; + long value = 1; + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + assertEquals(ts, hdp.timestamp()); + assertEquals(value, Bytes.getLong(hdp.getRawData(false))); + if (ts == 1450569600000L) { + ts = 1451779200000L; // skips a week + } else { + ts += 86400000L * 7; + } + if (value == 1) { + value = 5; + } else { + value = 9; + } + } + + // 12 hour offset from UTC + ((MockHistogramSeekableView) source).resetIndex(); + specification = new DownsamplingSpecification("1wc-sum"); + specification.setTimezone(TV); + downsampler = new HistogramDownsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1449921600000L; + value = 1; + while (downsampler.hasNext()) { + HistogramDataPoint dp = downsampler.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(value, Bytes.getLong(dp.getRawData(false))); + if (ts == 1450526400000L) { + ts = 1451736000000L; // skip a week + } else { + ts += 86400000L * 7; + } + if (value == 1) { + value = 5; + } else if (value == 5) { + value = 4; + } else { + value = 5; + } + } + + // 11 hour offset from UTC + ((MockHistogramSeekableView) source).resetIndex(); + specification = new DownsamplingSpecification("1wc-sum"); + specification.setTimezone(FJ); + downsampler = new HistogramDownsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1449918000000L; + value = 1; + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + assertEquals(ts, hdp.timestamp()); + assertEquals(value, Bytes.getLong(hdp.getRawData(false))); + ts += 86400000L * 7; + value++; + } + + // 30m offset + ((MockHistogramSeekableView) source).resetIndex(); + specification = new DownsamplingSpecification("1wc-sum"); + specification.setTimezone(AF); + downsampler = new HistogramDownsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1449948600000L; + value = 1; + while (downsampler.hasNext()) { + HistogramDataPoint dp = downsampler.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(value, Bytes.getLong(dp.getRawData(false))); + if (ts == 1449948600000L) { + ts = 1450553400000L; + } else { + ts = 1451763000000L; + } + if (value == 1) { + value = 5; + } else { + value = 9; + } + } + + // multiple weeks + ((MockHistogramSeekableView) source).resetIndex(); + specification = new DownsamplingSpecification("2wc-sum"); + specification.setTimezone(AF); + downsampler = new HistogramDownsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1449948600000L; + value = 6; + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + assertEquals(ts, hdp.timestamp()); + assertEquals(value, Bytes.getLong(hdp.getRawData(false))); + ts = 1451158200000L; + value = 9; + } + } + + @Test + public void testDownsampler_calendarMonth() { + final long dec_1st = 1448928000000L; + source = spy(HistogramSeekableViewForTest + .fromArray(new HistogramDataPoint[] { + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1L), dec_1st), + // falls to the next in FJ + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2L), 1451559600000L), + // jan 1st + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 3L), 1451606400000L), + // feb 1st + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 4L), 1454284800000L), + // feb 29th (leap year) + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 5L), 1456704000000L), + // falls within 30m offset AT + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 6L), 1456772400000L) + })); + + // control + specification = new DownsamplingSpecification("1nc-sum"); + downsampler = new HistogramDownsampler(source, specification, 0, Long.MAX_VALUE); + + long ts = dec_1st; + long value = 3; + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + assertEquals(ts, hdp.timestamp()); + assertEquals(value, Bytes.getLong(hdp.getRawData(false))); + if (ts == 1448928000000L) { + ts = 1451606400000L; + } else { + ts = 1454284800000L; + value = 15; + } + } + + // 12h offset + ((MockHistogramSeekableView) source).resetIndex(); + specification = new DownsamplingSpecification("1nc-sum"); + specification.setTimezone(TV); + downsampler = new HistogramDownsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1448884800000L; + value = 3; + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + assertEquals(ts, hdp.timestamp()); + assertEquals(value, Bytes.getLong(hdp.getRawData(false))); + if (ts == 1448884800000L) { + ts = 1451563200000L; + } else if (ts == 1451563200000L) { + value = 9; + ts = 1454241600000L; + } else { + ts = 1456747200000L; + value = 6; + } + } + + // 11h offset + ((MockHistogramSeekableView) source).resetIndex(); + specification = new DownsamplingSpecification("1nc-sum"); + specification.setTimezone(FJ); + downsampler = new HistogramDownsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1448881200000L; + value = 1; + while (downsampler.hasNext()) { + HistogramDataPoint dp = downsampler.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(value, Bytes.getLong(dp.getRawData(false))); + if (ts == 1448881200000L) { + ts = 1451559600000L; + value = 5; + } else if (ts == 1451559600000L) { + ts = 1454241600000L; + value = 9; + } else { + ts = 1456747200000L; + value = 6; + } + } + + // 30m offset + ((MockHistogramSeekableView) source).resetIndex(); + specification = new DownsamplingSpecification("1nc-sum"); + specification.setTimezone(AF); + downsampler = new HistogramDownsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1448911800000L; + value = 3; + while (downsampler.hasNext()) { + HistogramDataPoint dp = downsampler.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(value, Bytes.getLong(dp.getRawData(false))); + if (ts == 1448911800000L) { + ts = 1451590200000L; + } else { + ts = 1454268600000L; + value = 15; + } + } + + // multiple months + ((MockHistogramSeekableView) source).resetIndex(); + specification = new DownsamplingSpecification("3nc-sum"); + specification.setTimezone(TV); + downsampler = new HistogramDownsampler(source, specification, 0, Long.MAX_VALUE); + + ts = 1443614400000L; + value = 3; + while (downsampler.hasNext()) { + HistogramDataPoint dp = downsampler.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(value, Bytes.getLong(dp.getRawData(false))); + ts = 1451563200000L; + value = 18; + } + } + + @Test + public void testDownsampler_calendarSkipSomePoints() { + source = spy(HistogramSeekableViewForTest + .fromArray(new HistogramDataPoint[] { + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1L), BASE_TIME), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2L), BASE_TIME + 1800000), + // skip an hour + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 6L), BASE_TIME + 7200000) })); + specification = new DownsamplingSpecification("1hc-sum"); + specification.setTimezone(TV); + downsampler = new HistogramDownsampler(source, specification, 0, Long.MAX_VALUE); + + long ts = BASE_TIME; + long value = 3; + while (downsampler.hasNext()) { + HistogramDataPoint dp = downsampler.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(value, Bytes.getLong(dp.getRawData(false))); + ts = 1357005600000L; + value = 6; + } + } + + @Test + public void testDownsampler_noData() { + source = spy(HistogramSeekableViewForTest.fromArray(new HistogramDataPoint[] {})); + specification = new DownsamplingSpecification("1d-sum"); + downsampler = new HistogramDownsampler(source, specification, 0, Long.MAX_VALUE); + verify(source, never()).next(); + assertFalse(downsampler.hasNext()); + } + + @Test + public void testDownsampler_noDataCalendar() { + source = spy(HistogramSeekableViewForTest.fromArray(new HistogramDataPoint[] {})); + specification = new DownsamplingSpecification("1mc-sum"); + downsampler = new HistogramDownsampler(source, specification, 0, Long.MAX_VALUE); + verify(source, never()).next(); + assertFalse(downsampler.hasNext()); + } + + @Test(expected = UnsupportedOperationException.class) + public void testRemove() { + specification = new DownsamplingSpecification("1d-sum"); + new HistogramDownsampler(source, specification, 0, Long.MAX_VALUE).remove(); + } + + @Test + public void testSeek() { + specification = new DownsamplingSpecification("1000s-sum"); + downsampler = new HistogramDownsampler(source, specification, 0, 0); + downsampler.seek(BASE_TIME + 3600000L); + verify(source, never()).next(); + List<Long> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + HistogramDataPoint dp = downsampler.next(); + values.add(Bytes.getLong(dp.getRawData(false))); + timestamps_in_millis.add(dp.timestamp()); + } + + assertEquals(3, values.size()); + assertEquals(90, values.get(0).longValue()); + assertEquals(BASE_TIME + 3600000L, timestamps_in_millis.get(0).longValue()); + assertEquals(40, values.get(1).longValue()); + assertEquals(BASE_TIME + 6600000L, timestamps_in_millis.get(1).longValue()); + assertEquals(50, values.get(2).longValue()); + assertEquals(BASE_TIME + 8600000L, timestamps_in_millis.get(2).longValue()); + } + + @Test + public void testSeek_skipPartialInterval() { + specification = new DownsamplingSpecification("1000s-sum"); + downsampler = new HistogramDownsampler(source, specification, 0, 0); + downsampler.seek(BASE_TIME + 3800000L); + verify(source, never()).next(); + List<Long> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + HistogramDataPoint dp = downsampler.next(); + values.add(Bytes.getLong(dp.getRawData(false))); + timestamps_in_millis.add(dp.timestamp()); + } + + // seek timestamp was BASE_TIME + 3800000L or 1,357,002,200,000 ms. + // The interval that has the timestamp began at 1,357,002,000,000 ms. It + // had two data points but was abandoned because the requested timestamp + // was not aligned. The next two intervals at 1,357,003,000,000 and + // at 1,357,004,000,000 did not have data points. The first interval that + // had a data point began at 1,357,002,005,000 ms or BASE_TIME + 6600000L. + assertEquals(2, values.size()); + assertEquals(40, values.get(0).longValue()); + assertEquals(BASE_TIME + 6600000L, timestamps_in_millis.get(0).longValue()); + assertEquals(50, values.get(1).longValue()); + assertEquals(BASE_TIME + 8600000L, timestamps_in_millis.get(1).longValue()); + } + + @Test + public void testSeek_doubleIteration() { + specification = new DownsamplingSpecification("1000s-sum"); + downsampler = new HistogramDownsampler(source, specification, 0, 0); + while (downsampler.hasNext()) { + downsampler.next(); + } + downsampler.seek(BASE_TIME + 3600000L); + List<Long> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + values.add(Bytes.getLong(hdp.getRawData(false))); + timestamps_in_millis.add(hdp.timestamp()); + } + + assertEquals(3, values.size()); + assertEquals(90, values.get(0).longValue()); + assertEquals(BASE_TIME + 3600000L, timestamps_in_millis.get(0).longValue()); + assertEquals(40, values.get(1).longValue()); + assertEquals(BASE_TIME + 6600000L, timestamps_in_millis.get(1).longValue()); + assertEquals(50, values.get(2).longValue()); + assertEquals(BASE_TIME + 8600000L, timestamps_in_millis.get(2).longValue()); + } + + @Test + public void testSeek_abandoningIncompleteInterval() { + source = HistogramSeekableViewForTest + .fromArray(new HistogramDataPoint[] { + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 40L), BASE_TIME + 100L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 40L), BASE_TIME + 1100L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 40L), BASE_TIME + 2100L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 40L), BASE_TIME + 3100L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 40L), BASE_TIME + 4100L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 40L), BASE_TIME + 5100L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 40L), BASE_TIME + 6100L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 40L), BASE_TIME + 7100L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 40L), BASE_TIME + 8100L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 40L), BASE_TIME + 9100L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 40L), BASE_TIME + 10100L) }); + specification = new DownsamplingSpecification("10s-sum"); + downsampler = new HistogramDownsampler(source, specification, 0, 0); + // The seek is aligned by the downsampling window. + downsampler.seek(BASE_TIME); + assertTrue("seek(BASE_TIME)", downsampler.hasNext()); + HistogramDataPoint first_dp = downsampler.next(); + assertEquals("seek(1356998400000)", BASE_TIME, first_dp.timestamp()); + assertEquals("seek(1356998400000)", 400, Bytes.getLong(first_dp.getRawData(false))); + + // No seeks but the last one is aligned by the downsampling window. + for (long seek_timestamp = BASE_TIME + 1000L; seek_timestamp < BASE_TIME + + 10100L; seek_timestamp += 1000) { + downsampler.seek(seek_timestamp); + assertTrue("ts = " + seek_timestamp, downsampler.hasNext()); + HistogramDataPoint dp = downsampler.next(); + // Timestamp should be greater than or equal to the seek timestamp. + assertTrue(String.format("%d >= %d", dp.timestamp(), seek_timestamp), + dp.timestamp() >= seek_timestamp); + assertEquals(String.format("seek(%d)", seek_timestamp), + BASE_TIME + 10000L, dp.timestamp()); + assertEquals(String.format("seek(%d)", seek_timestamp), + 40, Bytes.getLong(dp.getRawData(false))); + } + } + + @Test + public void testSeek_useCalendar() { + source = spy(HistogramSeekableViewForTest + .fromArray(new HistogramDataPoint[] { + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1L), 1356998400000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2L), 1388534400000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 4L), 1420070400000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 8L), 1451606400000L) })); + + specification = new DownsamplingSpecification("1yc-sum"); + downsampler = new HistogramDownsampler(source, specification, 0, Long.MAX_VALUE); + + downsampler.seek(1420070400000L); + verify(source, never()).next(); + + long timestamp = 1420070400000L; + long value = 4; + while (downsampler.hasNext()) { + HistogramDataPoint dp = downsampler.next(); + System.out.println(dp.timestamp() + " " + Bytes.getLong(dp.getRawData(false))); + assertEquals(timestamp, dp.timestamp()); + assertEquals(value, Bytes.getLong(dp.getRawData(false))); + timestamp = 1451606400000L; + value = 8; + } + + ((MockHistogramSeekableView) source).resetIndex(); + specification = new DownsamplingSpecification("1yc-sum"); + downsampler = new HistogramDownsampler(source, specification, 0, 0); + downsampler.seek(1420070400001L); + + while (downsampler.hasNext()) { + HistogramDataPoint dp = downsampler.next(); + assertEquals(timestamp, dp.timestamp()); + assertEquals(value, Bytes.getLong(dp.getRawData(false))); + } + } + + @Test + public void testHistogramSpanDownSampler() { + List<HistogramDataPoint> row = Arrays.asList(HIST_DATA_POINTS); + final HistogramSpan hspan = new HistogramSpan(tsdb); + hspan.addRow(KEY, row); + + // check the data points using iterator + List<Long> it_values = Lists.newArrayList(); + HistogramSpan.Iterator it = hspan.spanIterator(); + while (it.hasNext()) { + HistogramDataPoint hdp = it.next(); + it_values.add(Bytes.getLong(hdp.getRawData(false))); + } + assertEquals(6, it_values.size()); + assertEquals(40L, it_values.get(0).longValue()); + assertEquals(50L, it_values.get(1).longValue()); + assertEquals(40L, it_values.get(2).longValue()); + assertEquals(50L, it_values.get(3).longValue()); + assertEquals(40L, it_values.get(4).longValue()); + assertEquals(50L, it_values.get(5).longValue()); + + + List<Long> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + specification = new DownsamplingSpecification("1000s-sum"); + downsampler = hspan.downsampler(0, 0, specification, false, 0, 0); + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + values.add(Bytes.getLong(hdp.getRawData(false))); + timestamps_in_millis.add(hdp.timestamp()); + } + + assertEquals(5, values.size()); + assertEquals(40L, values.get(0).longValue()); + assertEquals(BASE_TIME - 400000L, timestamps_in_millis.get(0).longValue()); + assertEquals(50, values.get(1).longValue()); + assertEquals(BASE_TIME + 1600000, timestamps_in_millis.get(1).longValue()); + assertEquals(90, values.get(2).longValue()); + assertEquals(BASE_TIME + 3600000L, timestamps_in_millis.get(2).longValue()); + assertEquals(40, values.get(3).longValue()); + assertEquals(BASE_TIME + 6600000L, timestamps_in_millis.get(3).longValue()); + assertEquals(50, values.get(4).longValue()); + assertEquals(BASE_TIME + 8600000L, timestamps_in_millis.get(4).longValue()); + } + + @Test + public void testHistogramSpanDownSampler_10seconds() { + List<HistogramDataPoint> row = Arrays.asList(new HistogramDataPoint[] { + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1L), BASE_TIME + 5000L * 0), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2L), BASE_TIME + 5000L * 1), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 4L), BASE_TIME + 5000L * 2), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 8L), BASE_TIME + 5000L * 3), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 16L), BASE_TIME + 5000L * 4), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 32L), BASE_TIME + 5000L * 5), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 64L), BASE_TIME + 5000L * 6), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 128L), BASE_TIME + 5000L * 7), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 256L), BASE_TIME + 5000L * 8), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 512L), BASE_TIME + 5000L * 9), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1024L), BASE_TIME + 5000L * 10) }); + + final HistogramSpan hspan = new HistogramSpan(tsdb); + hspan.addRow(KEY, row); + + // downsample iterator the span + List<Long> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + specification = new DownsamplingSpecification("10s-sum"); + downsampler = hspan.downsampler(0, 0, specification, false, 0, 0); + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + values.add(Bytes.getLong(hdp.getRawData(false))); + timestamps_in_millis.add(hdp.timestamp()); + } + + assertEquals(6, values.size()); + assertEquals(3, values.get(0).longValue()); + assertEquals(BASE_TIME + 00000L, timestamps_in_millis.get(0).longValue()); + assertEquals(12, values.get(1).longValue()); + assertEquals(BASE_TIME + 10000L, timestamps_in_millis.get(1).longValue()); + assertEquals(48, values.get(2).longValue()); + assertEquals(BASE_TIME + 20000L, timestamps_in_millis.get(2).longValue()); + assertEquals(192, values.get(3).longValue()); + assertEquals(BASE_TIME + 30000L, timestamps_in_millis.get(3).longValue()); + assertEquals(768, values.get(4).longValue()); + assertEquals(BASE_TIME + 40000L, timestamps_in_millis.get(4).longValue()); + assertEquals(1024, values.get(5).longValue()); + assertEquals(BASE_TIME + 50000L, timestamps_in_millis.get(5).longValue()); + } + + @Test + public void testHistogramSpanDownSampler_15seconds() { + List<HistogramDataPoint> row = Arrays.asList(new HistogramDataPoint[] { + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1L), BASE_TIME + 5000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2L), BASE_TIME + 15000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 4L), BASE_TIME + 25000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 8L), BASE_TIME + 35000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 16L), BASE_TIME + 45000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 32L), BASE_TIME + 55000L) }); + + final HistogramSpan hspan = new HistogramSpan(tsdb); + hspan.addRow(KEY, row); + + // downsample iterator the span + List<Long> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + specification = new DownsamplingSpecification("15s-sum"); + downsampler = hspan.downsampler(0, 0, specification, false, 0, 0); + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + values.add(Bytes.getLong(hdp.getRawData(false))); + timestamps_in_millis.add(hdp.timestamp()); + } + + assertEquals(4, values.size()); + assertEquals(1, values.get(0).longValue()); + assertEquals(BASE_TIME + 00000L, timestamps_in_millis.get(0).longValue()); + assertEquals(6, values.get(1).longValue()); + assertEquals(BASE_TIME + 15000L, timestamps_in_millis.get(1).longValue()); + assertEquals(8, values.get(2).longValue()); + assertEquals(BASE_TIME + 30000L, timestamps_in_millis.get(2).longValue()); + assertEquals(48, values.get(3).longValue()); + assertEquals(BASE_TIME + 45000L, timestamps_in_millis.get(3).longValue()); + } + + @Test + public void testHistogramSpanDownsampler_allFullRange() { + List<HistogramDataPoint> row = Arrays.asList(new HistogramDataPoint[] { + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1L), BASE_TIME + 5000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2L), BASE_TIME + 15000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 4L), BASE_TIME + 25000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 8L), BASE_TIME + 35000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 16L), BASE_TIME + 45000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 32L), BASE_TIME + 55000L) }); + + final HistogramSpan hspan = new HistogramSpan(tsdb); + hspan.addRow(KEY, row); + + specification = new DownsamplingSpecification("0all-sum"); + downsampler = hspan.downsampler(0, 0, specification, false, 0, Long.MAX_VALUE); + + List<Long> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + values.add(Bytes.getLong(hdp.getRawData(false))); + timestamps_in_millis.add(hdp.timestamp()); + } + + assertEquals(1, values.size()); + assertEquals(63, values.get(0).longValue()); + assertEquals(0L, timestamps_in_millis.get(0).longValue()); + } + + @Test + public void testHistogramSpanDownsampler_allFilterOnQuery() { + List<HistogramDataPoint> row = Arrays.asList(new HistogramDataPoint[] { + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1L), BASE_TIME + 5000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2L), BASE_TIME + 15000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 4L), BASE_TIME + 25000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 8L), BASE_TIME + 35000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 16L), BASE_TIME + 45000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 32L), BASE_TIME + 55000L) }); + + final HistogramSpan hspan = new HistogramSpan(tsdb); + hspan.addRow(KEY, row); + specification = new DownsamplingSpecification("0all-sum"); + downsampler = hspan.downsampler(0, 0, specification, false, BASE_TIME + 15000L, BASE_TIME + 45000L); + + List<Long> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + values.add(Bytes.getLong(hdp.getRawData(false))); + timestamps_in_millis.add(hdp.timestamp()); + } + + assertEquals(1, values.size()); + assertEquals(14, values.get(0).longValue()); + assertEquals(BASE_TIME + 15000L, timestamps_in_millis.get(0).longValue()); + } + + @Test + public void testHistogramSpanDownsampler_allFilterOnQueryOutOfRangeEarly() { + List<HistogramDataPoint> row = Arrays.asList(new HistogramDataPoint[] { + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1L), BASE_TIME + 5000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2L), BASE_TIME + 15000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 4L), BASE_TIME + 25000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 8L), BASE_TIME + 35000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 16L), BASE_TIME + 45000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 32L), BASE_TIME + 55000L) }); + + final HistogramSpan hspan = new HistogramSpan(tsdb); + hspan.addRow(KEY, row); + + specification = new DownsamplingSpecification("0all-sum"); + downsampler = hspan.downsampler(0, 0, specification, false, BASE_TIME + 65000L, BASE_TIME + 75000L); + + List<Long> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + values.add(Bytes.getLong(hdp.getRawData(false))); + timestamps_in_millis.add(hdp.timestamp()); + } + + assertEquals(0, values.size()); + } + + @Test + public void testHistogramSpanDownsampler_allFilterOnQueryOutOfRangeLate() { + List<HistogramDataPoint> row = Arrays.asList(new HistogramDataPoint[] { + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1L), BASE_TIME + 5000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2L), BASE_TIME + 15000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 4L), BASE_TIME + 25000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 8L), BASE_TIME + 35000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 16L), BASE_TIME + 45000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 32L), BASE_TIME + 55000L) }); + + final HistogramSpan hspan = new HistogramSpan(tsdb); + hspan.addRow(KEY, row); + + specification = new DownsamplingSpecification("0all-sum"); + downsampler = hspan.downsampler(0, 0, specification, false, BASE_TIME - 15000L, BASE_TIME - 5000L); + + List<Long> values = Lists.newArrayList(); + List<Long> timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + values.add(Bytes.getLong(hdp.getRawData(false))); + timestamps_in_millis.add(hdp.timestamp()); + } + + assertEquals(0, values.size()); + } + + @Test + public void testHistogramSpanDownsampler_calendarHour() { + List<HistogramDataPoint> row = Arrays.asList(new HistogramDataPoint[] { + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1L), BASE_TIME), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2L), BASE_TIME + 1800000), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 3L), BASE_TIME + 3599000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 4L), BASE_TIME + 3600000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 5L), BASE_TIME + 5400000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 6L), BASE_TIME + 7199000L) }); + + final HistogramSpan hspan = new HistogramSpan(tsdb); + hspan.addRow(KEY, row); + + { + specification = new DownsamplingSpecification("1hc-sum"); + specification.setTimezone(TV); + downsampler = hspan.downsampler(0, 0, specification, false, 0, Long.MAX_VALUE); + + long ts = BASE_TIME; + long value = 6; + while (downsampler.hasNext()) { + HistogramDataPoint dp = downsampler.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(value, Bytes.getLong(dp.getRawData(false))); + ts += 3600000; + value = 15; + } + } + + // hour offset by 30m + { + specification = new DownsamplingSpecification("1hc-sum"); + specification.setTimezone(AF); + downsampler = hspan.downsampler(0, 0, specification, false, 0, Long.MAX_VALUE); + + long ts = 1356996600000L; + long value = 1; + while (downsampler.hasNext()) { + HistogramDataPoint dp = downsampler.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(value, Bytes.getLong(dp.getRawData(false))); + ts += 3600000; + if (value == 1) { + value = 9; + } else { + value = 11; + } + } + + } + + // multiple hours + { + specification = new DownsamplingSpecification("4hc-sum"); + specification.setTimezone(AF); + downsampler = hspan.downsampler(0, 0, specification, false, 0, Long.MAX_VALUE); + + long ts = 1356996600000L; + long value = 21; + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + assertEquals(ts, hdp.timestamp()); + assertEquals(value, Bytes.getLong(hdp.getRawData(false))); + } + } + } + + @Test + public void testHistogramSpanDownsampler_calendarDay() { + // UTC + List<HistogramDataPoint> row = Arrays.asList(new HistogramDataPoint[] { + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1L), DST_TS), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2L), DST_TS + 86399000), + // falls to the next in FJ + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 3L), DST_TS + 126001000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 4L), DST_TS + 172799000L), + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 5L), DST_TS + 172800000L), + // falls within 30m offset + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 6L), DST_TS + 242999000L) }); + + final HistogramSpan hspan = new HistogramSpan(tsdb); + hspan.addRow(KEY, row); + + // control + { + specification = new DownsamplingSpecification("1d-sum"); + downsampler = hspan.downsampler(0, 0, specification, false, 0, Long.MAX_VALUE); + + long ts = DST_TS; + long value = 3; + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + assertEquals(ts, hdp.timestamp()); + assertEquals(value, Bytes.getLong(hdp.getRawData(false))); + ts += 86400000; + if (value == 3) { + value = 7; + } else if (value == 7) { + value = 11; + } + } + } + + // 12 hour offset from UTC + { + specification = new DownsamplingSpecification("1dc-sum"); + specification.setTimezone(TV); + downsampler = hspan.downsampler(0, 0, specification, false, 0, Long.MAX_VALUE); + + long ts = 1450094400000L; + long value = 1; + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + assertEquals(ts, hdp.timestamp()); + assertEquals(value, Bytes.getLong(hdp.getRawData(false))); + ts += 86400000; + if (value == 1) { + value = 5; + } else if (value == 5) { + value = 9; + } else { + value = 6; + } + } + } + + // 11 hour offset from UTC + { + specification = new DownsamplingSpecification("1dc-sum"); + specification.setTimezone(FJ); + downsampler = hspan.downsampler(0, 0, specification, false, 0, Long.MAX_VALUE); + + long ts = 1450090800000L; + long value = 1; + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + assertEquals(ts, hdp.timestamp()); + assertEquals(value, Bytes.getLong(hdp.getRawData(false))); + ts += 86400000; + if (value == 1) { + value = 2; + } else if (value == 2) { + value = 12; + } else { + value = 6; + } + } + } + + { + // 30m offset + specification = new DownsamplingSpecification("1dc-sum"); + specification.setTimezone(AF); + downsampler = hspan.downsampler(0, 0, specification, false, 0, Long.MAX_VALUE); + + long ts = 1450121400000L; + long value = 1; + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + assertEquals(ts, hdp.timestamp()); + assertEquals(value, Bytes.getLong(hdp.getRawData(false))); + ts += 86400000; + if (value == 1) { + value = 5; + } else if (value == 5) { + value = 15; + } + } + } + + { + // multiple days + specification = new DownsamplingSpecification("3dc-sum"); + specification.setTimezone(AF); + downsampler = hspan.downsampler(0, 0, specification, false, 0, Long.MAX_VALUE); + + long ts = 1450121400000L; + long value = 21; + while (downsampler.hasNext()) { + HistogramDataPoint hdp = downsampler.next(); + assertEquals(ts, hdp.timestamp()); + assertEquals(value, Bytes.getLong(hdp.getRawData(false))); + } + } + } +} diff --git a/test/core/TestHistogramPojo.java b/test/core/TestHistogramPojo.java new file mode 100644 index 0000000000..57e0fb1799 --- /dev/null +++ b/test/core/TestHistogramPojo.java @@ -0,0 +1,55 @@ +package net.opentsdb.core; + +import static org.mockito.Mockito.when; + +import org.hbase.async.Bytes; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import net.opentsdb.utils.Config; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({ TSDB.class }) +public class TestHistogramPojo { + + private TSDB tsdb; + private Config config; + private HistogramCodecManager manager; + + @Before + public void before() throws Exception { + tsdb = PowerMockito.mock(TSDB.class); + config = new Config(false); + + config.overrideConfig("tsd.core.histograms.config", + "{\"net.opentsdb.core.SimpleHistogramDecoder\": 0}"); + when(tsdb.getConfig()).thenReturn(config); + + manager = new HistogramCodecManager(tsdb); + when(tsdb.histogramManager()).thenReturn(manager); + } + + @Test + public void foo() throws Exception { + int v = 255; + + System.out.println("CV: " + (byte) v); + +// SimpleHistogram y1Hist = new SimpleHistogram(); +// y1Hist.addBucket(1.0f, 2.0f, 5L); +// y1Hist.addBucket(2.0f, 3.0f, 5L); +// y1Hist.addBucket(3.0f, 10.0f, 0L); +// +// byte[] raw = y1Hist.histogram(manager); +// System.out.println(HistogramPojo.bytesToBase64String(raw)); +// System.out.println(HistogramPojo.bytesToHexString(raw)); +// + //HistogramDataPoint h = tsdb.histogramManager().getDecoder((byte) 0).decode(raw, 1356998400000L); + //System.out.println(h); + } + +} diff --git a/test/core/TestHistogramRowSeq.java b/test/core/TestHistogramRowSeq.java new file mode 100644 index 0000000000..16394f2068 --- /dev/null +++ b/test/core/TestHistogramRowSeq.java @@ -0,0 +1,478 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; +import static org.powermock.api.mockito.PowerMockito.mock; + +import java.util.NoSuchElementException; + +import net.opentsdb.uid.UniqueId; +import net.opentsdb.uid.UniqueId.UniqueIdType; +import net.opentsdb.utils.Config; + +import org.hbase.async.Bytes; +import org.hbase.async.Bytes.ByteMap; +import org.hbase.async.KeyValue; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +import com.stumbleupon.async.Deferred; + +import java.util.ArrayList; +import java.util.List; + +@RunWith(PowerMockRunner.class) +//"Classloader hell"... It's real. Tell PowerMock to ignore these classes +//because they fiddle with the class loader. We don't test them anyway. +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ HistogramRowSeq.class, TSDB.class, UniqueId.class, KeyValue.class, + Config.class, RowKey.class }) +public final class TestHistogramRowSeq { + private TSDB tsdb = mock(TSDB.class); + private Config config = mock(Config.class); + private UniqueId metrics = mock(UniqueId.class); + private static final byte[] TABLE = { 't', 'a', 'b', 'l', 'e' }; + public byte[] key = null; + public static final byte[] FAMILY = { 't' }; + public static final byte[] ZERO = { 0 }; + + @Before + public void before() throws Exception { + // Inject the attributes we need into the "tsdb" object. + Whitebox.setInternalState(tsdb, "metrics", metrics); + Whitebox.setInternalState(tsdb, "table", TABLE); + Whitebox.setInternalState(tsdb, "config", config); + when(tsdb.getConfig()).thenReturn(config); + when(tsdb.metrics.width()).thenReturn((short) 3); + key = BaseTsdbTest.getRowKey( + BaseTsdbTest.generateUID(UniqueIdType.METRIC, (byte) 1), + 1356998400, + BaseTsdbTest.generateUID(UniqueIdType.TAGK, (byte) 1), + BaseTsdbTest.generateUID(UniqueIdType.TAGV, (byte) 1)); + when(RowKey.metricNameAsync(tsdb, key)) + .thenReturn(Deferred.fromResult("in.rps.latency")); + } + + @Test + public void setRow() throws Exception { + List<HistogramDataPoint> hdps = new ArrayList<HistogramDataPoint>(); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 0), 100L)); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1), 105L)); + + final HistogramRowSeq hrs = new HistogramRowSeq(tsdb); + hrs.setRow(key, hdps); + + assertEquals(2, hrs.size()); + assertEquals(100L, hrs.timestamp(0)); + assertEquals(105L, hrs.timestamp(1)); + } + + @Test (expected = IllegalStateException.class) + public void setRowAlreadySet() throws Exception { + List<HistogramDataPoint> hdps = new ArrayList<HistogramDataPoint>(); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 0), 100L)); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1), 105L)); + + final HistogramRowSeq hrs = new HistogramRowSeq(tsdb); + hrs.setRow(key, hdps); + hrs.setRow(key, hdps); + } + + @Test + public void addRowMergeLater() throws Exception { + List<HistogramDataPoint> hdps = new ArrayList<HistogramDataPoint>(); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 0), 100L)); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1), 105L)); + + final HistogramRowSeq hrs = new HistogramRowSeq(tsdb); + hrs.setRow(key, hdps); + assertEquals(2, hrs.size()); + + + List<HistogramDataPoint> hdps2 = new ArrayList<HistogramDataPoint>(); + hdps2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2), 110L)); + hdps2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 3), 115L)); + hrs.addRow(hdps2); + assertEquals(4, hrs.size()); + + assertEquals(100L, hrs.timestamp(0)); + assertEquals(105L, hrs.timestamp(1)); + assertEquals(110L, hrs.timestamp(2)); + assertEquals(115L, hrs.timestamp(3)); + } + + @Test + public void addRowMergeEarlier() throws Exception { + List<HistogramDataPoint> hdps = new ArrayList<HistogramDataPoint>(); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2), 110L)); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 3), 115L)); + + final HistogramRowSeq hrs = new HistogramRowSeq(tsdb); + hrs.setRow(key, hdps); + assertEquals(2, hrs.size()); + + + List<HistogramDataPoint> hdps2 = new ArrayList<HistogramDataPoint>(); + hdps2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 0), 100L)); + hdps2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1), 105L)); + hrs.addRow(hdps2); + assertEquals(4, hrs.size()); + + assertEquals(100L, hrs.timestamp(0)); + assertEquals(105L, hrs.timestamp(1)); + assertEquals(110L, hrs.timestamp(2)); + assertEquals(115L, hrs.timestamp(3)); + } + + @Test + public void addRowMergeMiddle() throws Exception { + List<HistogramDataPoint> hdps = new ArrayList<HistogramDataPoint>(); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 4), 120L)); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 5), 125L)); + + final HistogramRowSeq hrs = new HistogramRowSeq(tsdb); + hrs.setRow(key, hdps); + assertEquals(2, hrs.size()); + + + List<HistogramDataPoint> hdps2 = new ArrayList<HistogramDataPoint>(); + hdps2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 0), 100L)); + hdps2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1), 105L)); + hrs.addRow(hdps2); + assertEquals(4, hrs.size()); + + List<HistogramDataPoint> hdps3 = new ArrayList<HistogramDataPoint>(); + hdps3.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2), 110L)); + hdps3.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 3), 115L)); + hrs.addRow(hdps3); + assertEquals(6, hrs.size()); + + assertEquals(100L, hrs.timestamp(0)); + assertEquals(105L, hrs.timestamp(1)); + assertEquals(110L, hrs.timestamp(2)); + assertEquals(115L, hrs.timestamp(3)); + assertEquals(120L, hrs.timestamp(4)); + assertEquals(125L, hrs.timestamp(5)); + } + + @Test + public void addRowMergeDuplicateLater() throws Exception { + List<HistogramDataPoint> hdps = new ArrayList<HistogramDataPoint>(); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 0), 100L)); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1), 105L)); + + final HistogramRowSeq hrs = new HistogramRowSeq(tsdb); + hrs.setRow(key, hdps); + assertEquals(2, hrs.size()); + + + List<HistogramDataPoint> hdps2 = new ArrayList<HistogramDataPoint>(); + hdps2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 100), 100L)); + hdps2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2), 110L)); + hrs.addRow(hdps2); + assertEquals(3, hrs.size()); + + assertEquals(100L, hrs.timestamp(0)); + assertEquals(105L, hrs.timestamp(1)); + assertEquals(110L, hrs.timestamp(2)); + } + + @Test + public void timestamp() throws Exception { + List<HistogramDataPoint> hdps = new ArrayList<HistogramDataPoint>(); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 0), 100L)); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1), 105L)); + + final HistogramRowSeq hrs = new HistogramRowSeq(tsdb); + hrs.setRow(key, hdps); + + assertEquals(2, hrs.size()); + assertEquals(100L, hrs.timestamp(0)); + assertEquals(105L, hrs.timestamp(1)); + } + + @Test (expected = IndexOutOfBoundsException.class) + public void timestampOutofBounds() throws Exception { + List<HistogramDataPoint> hdps = new ArrayList<HistogramDataPoint>(); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 0), 100L)); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1), 105L)); + + final HistogramRowSeq hrs = new HistogramRowSeq(tsdb); + hrs.setRow(key, hdps); + + assertEquals(2, hrs.size()); + assertEquals(100L, hrs.timestamp(0)); + assertEquals(105L, hrs.timestamp(1)); + hrs.timestamp(2); + } + + @Test + public void iterateAllItems() throws Exception { + List<HistogramDataPoint> hdps = new ArrayList<HistogramDataPoint>(); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 0), 100L)); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1), 105L)); + + final HistogramRowSeq hrs = new HistogramRowSeq(tsdb); + hrs.setRow(key, hdps); + + assertEquals(2, hrs.size()); + + final HistogramSeekableView it = hrs.iterator(); + HistogramDataPoint hdp = it.next(); + + assertEquals(100L, hdp.timestamp()); + assertEquals(0L, Bytes.getLong(hdp.getRawData(false))); + + hdp = it.next(); + assertEquals(105L, hdp.timestamp()); + assertEquals(1L, Bytes.getLong(hdp.getRawData(false))); + + assertFalse(it.hasNext()); + } + + @Test + public void iterateAfterMergeDuplicate() throws Exception { + List<HistogramDataPoint> hdps = new ArrayList<HistogramDataPoint>(); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2), 110L)); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 3), 115L)); + + final HistogramRowSeq hrs = new HistogramRowSeq(tsdb); + hrs.setRow(key, hdps); + + assertEquals(2, hrs.size()); + + List<HistogramDataPoint> hdps2 = new ArrayList<HistogramDataPoint>(); + hdps2.add( + new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 20), 110L)); + hdps2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 4), 120L)); + + final HistogramSeekableView it = hrs.iterator(); + HistogramDataPoint hdp = it.next(); + + assertEquals(110L, hdp.timestamp()); + assertEquals(2L, Bytes.getLong(hdp.getRawData(false))); + + hdp = it.next(); + assertEquals(115L, hdp.timestamp()); + assertEquals(3L, Bytes.getLong(hdp.getRawData(false))); + + assertFalse(it.hasNext()); + } + + @Test + public void iterateLarge() throws Exception { + long ts = 100L; + final int limit = 64 * 1000; + List<HistogramDataPoint> hdps = new ArrayList<HistogramDataPoint>(); + for (int i = 0; i < limit; ++i) { + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, i), ts + 5 * i)); + } + + final HistogramRowSeq hrs = new HistogramRowSeq(tsdb); + hrs.setRow(key, hdps); + + final HistogramSeekableView it = hrs.iterator(); + while (it.hasNext()) { + assertEquals(ts, it.next().timestamp()); + ts += 5; + } + assertFalse(it.hasNext()); + } + + @Test + public void seekStart() throws Exception { + List<HistogramDataPoint> hdps = new ArrayList<HistogramDataPoint>(); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 0), 100L)); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1), 105L)); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2), 110L)); + + final HistogramRowSeq hrs = new HistogramRowSeq(tsdb); + hrs.setRow(key, hdps); + + assertEquals(3, hrs.size()); + + final HistogramSeekableView it = hrs.iterator(); + it.seek(100L); + HistogramDataPoint hdp = it.next(); + assertEquals(100L, hdp.timestamp()); + assertEquals(0, Bytes.getLong(hdp.getRawData(false))); + + assertTrue(it.hasNext()); + } + + @Test + public void seekMsBetween() throws Exception { + List<HistogramDataPoint> hdps = new ArrayList<HistogramDataPoint>(); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 0), 100L)); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1), 105L)); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2), 110L)); + + final HistogramRowSeq hrs = new HistogramRowSeq(tsdb); + hrs.setRow(key, hdps); + + assertEquals(3, hrs.size()); + + final HistogramSeekableView it = hrs.iterator(); + it.seek(105L); + HistogramDataPoint hdp = it.next(); + assertEquals(105L, hdp.timestamp()); + assertEquals(1, Bytes.getLong(hdp.getRawData(false))); + + assertTrue(it.hasNext()); + } + + @Test + public void seekMsEnd() throws Exception { + List<HistogramDataPoint> hdps = new ArrayList<HistogramDataPoint>(); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 0), 100L)); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1), 105L)); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2), 110L)); + + final HistogramRowSeq hrs = new HistogramRowSeq(tsdb); + hrs.setRow(key, hdps); + + assertEquals(3, hrs.size()); + + final HistogramSeekableView it = hrs.iterator(); + it.seek(110L); + HistogramDataPoint hdp = it.next(); + assertEquals(110L, hdp.timestamp()); + assertEquals(2, Bytes.getLong(hdp.getRawData(false))); + + assertFalse(it.hasNext()); + } + + @Test + public void seekMsTooEarly() throws Exception { + List<HistogramDataPoint> hdps = new ArrayList<HistogramDataPoint>(); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1), 105L)); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2), 110L)); + + final HistogramRowSeq hrs = new HistogramRowSeq(tsdb); + hrs.setRow(key, hdps); + + assertEquals(2, hrs.size()); + + final HistogramSeekableView it = hrs.iterator(); + it.seek(100L); + HistogramDataPoint hdp = it.next(); + assertEquals(105L, hdp.timestamp()); + assertEquals(1, Bytes.getLong(hdp.getRawData(false))); + + assertTrue(it.hasNext()); + } + + @Test (expected = NoSuchElementException.class) + public void seekMsPastLastDp() throws Exception { + List<HistogramDataPoint> hdps = new ArrayList<HistogramDataPoint>(); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 0), 100L)); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1), 105L)); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2), 110L)); + + final HistogramRowSeq hrs = new HistogramRowSeq(tsdb); + hrs.setRow(key, hdps); + + assertEquals(3, hrs.size()); + + final HistogramSeekableView it = hrs.iterator(); + it.seek(200L); + + it.next(); + } + + @Test + public void getTagUids() throws Exception { + List<HistogramDataPoint> hdps = new ArrayList<HistogramDataPoint>(); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 0), 100L)); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1), 105L)); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2), 110L)); + + final HistogramRowSeq hrs = new HistogramRowSeq(tsdb); + hrs.setRow(key, hdps); + + final ByteMap<byte[]> uids = hrs.getTagUids(); + assertEquals(1, uids.size()); + assertEquals(0, Bytes.memcmp( + BaseTsdbTest.generateUID(UniqueIdType.TAGK, (byte) 1), uids.firstKey())); + assertEquals(0, Bytes.memcmp( + BaseTsdbTest.generateUID(UniqueIdType.TAGV, (byte) 1), + uids.firstEntry().getValue())); + } + + @Test (expected = NullPointerException.class) + public void getTagUidsNotSet() throws Exception { + final HistogramRowSeq hrs = new HistogramRowSeq(tsdb); + hrs.getTagUids(); + } +} diff --git a/test/core/TestHistogramSpan.java b/test/core/TestHistogramSpan.java new file mode 100644 index 0000000000..c613374401 --- /dev/null +++ b/test/core/TestHistogramSpan.java @@ -0,0 +1,300 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; +import static org.powermock.api.mockito.PowerMockito.mock; +import net.opentsdb.uid.UniqueId; +import net.opentsdb.uid.UniqueId.UniqueIdType; +import net.opentsdb.utils.Config; + +import org.hbase.async.Bytes; +import org.hbase.async.KeyValue; +import org.hbase.async.Bytes.ByteMap; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +import com.stumbleupon.async.Deferred; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +@RunWith(PowerMockRunner.class) +//"Classloader hell"... It's real. Tell PowerMock to ignore these classes +//because they fiddle with the class loader. We don't test them anyway. +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ HistogramSpan.class, HistogramRowSeq.class, TSDB.class, + UniqueId.class, KeyValue.class, Config.class, RowKey.class }) +public final class TestHistogramSpan { + protected TSDB tsdb = mock(TSDB.class); + protected Config config = mock(Config.class); + protected UniqueId metrics = mock(UniqueId.class); + protected static final byte[] TABLE = { 't', 'a', 'b', 'l', 'e' }; + protected static final byte[] FAMILY = { 't' }; + protected static final byte[] ZERO = { 0 }; + + protected byte[] hour1 = null; + protected byte[] hour2 = null; + protected byte[] hour3 = null; + + @Before + public void before() throws Exception { + // Inject the attributes we need into the "tsdb" object. + Whitebox.setInternalState(tsdb, "metrics", metrics); + Whitebox.setInternalState(tsdb, "table", TABLE); + Whitebox.setInternalState(tsdb, "config", config); + when(tsdb.getConfig()).thenReturn(config); + when(tsdb.metrics.width()).thenReturn((short) 3); + hour1 = BaseTsdbTest.getRowKey( + BaseTsdbTest.generateUID(UniqueIdType.METRIC, (byte) 1), + 1356998400, + BaseTsdbTest.generateUID(UniqueIdType.TAGK, (byte) 1), + BaseTsdbTest.generateUID(UniqueIdType.TAGV, (byte) 1)); + hour2 = BaseTsdbTest.getRowKey( + BaseTsdbTest.generateUID(UniqueIdType.METRIC, (byte) 1), + 1357002000, + BaseTsdbTest.generateUID(UniqueIdType.TAGK, (byte) 1), + BaseTsdbTest.generateUID(UniqueIdType.TAGV, (byte) 1)); + hour3 = BaseTsdbTest.getRowKey( + BaseTsdbTest.generateUID(UniqueIdType.METRIC, (byte) 1), + 1357005600, + BaseTsdbTest.generateUID(UniqueIdType.TAGK, (byte) 1), + BaseTsdbTest.generateUID(UniqueIdType.TAGV, (byte) 1)); + + when(RowKey.metricNameAsync(tsdb, hour1)) + .thenReturn(Deferred.fromResult("in.rps.latency")); + } + + @Test + public void addRow() { + List<HistogramDataPoint> hdps = new ArrayList<HistogramDataPoint>(); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 0), 100L)); + hdps.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1), 105L)); + + final HistogramSpan histSpan = new HistogramSpan(tsdb); + histSpan.addRow(hour1, hdps); + + assertEquals(2, histSpan.size()); + } + + @Test (expected = NullPointerException.class) + public void addRowNull() { + final HistogramSpan histSpan = new HistogramSpan(tsdb); + histSpan.addRow(hour1, null); + } + + @Test (expected = IllegalArgumentException.class) + public void addRowBadKeyLength() { + List<HistogramDataPoint> row1 = new ArrayList<HistogramDataPoint>(); + row1.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 0), 100L)); + row1.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1), 105L)); + + final HistogramSpan histSpan = new HistogramSpan(tsdb); + histSpan.addRow(hour1, row1); + + List<HistogramDataPoint> row2 = new ArrayList<HistogramDataPoint>(); + row2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2), 110L)); + row2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 3), 115L)); + + final byte[] bad_key = new byte[] { 0, 0, 0, 1, 0x50, (byte)0xE2, 0x43, + 0x20, 0, 0, 0, 1 }; + histSpan.addRow(bad_key, row2); + } + + @Test (expected = IllegalArgumentException.class) + public void addRowMissMatchedMetric() { + List<HistogramDataPoint> row1 = new ArrayList<HistogramDataPoint>(); + row1.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 0), 100L)); + row1.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1), 105L)); + + final HistogramSpan histSpan = new HistogramSpan(tsdb); + histSpan.addRow(hour1, row1); + + List<HistogramDataPoint> row2 = new ArrayList<HistogramDataPoint>(); + row2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2), 110L)); + row2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 3), 115L)); + + final byte[] not_matched_mitric_key = new byte[] { 0, 0, 0, 2, 0x50, + (byte)0xE2, 0x35, 0x10, 0, 0, 0, 1, 0, 0, 0, 2 }; + histSpan.addRow(not_matched_mitric_key, row2); + } + + @Test (expected = IllegalArgumentException.class) + public void addRowMissMatchedTagk() { + List<HistogramDataPoint> row1 = new ArrayList<HistogramDataPoint>(); + row1.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 0), 100L)); + row1.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1), 105L)); + + final HistogramSpan histSpan = new HistogramSpan(tsdb); + histSpan.addRow(hour1, row1); + + List<HistogramDataPoint> row2 = new ArrayList<HistogramDataPoint>(); + row2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2), 110L)); + row2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 3), 115L)); + + final byte[] not_matched_tagk_key = new byte[] { 0, 0, 0, 1, 0x50, + (byte)0xE2, 0x35, 0x10, 0, 0, 0, 2, 0, 0, 0, 2 }; + histSpan.addRow(not_matched_tagk_key, row2); + } + + @Test (expected = IllegalArgumentException.class) + public void addRowMissMatchedTagv() { + List<HistogramDataPoint> row1 = new ArrayList<HistogramDataPoint>(); + row1.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 0), 100L)); + row1.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1), 105L)); + + final HistogramSpan histSpan = new HistogramSpan(tsdb); + histSpan.addRow(hour1, row1); + + List<HistogramDataPoint> row2 = new ArrayList<HistogramDataPoint>(); + row2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2), 110L)); + row2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 3), 115L)); + + final byte[] not_matched_tagv_key = new byte[] { 0, 0, 0, 1, 0x50, + (byte)0xE2, 0x35, 0x10, 0, 0, 0, 1, 0, 0, 0, 3 }; + histSpan.addRow(not_matched_tagv_key, row2); + } + + @Test + public void addRowOutOfOrder() { + List<HistogramDataPoint> row2 = new ArrayList<HistogramDataPoint>(); + row2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2), 110L)); + row2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 3), 115L)); + + + final HistogramSpan histSpan = new HistogramSpan(tsdb); + histSpan.addRow(hour2, row2); + + List<HistogramDataPoint> row1 = new ArrayList<HistogramDataPoint>(); + row1.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 0), 100L)); + row1.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1), 105L)); + histSpan.addRow(hour1, row1); + + assertEquals(4, histSpan.size()); + + assertEquals(100L, histSpan.timestamp(0)); + assertEquals(105L, histSpan.timestamp(1)); + assertEquals(110L, histSpan.timestamp(2)); + assertEquals(115L, histSpan.timestamp(3)); + } + + @Test (expected = IllegalArgumentException.class) + public void addDifferentKey() throws Exception { + List<HistogramDataPoint> row1 = new ArrayList<HistogramDataPoint>(); + row1.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 0), 100L)); + row1.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1), 105L)); + + final HistogramSpan histSpan = new HistogramSpan(tsdb); + histSpan.addRow(hour1, row1); + + final byte[] hour1_with_diff_key = Arrays.copyOf(hour1, hour1.length); + hour1_with_diff_key[hour1_with_diff_key.length - 1] = 3; + + List<HistogramDataPoint> row2 = new ArrayList<HistogramDataPoint>(); + row2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 2), 110L)); + row2.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 3), 115L)); + + histSpan.addRow(hour1_with_diff_key, row2); + } + + @Test + public void getTagUids() { + List<HistogramDataPoint> row1 = new ArrayList<HistogramDataPoint>(); + row1.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 0), 100L)); + row1.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1), 105L)); + + final HistogramSpan histSpan = new HistogramSpan(tsdb); + histSpan.addRow(hour1, row1); + + final ByteMap<byte[]> uids = histSpan.getTagUids(); + assertEquals(1, uids.size()); + assertEquals(0, Bytes.memcmp( + BaseTsdbTest.generateUID(UniqueIdType.TAGK, (byte) 1), uids.firstKey())); + assertEquals(0, Bytes.memcmp( + BaseTsdbTest.generateUID(UniqueIdType.TAGK, (byte) 1), + uids.firstEntry().getValue())); + } + + @Test (expected = IllegalStateException.class) + public void getTagUidsNotSet() { + final HistogramSpan histSpan = new HistogramSpan(tsdb); + histSpan.getTagUids(); + } + + @Test + public void getAggregatedTagUids() { + List<HistogramDataPoint> row1 = new ArrayList<HistogramDataPoint>(); + row1.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 0), 100L)); + row1.add(new SimpleHistogramDataPointAdapter( + new LongHistogramDataPointForTest(0, 1), 105L)); + + final HistogramSpan histSpan = new HistogramSpan(tsdb); + histSpan.addRow(hour1, row1); + + final List<byte[]> uids = histSpan.getAggregatedTagUids(); + assertEquals(0, uids.size()); + } + + @Test + public void getAggregatedTagUidsNotSet() { + final HistogramSpan histSpan = new HistogramSpan(tsdb); + assertTrue(histSpan.getAggregatedTagUids().isEmpty()); + } + + @Test (expected = IndexOutOfBoundsException.class) + public void iteratorEmpty() { + final HistogramSpan histSpan = new HistogramSpan(tsdb); + assertFalse(histSpan.spanIterator().hasNext()); + } + +} diff --git a/test/core/TestHistogramSpanGroup.java b/test/core/TestHistogramSpanGroup.java new file mode 100644 index 0000000000..14ccfb21e3 --- /dev/null +++ b/test/core/TestHistogramSpanGroup.java @@ -0,0 +1,257 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. + +package net.opentsdb.core; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.List; + +import net.opentsdb.utils.ByteSet; +import net.opentsdb.utils.Config; + +import org.hbase.async.Bytes; +import org.hbase.async.Bytes.ByteMap; +import org.hbase.async.HBaseClient; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({ TSDB.class, HBaseClient.class, Config.class, HistogramSpanGroup.class, + HistogramSpan.class }) +public final class TestHistogramSpanGroup { + private static long start_ts = 1356998400L; + private static long end_ts = 1356998600L; + + private TSDB tsdb; + + @Before + public void before() { + tsdb = PowerMockito.mock(TSDB.class); + } + + @Test + public void getTagUids() throws Exception { + final ByteMap<byte[]> uids = new ByteMap<byte[]>(); + uids.put(new byte[] { 0, 0, 0, 1 }, new byte[] { 0, 0, 0, 2 }); + final HistogramSpan span = mock(HistogramSpan.class); + when(span.getTagUids()).thenReturn(uids); + + DownsamplingSpecification specification = + new DownsamplingSpecification("1dc-sum"); + final HistogramSpanGroup group = PowerMockito.spy( + new HistogramSpanGroup(tsdb, start_ts, + end_ts, null, HistogramAggregation.SUM, specification, 0, 0, 0, false, null)); + + final ArrayList<HistogramSpan> spans = + Whitebox.getInternalState(group, "spans"); + spans.add(span); + + final ByteMap<byte[]> uids_read = group.getTagUids(); + assertEquals(1, uids_read.size()); + assertEquals(0, Bytes.memcmp(new byte[] {0, 0, 0, 1 }, uids_read.firstKey())); + assertEquals(0, Bytes.memcmp(new byte[] {0, 0, 0, 2 }, + uids_read.firstEntry().getValue())); + } + + @Test + public void getTagUidsAggedOut() throws Exception { + final ByteMap<byte[]> uids = new ByteMap<byte[]>(); + uids.put(new byte[] { 0, 0, 0, 1 }, new byte[] { 0, 0, 0, 2 }); + final HistogramSpan span = mock(HistogramSpan.class); + when(span.getTagUids()).thenReturn(uids); + + final ByteMap<byte[]> uids2 = new ByteMap<byte[]>(); + uids2.put(new byte[] { 0, 0, 0, 1 }, new byte[] { 0, 0, 0, 3 }); + final HistogramSpan span2 = mock(HistogramSpan.class); + when(span2.getTagUids()).thenReturn(uids2); + + DownsamplingSpecification specification = + new DownsamplingSpecification("1dc-sum"); + final HistogramSpanGroup group = PowerMockito.spy( + new HistogramSpanGroup(tsdb, start_ts, + end_ts, null, HistogramAggregation.SUM, specification, 0, 0, 0, false, null)); + + final ArrayList<HistogramSpan> spans = + Whitebox.getInternalState(group, "spans"); + spans.add(span); + spans.add(span2); + + final ByteMap<byte[]> uids_read = group.getTagUids(); + assertEquals(0, uids_read.size()); + } + + @Test + public void getTagUidsNoSpans() throws Exception { + DownsamplingSpecification specification = + new DownsamplingSpecification("1dc-sum"); + final HistogramSpanGroup group = PowerMockito.spy( + new HistogramSpanGroup(tsdb, start_ts, + end_ts, null, HistogramAggregation.SUM, specification, 0, 0, 0, false, null)); + + final ByteMap<byte[]> uids_read = group.getTagUids(); + assertEquals(0, uids_read.size()); + } + + @Test + public void getAggregatedTagUidsNotAgged() throws Exception { + final ByteMap<byte[]> uids = new ByteMap<byte[]>(); + uids.put(new byte[] { 0, 0, 0, 1 }, new byte[] { 0, 0, 0, 2 }); + final HistogramSpan span = mock(HistogramSpan.class); + when(span.getTagUids()).thenReturn(uids); + + DownsamplingSpecification specification = + new DownsamplingSpecification("1dc-sum"); + final HistogramSpanGroup group = PowerMockito.spy( + new HistogramSpanGroup(tsdb, start_ts, + end_ts, null, HistogramAggregation.SUM, specification, 0, 0, 0, false, null)); + + final ArrayList<HistogramSpan> spans = + Whitebox.getInternalState(group, "spans"); + spans.add(span); + + final List<byte[]> uids_read = group.getAggregatedTagUids(); + assertEquals(0, uids_read.size()); + } + + @Test + public void getAggregatedTagUids() throws Exception { + final ByteMap<byte[]> uids = new ByteMap<byte[]>(); + uids.put(new byte[] { 0, 0, 0, 1 }, new byte[] { 0, 0, 0, 2 }); + final HistogramSpan span = mock(HistogramSpan.class); + when(span.getTagUids()).thenReturn(uids); + + final ByteMap<byte[]> uids2 = new ByteMap<byte[]>(); + uids2.put(new byte[] { 0, 0, 0, 1 }, new byte[] { 0, 0, 0, 3 }); + final HistogramSpan span2 = mock(HistogramSpan.class); + when(span2.getTagUids()).thenReturn(uids2); + + DownsamplingSpecification specification = + new DownsamplingSpecification("1dc-sum"); + final HistogramSpanGroup group = PowerMockito.spy( + new HistogramSpanGroup(tsdb, start_ts, + end_ts, null, HistogramAggregation.SUM, specification, 0, 0, 0, false, null)); + + final ArrayList<HistogramSpan> spans = + Whitebox.getInternalState(group, "spans"); + spans.add(span); + spans.add(span2); + + final List<byte[]> uids_read = group.getAggregatedTagUids(); + assertEquals(1, uids_read.size()); + assertArrayEquals(new byte[] { 0, 0, 0, 1 }, uids_read.get(0)); + } + + @Test + public void getAggregatedTagUidsNoSpans() throws Exception { + DownsamplingSpecification specification = + new DownsamplingSpecification("1dc-sum"); + final HistogramSpanGroup group = PowerMockito.spy( + new HistogramSpanGroup(tsdb, start_ts, + end_ts, null, HistogramAggregation.SUM, specification, 0, 0, 0, false, null)); + + final List<byte[]> uids_read = group.getAggregatedTagUids(); + assertEquals(0, uids_read.size()); + } + + @Test + public void getTagUidsAggedNotInQuery() throws Exception { + final ByteSet query_tags = new ByteSet(); + query_tags.add(new byte[] { 0, 0, 0, 3 }); + + final ByteMap<byte[]> uids = new ByteMap<byte[]>(); + uids.put(new byte[] { 0, 0, 0, 1 }, new byte[] { 0, 0, 0, 2 }); + final HistogramSpan span = mock(HistogramSpan.class); + when(span.getTagUids()).thenReturn(uids); + + DownsamplingSpecification specification = + new DownsamplingSpecification("1dc-sum"); + final HistogramSpanGroup group = PowerMockito.spy( + new HistogramSpanGroup(tsdb, start_ts, + end_ts, null, HistogramAggregation.SUM, specification, 0, 0, 0, false, + query_tags)); + + final ArrayList<HistogramSpan> spans = + Whitebox.getInternalState(group, "spans"); + spans.add(span); + + final ByteMap<byte[]> uids_read = group.getTagUids(); + assertEquals(0, uids_read.size()); + final List<byte[]> agg_tags = group.getAggregatedTagUids(); + assertEquals(1, agg_tags.size()); + assertArrayEquals(new byte[] { 0, 0, 0, 1 }, agg_tags.get(0)); + } + + @Test + public void getTagUidsInQueryTags() throws Exception { + final ByteSet query_tags = new ByteSet(); + query_tags.add(new byte[] { 0, 0, 0, 1 }); + + final ByteMap<byte[]> uids = new ByteMap<byte[]>(); + uids.put(new byte[] { 0, 0, 0, 1 }, new byte[] { 0, 0, 0, 2 }); + final HistogramSpan span = mock(HistogramSpan.class); + when(span.getTagUids()).thenReturn(uids); + + DownsamplingSpecification specification = + new DownsamplingSpecification("1dc-sum"); + final HistogramSpanGroup group = PowerMockito.spy( + new HistogramSpanGroup(tsdb, start_ts, + end_ts, null, HistogramAggregation.SUM, specification, 0, 0, 0, false, + query_tags)); + + final ArrayList<HistogramSpan> spans = + Whitebox.getInternalState(group, "spans"); + spans.add(span); + + final ByteMap<byte[]> uids_read = group.getTagUids(); + assertEquals(1, uids_read.size()); + assertEquals(0, Bytes.memcmp(new byte[] {0, 0, 0, 1 }, uids_read.firstKey())); + assertEquals(0, Bytes.memcmp(new byte[] {0, 0, 0, 2 }, + uids_read.firstEntry().getValue())); + assertEquals(0, group.getAggregatedTagUids().size()); + } + + @Test + public void getTagUidsNullQueryTags() throws Exception { + final ByteMap<byte[]> uids = new ByteMap<byte[]>(); + uids.put(new byte[] { 0, 0, 0, 1 }, new byte[] { 0, 0, 0, 2 }); + final HistogramSpan span = mock(HistogramSpan.class); + when(span.getTagUids()).thenReturn(uids); + + DownsamplingSpecification specification = + new DownsamplingSpecification("1dc-sum"); + final HistogramSpanGroup group = PowerMockito.spy( + new HistogramSpanGroup(tsdb, start_ts, + end_ts, null, HistogramAggregation.SUM, specification, 0, 0, 0, false, null)); + + final ArrayList<HistogramSpan> spans = + Whitebox.getInternalState(group, "spans"); + spans.add(span); + + final ByteMap<byte[]> uids_read = group.getTagUids(); + assertEquals(1, uids_read.size()); + assertEquals(0, Bytes.memcmp(new byte[] {0, 0, 0, 1 }, uids_read.firstKey())); + assertEquals(0, Bytes.memcmp(new byte[] {0, 0, 0, 2 }, + uids_read.firstEntry().getValue())); + assertEquals(0, group.getAggregatedTagUids().size()); + } +} diff --git a/test/core/TestIncomingDataPoints.java b/test/core/TestIncomingDataPoints.java new file mode 100644 index 0000000000..60f8696b54 --- /dev/null +++ b/test/core/TestIncomingDataPoints.java @@ -0,0 +1,155 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.when; +import net.opentsdb.uid.NoSuchUniqueName; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +public class TestIncomingDataPoints extends BaseTsdbTest { + + @Test + public void metricNameAsync() throws Exception { + final IncomingDataPoints dps = new IncomingDataPoints(tsdb); + dps.setSeries(METRIC_STRING, tags); + assertEquals(METRIC_STRING, dps.metricNameAsync().joinUninterruptibly()); + } + + @Test + public void metricNameAsyncSalted() throws Exception { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(2); + PowerMockito.when(Const.MAX_NUM_TAGS()).thenReturn((short) 8); + + final IncomingDataPoints dps = new IncomingDataPoints(tsdb); + dps.setSeries(METRIC_STRING, tags); + assertEquals(METRIC_STRING, dps.metricNameAsync().joinUninterruptibly()); + } + + @Test (expected = IllegalStateException.class) + public void metricNameAsyncRowNotSet() throws Exception { + final IncomingDataPoints dps = new IncomingDataPoints(tsdb); + dps.metricNameAsync().joinUninterruptibly(); + } + + @Test + public void rowKeyTemplate() throws Exception { + final byte[] expected = new byte[METRIC_BYTES.length + Const.TIMESTAMP_BYTES + + TAGK_BYTES.length + TAGV_BYTES.length]; + System.arraycopy(METRIC_BYTES, 0, expected, 0, METRIC_BYTES.length); + System.arraycopy(TAGK_BYTES, 0, expected, + METRIC_BYTES.length + Const.TIMESTAMP_BYTES, TAGK_BYTES.length); + System.arraycopy(TAGV_BYTES, 0, expected, + METRIC_BYTES.length + Const.TIMESTAMP_BYTES + TAGK_BYTES.length, + TAGV_BYTES.length); + + final byte[] key = IncomingDataPoints.rowKeyTemplate(tsdb, + METRIC_STRING, tags); + assertArrayEquals(expected, key); + } + + @Test + public void rowKeyTemplateWithSalt1Byte() throws Exception { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + final byte[] expected = new byte[METRIC_BYTES.length + Const.TIMESTAMP_BYTES + + TAGK_BYTES.length + TAGV_BYTES.length + 1]; + System.arraycopy(METRIC_BYTES, 0, expected, 1, METRIC_BYTES.length); + System.arraycopy(TAGK_BYTES, 0, expected, + METRIC_BYTES.length + Const.TIMESTAMP_BYTES + 1, TAGK_BYTES.length); + System.arraycopy(TAGV_BYTES, 0, expected, + METRIC_BYTES.length + Const.TIMESTAMP_BYTES + TAGK_BYTES.length + 1, + TAGV_BYTES.length); + + final byte[] key = IncomingDataPoints.rowKeyTemplate(tsdb, + METRIC_STRING, tags); + assertArrayEquals(expected, key); + } + + @Test + public void rowKeyTemplateWithSalt2Bytes() throws Exception { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(2); + final byte[] expected = new byte[METRIC_BYTES.length + Const.TIMESTAMP_BYTES + + TAGK_BYTES.length + TAGV_BYTES.length + 2]; + System.arraycopy(METRIC_BYTES, 0, expected, 2, METRIC_BYTES.length); + System.arraycopy(TAGK_BYTES, 0, expected, + METRIC_BYTES.length + Const.TIMESTAMP_BYTES + 2, TAGK_BYTES.length); + System.arraycopy(TAGV_BYTES, 0, expected, + METRIC_BYTES.length + Const.TIMESTAMP_BYTES + TAGK_BYTES.length + 2, + TAGV_BYTES.length); + + final byte[] key = IncomingDataPoints.rowKeyTemplate(tsdb, + METRIC_STRING, tags); + assertArrayEquals(expected, key); + } + + @Test (expected = NoSuchUniqueName.class) + public void rowKeyTemplateNoSuchMetric() throws Exception { + IncomingDataPoints.rowKeyTemplate(tsdb, NSUN_METRIC, tags); + } + + @Test (expected = NoSuchUniqueName.class) + public void rowKeyTemplateNoSuchTagK() throws Exception { + tags.clear(); + tags.put(NSUN_TAGK, TAGV_STRING); + IncomingDataPoints.rowKeyTemplate(tsdb, NSUN_METRIC, tags); + } + + @Test (expected = NoSuchUniqueName.class) + public void rowKeyTemplateNoSuchTagV() throws Exception { + tags.put(TAGK_STRING, NSUN_TAGV); + IncomingDataPoints.rowKeyTemplate(tsdb, NSUN_METRIC, tags); + } + + @Test (expected = NullPointerException.class) + public void rowKeyTemplateNullTSDB() throws Exception { + IncomingDataPoints.rowKeyTemplate(null, NSUN_METRIC, tags); + } + + @Test (expected = NullPointerException.class) + public void rowKeyTemplateNullMetric() throws Exception { + IncomingDataPoints.rowKeyTemplate(tsdb, null, tags); + } + + @Test (expected = NoSuchUniqueName.class) + public void rowKeyTemplateEmptyMetric() throws Exception { + when(metrics.getOrCreateId("")).thenThrow(new NoSuchUniqueName("metrics", "")); + IncomingDataPoints.rowKeyTemplate(tsdb, "", tags); + } + + @Test (expected = NullPointerException.class) + public void rowKeyTemplateNullTags() throws Exception { + IncomingDataPoints.rowKeyTemplate(tsdb, METRIC_STRING, null); + } + + // NOTE: This method doesn't enforce that we have tags + @Test + public void rowKeyTemplateEmptyTags() throws Exception { + tags.clear(); + final byte[] expected = new byte[METRIC_BYTES.length + Const.TIMESTAMP_BYTES]; + System.arraycopy(METRIC_BYTES, 0, expected, 0, METRIC_BYTES.length); + + final byte[] key = IncomingDataPoints.rowKeyTemplate(tsdb, + METRIC_STRING, tags); + assertArrayEquals(expected, key); + } +} diff --git a/test/core/TestInternal.java b/test/core/TestInternal.java index 8e1fc88bfb..991d5b2484 100644 --- a/test/core/TestInternal.java +++ b/test/core/TestInternal.java @@ -15,7 +15,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.util.ArrayList; @@ -30,7 +32,7 @@ import org.powermock.modules.junit4.PowerMockRunner; @RunWith(PowerMockRunner.class) -@PrepareForTest({ Internal.class }) +@PrepareForTest({ Internal.class, Const.class }) public final class TestInternal { private static final byte[] KEY = { 0, 0, 1, 0x50, (byte)0xE2, 0x27, 0, 0, 0, 1, 0, 0, 2 }; @@ -809,7 +811,90 @@ public void extractQualifierMilliSeconds() { assertArrayEquals(new byte[] { (byte) 0xF0, 0x00, 0x02, 0x07 }, Internal.extractQualifier(qual, 2)); } + + @Test + public void getMaxUnsignedValueOnBytes() throws Exception { + assertEquals(0, Internal.getMaxUnsignedValueOnBytes(0)); + assertEquals(255, Internal.getMaxUnsignedValueOnBytes(1)); + assertEquals(65535, Internal.getMaxUnsignedValueOnBytes(2)); + assertEquals(16777215, Internal.getMaxUnsignedValueOnBytes(3)); + assertEquals(4294967295L, Internal.getMaxUnsignedValueOnBytes(4)); + assertEquals(1099511627775L, Internal.getMaxUnsignedValueOnBytes(5)); + assertEquals(281474976710655L, Internal.getMaxUnsignedValueOnBytes(6)); + assertEquals(72057594037927935L, Internal.getMaxUnsignedValueOnBytes(7)); + assertEquals(Long.MAX_VALUE, Internal.getMaxUnsignedValueOnBytes(8)); + + try { + Internal.getMaxUnsignedValueOnBytes(9); + fail("Expected an IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertNotNull(e); + } + + try { + Internal.getMaxUnsignedValueOnBytes(-1); + fail("Expected an IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertNotNull(e); + } + } + @Test + public void vleEncodeLong0() throws Exception { + final byte[] expected = new byte[1]; + assertArrayEquals(expected, Internal.vleEncodeLong(0)); + } + + @Test + public void vleEncodeLong1byte() throws Exception { + final byte[] expected = new byte[] { 42 }; + assertArrayEquals(expected, Internal.vleEncodeLong(42)); + } + + @Test + public void vleEncodeLong1byteNegative() throws Exception { + final byte[] expected = new byte[] { -42 }; + assertArrayEquals(expected, Internal.vleEncodeLong(-42)); + } + + @Test + public void vleEncodeLong2bytes() throws Exception { + final byte[] expected = new byte[] { 1, 1 }; + assertArrayEquals(expected, Internal.vleEncodeLong(257)); + } + + @Test + public void vleEncodeLong2bytesNegative() throws Exception { + final byte[] expected = new byte[] { (byte) 0xFE, (byte) 0xFF }; + assertArrayEquals(expected, Internal.vleEncodeLong(-257)); + } + + @Test + public void vleEncodeLong4bytes() throws Exception { + final byte[] expected = new byte[] { 0, 1, 0, 1 }; + assertArrayEquals(expected, Internal.vleEncodeLong(65537)); + } + + @Test + public void vleEncodeLong4bytesNegative() throws Exception { + final byte[] expected = + new byte[] { (byte) 0xFF, (byte) 0xFE, (byte) 0xFF, (byte) 0xFF }; + assertArrayEquals(expected, Internal.vleEncodeLong(-65537)); + } + + @Test + public void vleEncodeLong8bytes() throws Exception { + final byte[] expected = new byte[] { 0, 0, 0, 1, 0, 0, 0, 0 }; + assertArrayEquals(expected, Internal.vleEncodeLong(4294967296L)); + } + + @Test + public void vleEncodeLong8bytesNegative() throws Exception { + final byte[] expected = new byte[] { + (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, 0, 0, 0, 0 }; + assertArrayEquals(expected, Internal.vleEncodeLong(-4294967296L)); + } + /** Shorthand to create a {@link KeyValue}. */ private static KeyValue makekv(final byte[] qualifier, final byte[] value) { return new KeyValue(KEY, FAMILY, qualifier, value); diff --git a/test/core/TestMultiGetQuery.java b/test/core/TestMultiGetQuery.java new file mode 100644 index 0000000000..bf04d90c87 --- /dev/null +++ b/test/core/TestMultiGetQuery.java @@ -0,0 +1,1103 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.anyList; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; +import java.util.SortedMap; + +import org.hbase.async.Bytes.ByteMap; +import org.hbase.async.GetRequest; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +import com.google.common.collect.Lists; + +import net.opentsdb.core.MultiGetQuery.MultiGetTask; +import net.opentsdb.rollup.RollupConfig; +import net.opentsdb.rollup.RollupInterval; +import net.opentsdb.rollup.RollupQuery; +import net.opentsdb.stats.QueryStats; +import net.opentsdb.utils.ByteSet; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*", "javax.net.ssl.*"}) +public class TestMultiGetQuery extends BaseTsdbTest { + protected List<ByteMap<byte[][]>> q_tags; + protected List<ByteMap<byte[][]>> q_tags_nometa; + protected List<ByteMap<byte[][]>> q_tags_AD; + + protected long start_ts; + protected long end_ts; + protected TreeMap<byte[], Span> spans; + protected TreeMap<byte[], HistogramSpan> histogramSpans; + protected QueryStats query_stats; + protected Aggregator aggregator; + protected int max_bytes; + protected boolean multiget_no_meta; + protected TsdbQuery query; + + @Before + public void localBefore() { + max_bytes = 1000000; + multiget_no_meta = false; + start_ts = 1481227200; + end_ts = 1481284800; + q_tags = new ArrayList<ByteMap<byte[][]>>(); + q_tags_AD = new ArrayList<ByteMap<byte[][]>>(); + ByteMap<byte[][]> q_tags1; + q_tags1 = new ByteMap<byte[][]>(); + byte[][] val; + val = new byte[1][]; + val[0] = UIDS.get("A"); + + q_tags1.put(TAGK_BYTES, val); + val = new byte[1][]; + val[0] = UIDS.get("D"); + q_tags1.put(TAGK_B_BYTES, val); + ByteMap<byte[][]> q_tags2; + q_tags2 = new ByteMap<byte[][]>(); + val = new byte[1][]; + val[0] = UIDS.get("B"); + q_tags2.put(TAGK_BYTES, val); + val = new byte[1][]; + val[0] = UIDS.get("D"); + q_tags2.put(TAGK_B_BYTES, val); + ByteMap<byte[][]> q_tags3; + q_tags3 = new ByteMap<byte[][]>(); + val = new byte[1][]; + val[0] = UIDS.get("C"); + q_tags3.put(TAGK_BYTES, val); + val = new byte[1][]; + val[0] = UIDS.get("D"); + q_tags3.put(TAGK_B_BYTES, val); + + q_tags.add(q_tags1); + q_tags.add(q_tags2); + q_tags.add(q_tags3); + + q_tags_AD.add(q_tags1); + + q_tags_nometa = new ArrayList<ByteMap<byte[][]>>(); + ByteMap<byte[][]> q_tags_map = new ByteMap<byte[][]>(); + q_tags_map.put(TAGK_BYTES, new byte[][] { UIDS.get("A"), UIDS.get("B"), UIDS.get("C") }); + q_tags_map.put(TAGK_B_BYTES, new byte[][] { UIDS.get("D") }); + q_tags_map.put(UIDS.get("E"), new byte[][] { UIDS.get("F"), UIDS.get("G") }); + q_tags_nometa.add(q_tags_map); +// q_tags1.put(TAGK_BYTES, new byte[][] { UIDS.get("A"), UIDS.get("B"), UIDS.get("C") }); +// q_tags1.put(TAGK_B_BYTES, new byte[][] { UIDS.get("D") }); +// q_tags1.put(UIDS.get("E"), new byte[][] { UIDS.get("F"), UIDS.get("G") }); + + aggregator = Aggregators.get("sum"); + + RollupConfig rollup_config = RollupConfig.builder() + .addAggregationId("sum", 0) + .addAggregationId("count", 1) + .addAggregationId("max", 2) + .addAggregationId("min", 3) + .addInterval(RollupInterval.builder() + .setTable("tsdb-rollup-10m") + .setPreAggregationTable("tsdb-rollup-agg-10m") + .setInterval("10m") + .setRowSpan("6h")) + .addInterval(RollupInterval.builder() + .setTable("tsdb-rollup-1h") + .setPreAggregationTable("tsdb-rollup-agg-1h") + .setInterval("1h") + .setRowSpan("1d")) + .addInterval(RollupInterval.builder() + .setTable("tsdb-rollup-1d") + .setPreAggregationTable("tsdb-rollup-agg-1d") + .setInterval("1d") + .setRowSpan("1n")) + .build(); + Whitebox.setInternalState(tsdb, "rollup_config", rollup_config); + } + + @Test + public void ctor() throws Exception { + TsdbQuery query = new TsdbQuery(tsdb); + final MultiGetQuery mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags, + start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, + 0, max_bytes, false, multiget_no_meta); + // TODO - validations + } + + @Test + public void prepareAllTagvCompounds() throws Exception { + multiget_no_meta = true; + MultiGetQuery mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags_nometa, + start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, + 0, 0, false, multiget_no_meta); + + List<byte[][]> tagvs = mgq.prepareAllTagvCompounds(); + assertEquals(6, tagvs.size()); + assertEquals(3, tagvs.get(0).length); + assertArrayEquals(UIDS.get("A"), tagvs.get(0)[0]); + assertArrayEquals(UIDS.get("D"), tagvs.get(0)[1]); + assertArrayEquals(UIDS.get("F"), tagvs.get(0)[2]); + + assertEquals(3, tagvs.get(1).length); + assertArrayEquals(UIDS.get("B"), tagvs.get(1)[0]); + assertArrayEquals(UIDS.get("D"), tagvs.get(1)[1]); + assertArrayEquals(UIDS.get("F"), tagvs.get(1)[2]); + + assertEquals(3, tagvs.get(2).length); + assertArrayEquals(UIDS.get("C"), tagvs.get(2)[0]); + assertArrayEquals(UIDS.get("D"), tagvs.get(2)[1]); + assertArrayEquals(UIDS.get("F"), tagvs.get(2)[2]); + + assertEquals(3, tagvs.get(3).length); + assertArrayEquals(UIDS.get("A"), tagvs.get(3)[0]); + assertArrayEquals(UIDS.get("D"), tagvs.get(3)[1]); + assertArrayEquals(UIDS.get("G"), tagvs.get(3)[2]); + + assertEquals(3, tagvs.get(4).length); + assertArrayEquals(UIDS.get("B"), tagvs.get(4)[0]); + assertArrayEquals(UIDS.get("D"), tagvs.get(4)[1]); + assertArrayEquals(UIDS.get("G"), tagvs.get(4)[2]); + + assertEquals(3, tagvs.get(5).length); + assertArrayEquals(UIDS.get("C"), tagvs.get(5)[0]); + assertArrayEquals(UIDS.get("D"), tagvs.get(5)[1]); + assertArrayEquals(UIDS.get("G"), tagvs.get(5)[2]); + + // simple test + q_tags_nometa = new ArrayList<ByteMap<byte[][]>>(); + ByteMap<byte[][]> q_tags_nometa_map = new ByteMap<byte[][]>(); + q_tags_nometa_map.put(TAGK_BYTES, new byte[][] { UIDS.get("A") }); + q_tags_nometa.add(q_tags_nometa_map); + mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags_nometa, + start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, + 0, 0, false, multiget_no_meta); + tagvs = mgq.prepareAllTagvCompounds(); + assertEquals(1, tagvs.size()); + assertEquals(1, tagvs.get(0).length); + assertArrayEquals(UIDS.get("A"), tagvs.get(0)[0]); + } + + @Test + public void prepareRowBaseTimes() throws Exception { + // aligned timestamps + MultiGetQuery mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags, + start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, + 0, max_bytes, false, multiget_no_meta); + List<Long> timestamps = mgq.prepareRowBaseTimes(); + assertEquals(17, timestamps.size()); + long expected = 1481227200; + for (final long ts : timestamps) { + assertEquals(expected, ts); + expected += 3600; + } + + // unaligned + start_ts = 1481229792; + end_ts = 1481284801; + mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags, + start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, + 0, max_bytes, false, multiget_no_meta); + timestamps = mgq.prepareRowBaseTimes(); + assertEquals(17, timestamps.size()); + expected = 1481227200; + for (final long ts : timestamps) { + assertEquals(expected, ts); + expected += 3600; + } + + // short interval + start_ts = 1481229792; + end_ts = 1481229961; + mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags, + start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, + 0, max_bytes, false, multiget_no_meta); + timestamps = mgq.prepareRowBaseTimes(); + assertEquals(1, timestamps.size()); + assertEquals(1481227200, (long) timestamps.get(0)); + } + + @Test + public void prepareRowBaseTimesRollup() throws Exception { + RollupInterval interval = RollupInterval.builder() + .setTable("tsdb") + .setPreAggregationTable("tsdb_agg") + .setInterval("1m") + .setRowSpan("1h") + .build(); + RollupQuery rq = new RollupQuery(interval, aggregator, 0, aggregator); + MultiGetQuery mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags, + start_ts, end_ts, tsdb.dataTable(), spans, null, 0, rq, query_stats, + 0, max_bytes, false, multiget_no_meta); + List<Long> timestamps = mgq.prepareRowBaseTimesRollup(); + assertEquals(17, timestamps.size()); + long expected = 1481227200; + for (final long ts : timestamps) { + assertEquals(expected, ts); + expected += 3600; + } + + interval = RollupInterval.builder() + .setTable("tsdb") + .setPreAggregationTable("tsdb_agg") + .setInterval("1m") + .setRowSpan("1d") + .build(); + rq = new RollupQuery(interval, aggregator, 0, aggregator); + mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags, + start_ts, end_ts, tsdb.dataTable(), spans, null, 0, rq, query_stats, + 0, max_bytes, false, multiget_no_meta); + timestamps = mgq.prepareRowBaseTimesRollup(); + timestamps = mgq.prepareRowBaseTimesRollup(); + assertEquals(2, timestamps.size()); + expected = 1481155200; + for (final long ts : timestamps) { + assertEquals(expected, ts); + expected += 86400; + } + } + + @Test + public void prepareRequests() throws Exception { + MultiGetQuery mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags, + start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, + 0, max_bytes, false, multiget_no_meta); + List<Long> timestamps = mgq.prepareRowBaseTimes(); + ByteMap<ByteMap<List<GetRequest>>> row_map = mgq.prepareRequests(timestamps, q_tags); + ByteSet tsuids = new ByteSet(); + for (ByteMap<List<GetRequest>> rows : row_map.values()) { + tsuids.addAll(rows.keySet()); + } + assertEquals(3, tsuids.size()); + + List<GetRequest> rows = new ArrayList<GetRequest>(); + for (Entry<byte[], ByteMap<List<GetRequest>>> salt_entry : row_map.entrySet()) { + System.out.println(salt_entry.getValue()); + rows.addAll(salt_entry.getValue().get(getTSUID(METRIC_STRING, TAGK_STRING, + "A", TAGK_B_STRING, "D"))); + } + + assertEquals(timestamps.size(), rows.size()); + for (int i = 0; i < timestamps.size(); i++) { + byte[] key = getRowKey(METRIC_STRING, timestamps.get(i).intValue(), + TAGK_STRING, "A", TAGK_B_STRING, "D"); + assertArrayEquals(key, rows.get(i).key()); + } + + rows = new ArrayList<GetRequest>(); + for (Entry<byte[], ByteMap<List<GetRequest>>> salt_entry : row_map.entrySet()) { + rows.addAll(salt_entry.getValue().get(getTSUID(METRIC_STRING, TAGK_STRING, + "B", TAGK_B_STRING, "D"))); + } + assertEquals(timestamps.size(), rows.size()); + for (int i = 0; i < timestamps.size(); i++) { + byte[] key = getRowKey(METRIC_STRING, timestamps.get(i).intValue(), + TAGK_STRING, "B", TAGK_B_STRING, "D"); + assertArrayEquals(key, rows.get(i).key()); + } + + rows = new ArrayList<GetRequest>(); + for (Entry<byte[], ByteMap<List<GetRequest>>> salt_entry : row_map.entrySet()) { + rows.addAll(salt_entry.getValue().get(getTSUID(METRIC_STRING, TAGK_STRING, + "C", TAGK_B_STRING, "D"))); + } + assertEquals(timestamps.size(), rows.size()); + for (int i = 0; i < timestamps.size(); i++) { + byte[] key = getRowKey(METRIC_STRING, timestamps.get(i).intValue(), + TAGK_STRING, "C", TAGK_B_STRING, "D"); + assertArrayEquals(key, rows.get(i).key()); + } + + rows = new ArrayList<GetRequest>(); + for (Entry<byte[], ByteMap<List<GetRequest>>> salt_entry : row_map.entrySet()) { + rows.addAll(salt_entry.getValue().get(getTSUID(METRIC_STRING, TAGK_STRING, + "A", TAGK_B_STRING, "D"))); + } + + assertEquals(timestamps.size(), rows.size()); + for (int i = 0; i < timestamps.size(); i++) { + byte[] key = getRowKey(METRIC_STRING, timestamps.get(i).intValue(), + TAGK_STRING, "A", TAGK_B_STRING, "D"); + assertArrayEquals(key, rows.get(i).key()); + } + + rows = new ArrayList<GetRequest>(); + for (Entry<byte[], ByteMap<List<GetRequest>>> salt_entry : row_map.entrySet()) { + rows.addAll(salt_entry.getValue().get(getTSUID(METRIC_STRING, TAGK_STRING, + "B", TAGK_B_STRING, "D"))); + } + assertEquals(timestamps.size(), rows.size()); + for (int i = 0; i < timestamps.size(); i++) { + byte[] key = getRowKey(METRIC_STRING, timestamps.get(i).intValue(), + TAGK_STRING, "B", TAGK_B_STRING, "D"); + assertArrayEquals(key, rows.get(i).key()); + } + + rows = new ArrayList<GetRequest>(); + for (Entry<byte[], ByteMap<List<GetRequest>>> salt_entry : row_map.entrySet()) { + rows.addAll(salt_entry.getValue().get(getTSUID(METRIC_STRING, TAGK_STRING, + "C", TAGK_B_STRING, "D"))); + } + assertEquals(timestamps.size(), rows.size()); + for (int i = 0; i < timestamps.size(); i++) { + byte[] key = getRowKey(METRIC_STRING, timestamps.get(i).intValue(), + TAGK_STRING, "C", TAGK_B_STRING, "D"); + assertArrayEquals(key, rows.get(i).key()); + } + } + + @Test + public void prepareRowKeysAllSalts() throws Exception { + multiget_no_meta = true; + if (Const.SALT_WIDTH() == 0) { + return; + } + config.overrideConfig("tsd.query.multi_get.get_all_salts", "true"); + + MultiGetQuery mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags_nometa, + start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, + 0, 0, false, multiget_no_meta); + List<Long> timestamps = Lists.newArrayList(1481227200L, 1481230800L); + List<byte[][]> q_tags_compounds = mgq.prepareAllTagvCompounds(); + ByteMap<ByteMap<List<GetRequest>>> row_map_map = mgq.prepareRequestsNoMeta( q_tags_compounds, timestamps); + ByteMap<List<GetRequest>> row_map = row_map_map.get("0".getBytes()); + assertEquals(6, row_map.size()); + + List<GetRequest> rows = row_map.get(getTSUID(METRIC_STRING, TAGK_STRING, + "A", TAGK_B_STRING, "D", "E", "F")); + assertEquals(timestamps.size() * Const.SALT_BUCKETS(), rows.size()); + for (int i = 0; i < timestamps.size(); i += Const.SALT_BUCKETS()) { + for (int x = 0; x < Const.SALT_BUCKETS(); x++) { + byte[] key = getRowKey(METRIC_STRING, timestamps.get(i).intValue(), + TAGK_STRING, "A", TAGK_B_STRING, "D", "E", "F"); + key[0] = (byte) x; + assertArrayEquals(key, rows.get(i + x).key()); + } + } + + rows = row_map.get(getTSUID(METRIC_STRING, TAGK_STRING, + "B", TAGK_B_STRING, "D", "E", "F")); + assertEquals(timestamps.size() * Const.SALT_BUCKETS(), rows.size()); + for (int i = 0; i < timestamps.size(); i += Const.SALT_BUCKETS()) { + for (int x = 0; x < Const.SALT_BUCKETS(); x++) { + byte[] key = getRowKey(METRIC_STRING, timestamps.get(i).intValue(), + TAGK_STRING, "B", TAGK_B_STRING, "D", "E", "F"); + key[0] = (byte) x; + assertArrayEquals(key, rows.get(i + x).key()); + } + } + + rows = row_map.get(getTSUID(METRIC_STRING, TAGK_STRING, + "C", TAGK_B_STRING, "D", "E", "F")); + assertEquals(timestamps.size() * Const.SALT_BUCKETS(), rows.size()); + for (int i = 0; i < timestamps.size(); i += Const.SALT_BUCKETS()) { + for (int x = 0; x < Const.SALT_BUCKETS(); x++) { + byte[] key = getRowKey(METRIC_STRING, timestamps.get(i).intValue(), + TAGK_STRING, "C", TAGK_B_STRING, "D", "E", "F"); + key[0] = (byte) x; + assertArrayEquals(key, rows.get(i + x).key()); + } + } + + rows = row_map.get(getTSUID(METRIC_STRING, TAGK_STRING, + "B", TAGK_B_STRING, "D", "E", "G")); + assertEquals(timestamps.size() * Const.SALT_BUCKETS(), rows.size()); + for (int i = 0; i < timestamps.size(); i += Const.SALT_BUCKETS()) { + for (int x = 0; x < Const.SALT_BUCKETS(); x++) { + byte[] key = getRowKey(METRIC_STRING, timestamps.get(i).intValue(), + TAGK_STRING, "B", TAGK_B_STRING, "D", "E", "G"); + key[0] = (byte) x; + assertArrayEquals(key, rows.get(i + x).key()); + } + } + rows = row_map.get(getTSUID(METRIC_STRING, TAGK_STRING, + "C", TAGK_B_STRING, "D", "E", "G")); + assertEquals(timestamps.size() * Const.SALT_BUCKETS(), rows.size()); + for (int i = 0; i < timestamps.size(); i += Const.SALT_BUCKETS()) { + for (int x = 0; x < Const.SALT_BUCKETS(); x++) { + byte[] key = getRowKey(METRIC_STRING, timestamps.get(i).intValue(), + TAGK_STRING, "C", TAGK_B_STRING, "D", "E", "G"); + key[0] = (byte) x; + assertArrayEquals(key, rows.get(i + x).key()); + } + } + } + + @Test + public void prepareConcurrentMultiGetTasks() throws Exception { + MultiGetQuery mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags, + start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, + 0, max_bytes, false, multiget_no_meta); + mgq.prepareConcurrentMultiGetTasks(); + final List<List<MultiGetTask>> tasks = mgq.getMultiGetTasks(); + + assertEquals(config.getInt("tsd.query.multi_get.concurrent"), tasks.size()); + for (int i = 1; i < tasks.size(); i++) { + assertTrue(tasks.get(i).isEmpty()); + } + + for (List<MultiGetTask> taskList : tasks) { + for (MultiGetTask task : taskList) { + byte salt = task.getGets().get(0).key()[0]; + for (GetRequest request : task.getGets()) { + assertEquals(salt, request.key()[0]); + } + } + } + + assertEquals(1, tasks.get(0).size()); + MultiGetTask task = tasks.get(0).get(0); + assertEquals(3, task.getTSUIDs().size()); + assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, + "A", TAGK_B_STRING, "D"))); + assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, + "B", TAGK_B_STRING, "D"))); + assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, + "C", TAGK_B_STRING, "D"))); + assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, + "A", TAGK_B_STRING, "D"))); + assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, + "B", TAGK_B_STRING, "D"))); + assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, + "C", TAGK_B_STRING, "D"))); + + assertEquals(51, task.getGets().size()); + + + // 6 sets of 17 timestamps. Ugly UT + int idx = 0; + int ts = 1481227200; + while (idx < 17) { + + assertArrayEquals(task.getGets().get(idx++).key(), + getRowKey(METRIC_STRING, ts, TAGK_STRING, "A", TAGK_B_STRING, "D")); + ts += 3600; + } + + ts = 1481227200; + while (idx < 17) { + assertArrayEquals(task.getGets().get(idx++).key(), + getRowKey(METRIC_STRING, ts, TAGK_STRING, "B", TAGK_B_STRING, "D")); + ts += 3600; + } + ts = 1481227200; + + while (idx < 17) { + assertArrayEquals(task.getGets().get(idx++).key(), + getRowKey(METRIC_STRING, ts, TAGK_STRING, "C", TAGK_B_STRING, "D")); + ts += 3600; + } + + } + + @Test + public void prepareConcurrentMultiGetTasksSmallBatch() throws Exception { + config.overrideConfig("tsd.query.multi_get.batch_size", "17"); + MultiGetQuery mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags, + start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, + 0, max_bytes, false, multiget_no_meta); + mgq.prepareConcurrentMultiGetTasks(); + final List<List<MultiGetTask>> tasks = mgq.getMultiGetTasks(); + assertEquals(config.getInt("tsd.query.multi_get.concurrent"), tasks.size()); + assertEquals(1, tasks.get(0).size()); + + // first batch + MultiGetTask task = tasks.get(0).get(0); + Set<byte[]> tsuids = task.getTSUIDs(); + assertEquals(1, task.getTSUIDs().size()); + assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, + "A", TAGK_B_STRING, "D"))); + assertEquals(17, task.getGets().size()); + int idx = 0; + int ts = 1481227200; + while (idx < 17) { // notice the early cut off. The last hour should spill over. + assertArrayEquals(task.getGets().get(idx++).key(), + getRowKey(METRIC_STRING, ts, TAGK_STRING, "A", TAGK_B_STRING, "D")); + ts += 3600; + } + + // next batch + task = tasks.get(1).get(0); + assertEquals(1, task.getTSUIDs().size()); + assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, + "B", TAGK_B_STRING, "D"))); + assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, + "B", TAGK_B_STRING, "D"))); + assertEquals(17, task.getGets().size()); + idx = 0; + ts = 1481227200; + + + while (idx < 17) { + assertArrayEquals(task.getGets().get(idx++).key(), + getRowKey(METRIC_STRING, ts, TAGK_STRING, "B", TAGK_B_STRING, "D")); + ts += 3600; + } + + // next batch + task = tasks.get(2).get(0); + assertEquals(1, task.getTSUIDs().size()); + assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, + "C", TAGK_B_STRING, "D"))); + assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, + "C", TAGK_B_STRING, "D"))); + assertEquals(17, task.getGets().size()); + idx = 0; + ts = 1481227200; + while (idx < 17) { + assertArrayEquals(task.getGets().get(idx++).key(), + getRowKey(METRIC_STRING, ts, TAGK_STRING, "C", TAGK_B_STRING, "D")); + ts += 3600; + } + + } + + + @Test + public void prepareConcurrentMultiSortedSalts() + throws Exception { + config.overrideConfig("tsd.query.multi_get.concurrent", "2"); + config.overrideConfig("tsd.query.multi_get.batch_size", "2"); + Const.setSaltWidth(1); + MultiGetQuery mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags, + start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, + 0, max_bytes, false, multiget_no_meta); + mgq.prepareConcurrentMultiGetTasks(); + final List<List<MultiGetTask>> tasks = mgq.getMultiGetTasks(); + assertEquals(config.getInt("tsd.query.multi_get.concurrent"), tasks.size()); + + for (List<MultiGetTask> taskList : tasks) { + for (MultiGetTask task : taskList) { + byte salt = task.getGets().get(0).key()[0]; + for (GetRequest request : task.getGets()) { + assertEquals(salt, request.key()[0]); + } + } + } + Const.setSaltWidth(0); + + } + + @Test + public void prepareConcurrentMultiGetTasksSmallBatchAndSmallConcurrent() + throws Exception { + config.overrideConfig("tsd.query.multi_get.concurrent", "2"); + config.overrideConfig("tsd.query.multi_get.batch_size", "17"); + MultiGetQuery mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags, + start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, + 0, max_bytes, false, multiget_no_meta); + mgq.prepareConcurrentMultiGetTasks(); + final List<List<MultiGetTask>> tasks = mgq.getMultiGetTasks(); + assertEquals(config.getInt("tsd.query.multi_get.concurrent"), tasks.size()); + assertEquals(2, tasks.get(0).size()); + assertEquals(1, tasks.get(1).size()); + + for (List<MultiGetTask> taskList : tasks) { + for (MultiGetTask task : taskList) { + byte salt = task.getGets().get(0).key()[0]; + for (GetRequest request : task.getGets()) { + assertEquals(salt, request.key()[0]); + } + } + } + + // first batch + MultiGetTask task = tasks.get(0).get(0); + assertEquals(1, task.getTSUIDs().size()); + assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, + "A", TAGK_B_STRING, "D"))); + assertEquals(17, task.getGets().size()); + int idx = 0; + int ts = 1481227200; + while (idx < 16) { // notice the early cut off. The last hour should spill over. + + assertArrayEquals(task.getGets().get(idx++).key(), + getRowKey(METRIC_STRING, ts, TAGK_STRING, "A", TAGK_B_STRING, "D")); + ts += 3600; + } + // 17th request + assertArrayEquals(task.getGets().get(idx++).key(), + getRowKey(METRIC_STRING, ts, TAGK_STRING, "A", TAGK_B_STRING, "D")); + ts += 3600; + // next batch + task = tasks.get(1).get(0); + assertEquals(1, task.getTSUIDs().size()); + assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, + "B", TAGK_B_STRING, "D"))); + assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, + "B", TAGK_B_STRING, "D"))); + assertEquals(17, task.getGets().size()); + idx = 0; + ts = 1481227200; + +// assertArrayEquals(task.getGets().get(idx++).key(), +// getRowKey(METRIC_STRING, ts, TAGK_STRING, "B", TAGK_B_STRING, "D")); + + while (idx < 17) { + assertArrayEquals(task.getGets().get(idx++).key(), + getRowKey(METRIC_STRING, ts, TAGK_STRING, "B", TAGK_B_STRING, "D")); + ts += 3600; + } + + // next batch + task = tasks.get(0).get(1); + assertEquals(1, task.getTSUIDs().size()); + assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, + "C", TAGK_B_STRING, "D"))); + assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, + "C", TAGK_B_STRING, "D"))); + assertEquals(17, task.getGets().size()); + idx = 0; + ts = 1481227200; + while (idx < 17) { + assertArrayEquals(task.getGets().get(idx++).key(), + getRowKey(METRIC_STRING, ts, TAGK_STRING, "C", TAGK_B_STRING, "D")); + ts += 3600; + } + + + } + + @Test + public void fetch() throws Exception { + setupStorage(); + MultiGetQuery mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags, + start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, + 0, max_bytes, false, multiget_no_meta); + + final SortedMap<byte[], Span> results = mgq.fetch().join(); + assertSame(spans, results); + verify(client, times(1)).get(anyList()); + System.out.println(spans); + validateSpans(); + } + + @Test + public void fetchMultigetNoMeta() throws Exception { + setupStorageNoMeta(); + multiget_no_meta = true; + MultiGetQuery mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags_nometa, + start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, + 0, max_bytes, false, multiget_no_meta); + + final SortedMap<byte[], Span> results = mgq.fetch().join(); + assertSame(spans, results); + verify(client, times(1)).get(anyList()); + System.out.println(spans); + validateSpansNometa(); + } + + @Test (expected=QueryException.class) + public void fetchMoreThanMaxBytes() throws Exception { + setupStorage(); + max_bytes = 0; + MultiGetQuery mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags, + start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, + 0, max_bytes, false, multiget_no_meta); + final SortedMap<byte[], Span> results = mgq.fetch().join(); + } + + @Test + public void fetchEmptyTable() throws Exception { + setDataPointStorage(); + spans = new TreeMap<byte[], Span>(new TsdbQuery.SpanCmp(TSDB.metrics_width())); + MultiGetQuery mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags, + start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, + 0, max_bytes, false, multiget_no_meta); + + final SortedMap<byte[], Span> results = mgq.fetch().join(); + assertSame(spans, results); + assertTrue(spans.isEmpty()); + verify(client, times(1)).get(anyList()); + } + + @Test + public void fetchSmallBatch() throws Exception { + setupStorage(); + config.overrideConfig("tsd.query.multi_get.batch_size", "16"); + + MultiGetQuery mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags, + start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, + 0, max_bytes, false, multiget_no_meta); + + final SortedMap<byte[], Span> results = mgq.fetch().join(); + assertSame(spans, results); + verify(client, times(4)).get(anyList()); + validateSpans(); + } + + @Test + public void fetchSmallBatchAndSmallConcurrent() throws Exception { + setupStorage(); + config.overrideConfig("tsd.query.multi_get.concurrent", "2"); + config.overrideConfig("tsd.query.multi_get.batch_size", "16"); + + MultiGetQuery mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags, + start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, + 0, max_bytes, false, multiget_no_meta); + + final SortedMap<byte[], Span> results = mgq.fetch().join(); + assertSame(spans, results); + verify(client, times(4)).get(anyList()); + validateSpans(); + } + + @Test + public void fetchException() throws Exception { + setupStorage(); + final RuntimeException e = new RuntimeException("Boo!"); + storage.throwException(getRowKey(METRIC_STRING, (int) start_ts, TAGK_STRING, + "A", TAGK_B_STRING, "D"), e); + MultiGetQuery mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags_AD, + start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, + 0, 10000000, false, false); + + try { + mgq.fetch().join(); + fail("Expected RuntimeException"); + } catch (RuntimeException ex) { + assertSame(ex, e); + } + verify(client, times(1)).get(anyList()); + } + + /** + * Validates the data setup in {@link #setupStorage()} is returned in the requests. + * @throws Exception if something went pear shaped. + */ + protected void validateSpansNometa() throws Exception { + assertEquals(6, spans.size()); + Span span = spans.get(getRowKey(METRIC_STRING, (int) start_ts, TAGK_STRING, + "A", TAGK_B_STRING, "D", "E", "F")); + SeekableView view = span.iterator(); + long ts = start_ts * 1000; + long v = 1; + while (view.hasNext()) { + DataPoint dp = view.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(v++, dp.longValue()); + ts += 3600000; + } + + span = spans.get(getRowKey(METRIC_STRING, (int) start_ts, TAGK_STRING, + "B", TAGK_B_STRING, "D", "E", "F")); + view = span.iterator(); + ts = start_ts * 1000; + v = 11; + while (view.hasNext()) { + DataPoint dp = view.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(v++, dp.longValue()); + ts += 3600000; + } + + span = spans.get(getRowKey(METRIC_STRING, (int) start_ts, TAGK_STRING, + "C", TAGK_B_STRING, "D", "E", "F")); + view = span.iterator(); + ts = start_ts * 1000; + v = 111; + while (view.hasNext()) { + DataPoint dp = view.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(v++, dp.longValue()); + ts += 3600000; + } + + span = spans.get(getRowKey(METRIC_STRING, (int) start_ts, TAGK_STRING, + "A", TAGK_B_STRING, "D", "E", "G")); + view = span.iterator(); + ts = start_ts * 1000; + v = 1111; + while (view.hasNext()) { + DataPoint dp = view.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(v++, dp.longValue()); + ts += 3600000; + } + + span = spans.get(getRowKey(METRIC_STRING, (int) start_ts, TAGK_STRING, + "B", TAGK_B_STRING, "D", "E", "G")); + view = span.iterator(); + ts = start_ts * 1000; + v = 11111; + while (view.hasNext()) { + DataPoint dp = view.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(v++, dp.longValue()); + ts += 3600000; + } + + span = spans.get(getRowKey(METRIC_STRING, (int) start_ts, TAGK_STRING, + "C", TAGK_B_STRING, "D", "E", "G")); + view = span.iterator(); + ts = start_ts * 1000; + v = 111111; + while (view.hasNext()) { + DataPoint dp = view.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(v++, dp.longValue()); + ts += 3600000; + } + } + + /** + * Validates the data setup in {@link #setupStorage()} is returned in the requests. + * @throws Exception if something went pear shaped. + */ + protected void validateSpans() throws Exception { + System.out.println(spans); + assertEquals(3, spans.size()); + Span span = spans.get(getRowKey(METRIC_STRING, (int) start_ts, TAGK_STRING, + "A", TAGK_B_STRING, "D")); + SeekableView view = span.iterator(); + long ts = start_ts * 1000; + long v = 1; + while (view.hasNext()) { + DataPoint dp = view.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(v++, dp.longValue()); + ts += 3600000; + } + + span = spans.get(getRowKey(METRIC_STRING, (int) start_ts, TAGK_STRING, + "B", TAGK_B_STRING, "D")); + view = span.iterator(); + ts = start_ts * 1000; + v = 11; + while (view.hasNext()) { + DataPoint dp = view.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(v++, dp.longValue()); + ts += 3600000; + } + + span = spans.get(getRowKey(METRIC_STRING, (int) start_ts, TAGK_STRING, + "C", TAGK_B_STRING, "D")); + view = span.iterator(); + ts = start_ts * 1000; + v = 111; + while (view.hasNext()) { + DataPoint dp = view.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(v++, dp.longValue()); + ts += 3600000; + } + + span = spans.get(getRowKey(METRIC_STRING, (int) start_ts, TAGK_STRING, + "A", TAGK_B_STRING, "D")); + view = span.iterator(); + ts = start_ts * 1000; + v = 1; + while (view.hasNext()) { + DataPoint dp = view.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(v++, dp.longValue()); + ts += 3600000; + } + + span = spans.get(getRowKey(METRIC_STRING, (int) start_ts, TAGK_STRING, + "B", TAGK_B_STRING, "D")); + view = span.iterator(); + ts = start_ts * 1000; + v = 11; + while (view.hasNext()) { + DataPoint dp = view.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(v++, dp.longValue()); + ts += 3600000; + } + + span = spans.get(getRowKey(METRIC_STRING, (int) start_ts, TAGK_STRING, + "C", TAGK_B_STRING, "D")); + view = span.iterator(); + ts = start_ts * 1000; + v = 111; + while (view.hasNext()) { + DataPoint dp = view.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(v++, dp.longValue()); + ts += 3600000; + } + } + + /** + * Helper that writes a data point for each row that the get request should + * cover. + * @throws Exception if something went pear shaped. + */ + protected void setupStorageNoMeta() throws Exception { + setDataPointStorage(); + spans = new TreeMap<byte[], Span>(new TsdbQuery.SpanCmp(TSDB.metrics_width())); + + int value = 1; + int ts = (int) start_ts; + tags.clear(); + tags.put(TAGK_STRING, "A"); + tags.put(TAGK_B_STRING, "D"); + tags.put("E", "F"); + while (ts <= (int) end_ts) { + tsdb.addPoint(METRIC_STRING, (long) ts, (long) value++, tags).join(); + ts += 3600; + } + + value = 11; + ts = (int) start_ts; + tags.clear(); + tags.put(TAGK_STRING, "B"); + tags.put(TAGK_B_STRING, "D"); + tags.put("E", "F"); + while (ts <= (int) end_ts) { + tsdb.addPoint(METRIC_STRING, (long) ts, (long) value++, tags).join(); + ts += 3600; + } + + value = 111; + ts = (int) start_ts; + tags.clear(); + tags.put(TAGK_STRING, "C"); + tags.put(TAGK_B_STRING, "D"); + tags.put("E", "F"); + while (ts <= (int) end_ts) { + tsdb.addPoint(METRIC_STRING, (long) ts, (long) value++, tags).join(); + ts += 3600; + } + + value = 1111; + ts = (int) start_ts; + tags.clear(); + tags.put(TAGK_STRING, "A"); + tags.put(TAGK_B_STRING, "D"); + tags.put("E", "G"); + while (ts <= (int) end_ts) { + tsdb.addPoint(METRIC_STRING, (long) ts, (long) value++, tags).join(); + ts += 3600; + } + + value = 11111; + ts = (int) start_ts; + tags.clear(); + tags.put(TAGK_STRING, "B"); + tags.put(TAGK_B_STRING, "D"); + tags.put("E", "G"); + while (ts <= (int) end_ts) { + tsdb.addPoint(METRIC_STRING, (long) ts, (long) value++, tags).join(); + ts += 3600; + } + + value = 111111; + ts = (int) start_ts; + tags.clear(); + tags.put(TAGK_STRING, "C"); + tags.put(TAGK_B_STRING, "D"); + tags.put("E", "G"); + while (ts <= (int) end_ts) { + tsdb.addPoint(METRIC_STRING, (long) ts, (long) value++, tags).join(); + ts += 3600; + } + } + + + /** + * Helper that writes a data point for each row that the get request should + * cover. + * @throws Exception if something went pear shaped. + */ + protected void setupStorage() throws Exception { + setDataPointStorage(); + spans = new TreeMap<byte[], Span>(new TsdbQuery.SpanCmp(TSDB.metrics_width())); + + int value = 1; + int ts = (int) start_ts; + tags.clear(); + tags.put(TAGK_STRING, "A"); + tags.put(TAGK_B_STRING, "D"); + //tags.put("E", "F"); + while (ts <= (int) end_ts) { + tsdb.addPoint(METRIC_STRING, (long) ts, (long) value++, tags).join(); + ts += 3600; + } + + value = 11; + ts = (int) start_ts; + tags.clear(); + tags.put(TAGK_STRING, "B"); + tags.put(TAGK_B_STRING, "D"); + // tags.put("E", "F"); + while (ts <= (int) end_ts) { + tsdb.addPoint(METRIC_STRING, (long) ts, (long) value++, tags).join(); + ts += 3600; + } + + value = 111; + ts = (int) start_ts; + tags.clear(); + tags.put(TAGK_STRING, "C"); + tags.put(TAGK_B_STRING, "D"); + // tags.put("E", "F"); + while (ts <= (int) end_ts) { + tsdb.addPoint(METRIC_STRING, (long) ts, (long) value++, tags).join(); + ts += 3600; + } + +// value = 1111; +// ts = (int) start_ts; +// tags.clear(); +// tags.put(TAGK_STRING, "A"); +// tags.put(TAGK_B_STRING, "D"); +// // tags.put("E", "G"); +// while (ts <= (int) end_ts) { +// tsdb.addPoint(METRIC_STRING, (long) ts, (long) value++, tags).join(); +// ts += 3600; +// } +// +// value = 11111; +// ts = (int) start_ts; +// tags.clear(); +// tags.put(TAGK_STRING, "B"); +// tags.put(TAGK_B_STRING, "D"); +// // tags.put("E", "G"); +// while (ts <= (int) end_ts) { +// tsdb.addPoint(METRIC_STRING, (long) ts, (long) value++, tags).join(); +// ts += 3600; +// } +// +// value = 111111; +// ts = (int) start_ts; +// tags.clear(); +// tags.put(TAGK_STRING, "C"); +// tags.put(TAGK_B_STRING, "D"); +// // tags.put("E", "G"); +// while (ts <= (int) end_ts) { +// tsdb.addPoint(METRIC_STRING, (long) ts, (long) value++, tags).join(); +// ts += 3600; +// } + } +} + + diff --git a/test/core/TestRateSpan.java b/test/core/TestRateSpan.java index afa1d59488..296b322dce 100644 --- a/test/core/TestRateSpan.java +++ b/test/core/TestRateSpan.java @@ -254,4 +254,61 @@ public void testNext_counterWithResetValue() { assertFalse(rate_span.hasNext()); assertFalse(rate_span.hasNext()); } + + @Test + public void testNext_counterDroResets() { + final long RESET_VALUE = 1; + source = SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(1356998400000L, 40), + MutableDataPoint.ofLongValue(1356998401000L, 50), + MutableDataPoint.ofLongValue(1356998402000L, 40), + MutableDataPoint.ofLongValue(1356998403000L, 50) + }); + DataPoint[] rates = new DataPoint[] { + MutableDataPoint.ofDoubleValue(1356998400000L, 40 / 1356998400.0), + MutableDataPoint.ofDoubleValue(1356998401000L, 10), + // drop the point before + MutableDataPoint.ofDoubleValue(1356998403000L, 10) + }; + options = new RateOptions(true, COUNTER_MAX, RESET_VALUE, true); + RateSpan rate_span = new RateSpan(source, options); + for (DataPoint rate : rates) { + assertTrue(rate_span.hasNext()); + assertTrue(rate_span.hasNext()); + DataPoint dp = rate_span.next(); + String msg = String.format("expected rate = '%s' ", rate); + assertFalse(msg, dp.isInteger()); + assertEquals(msg, rate.timestamp(), dp.timestamp()); + assertEquals(msg, rate.doubleValue(), dp.doubleValue(), 0.0000001); + } + assertFalse(rate_span.hasNext()); + assertFalse(rate_span.hasNext()); + } + + @Test + public void testNext_counterDroResetsNothingAfter() { + final long RESET_VALUE = 1; + source = SeekableViewsForTest.fromArray(new DataPoint[] { + MutableDataPoint.ofLongValue(1356998400000L, 40), + MutableDataPoint.ofLongValue(1356998401000L, 50), + MutableDataPoint.ofLongValue(1356998402000L, 40) + }); + DataPoint[] rates = new DataPoint[] { + MutableDataPoint.ofDoubleValue(1356998400000L, 40 / 1356998400.0), + MutableDataPoint.ofDoubleValue(1356998401000L, 10), + }; + options = new RateOptions(true, COUNTER_MAX, RESET_VALUE, true); + RateSpan rate_span = new RateSpan(source, options); + for (DataPoint rate : rates) { + assertTrue(rate_span.hasNext()); + assertTrue(rate_span.hasNext()); + DataPoint dp = rate_span.next(); + String msg = String.format("expected rate = '%s' ", rate); + assertFalse(msg, dp.isInteger()); + assertEquals(msg, rate.timestamp(), dp.timestamp()); + assertEquals(msg, rate.doubleValue(), dp.doubleValue(), 0.0000001); + } + assertFalse(rate_span.hasNext()); + assertFalse(rate_span.hasNext()); + } } diff --git a/test/core/TestRollupSpan.java b/test/core/TestRollupSpan.java new file mode 100644 index 0000000000..5aa177b216 --- /dev/null +++ b/test/core/TestRollupSpan.java @@ -0,0 +1,445 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2013 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import org.hbase.async.Bytes; +import org.hbase.async.KeyValue; +import org.junit.Before; +import org.junit.Test; +import org.powermock.reflect.Whitebox; + +import net.opentsdb.rollup.RollupConfig; +import net.opentsdb.rollup.RollupInterval; +import net.opentsdb.rollup.RollupQuery; +import net.opentsdb.rollup.RollupSpan; +import static net.opentsdb.rollup.RollupUtils.ROLLUP_QUAL_DELIM; + +public class TestRollupSpan extends BaseTsdbTest { + protected byte[] hour1 = null; + protected byte[] hour2 = null; + protected byte[] hour3 = null; + protected RollupConfig rollup_config; + protected static final Aggregator aggr_sum = Aggregators.SUM; + + protected static final RollupQuery rollup_query = + new RollupQuery(RollupInterval.builder() + .setTable("tsdb") + .setPreAggregationTable("tsdb-agg") + .setInterval("1s") + .setRowSpan("1h") + .build(), + aggr_sum, + 1000, + aggr_sum); + + @Before + public void beforeLocal() throws Exception { + hour1 = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_STRING); + hour2 = getRowKey(METRIC_STRING, 1357002000, TAGK_STRING, TAGV_STRING); + hour3 = getRowKey(METRIC_STRING, 1357005600, TAGK_STRING, TAGV_STRING); + + rollup_config = RollupConfig.builder() + .addAggregationId("sum", 0) + .addAggregationId("count", 1) + .addAggregationId("max", 2) + .addAggregationId("min", 3) + .addInterval(RollupInterval.builder() + .setTable("tsdb-rollup-10m") + .setPreAggregationTable("tsdb-rollup-agg-10m") + .setInterval("10m") + .setRowSpan("6h")) + .addInterval(RollupInterval.builder() + .setTable("tsdb-rollup-1h") + .setPreAggregationTable("tsdb-rollup-agg-1h") + .setInterval("1h") + .setRowSpan("1d")) + .addInterval(RollupInterval.builder() + .setTable("tsdb-rollup-1d") + .setPreAggregationTable("tsdb-rollup-agg-1d") + .setInterval("1d") + .setRowSpan("1n")) + .build(); + Whitebox.setInternalState(tsdb, "rollup_config", rollup_config); + } + + @Test + public void addRow() { + final byte[] qual1 = {0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + + final Span span = new RollupSpan(tsdb, rollup_query); + span.addRow(new KeyValue(hour1, TSDB.FAMILY(), qual1, val1)); + + assertEquals(1, span.size()); + } + + @Test (expected = NullPointerException.class) + public void addRowNull() { + final Span span = new RollupSpan(tsdb, rollup_query); + span.addRow(null); + } + /* + * TODO - fix up these tests + @Test (expected = IllegalArgumentException.class) + public void addRowBadKeyLength() { + final byte[] qual1 = {0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = {0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }; + final byte[] val2 = Bytes.fromLong(8L); + + final Span span = new RollupSpan(tsdb, rollup_query); + span.addRow(new KeyValue(HOUR1, TSDB.FAMILY(), qual1, val1)); + + final byte[] bad_key = + new byte[] { 0, 0, 0, 1, 0x50, (byte)0xE2, 0x43, 0x20, 0, 0, 0, 1 }; + span.addRow(new KeyValue(bad_key, TSDB.FAMILY(), qual2, + MockBase.concatByteArrays(val1, val2, ZERO))); + } + + @Test (expected = IllegalArgumentException.class) + public void addRowMissMatchedMetric() { + final byte[] qual1 = {0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = {0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }; + final byte[] val2 = Bytes.fromLong(8L); + + final Span span = new RollupSpan(tsdb, rollup_query); + span.addRow(new KeyValue(HOUR1, TSDB.FAMILY(), qual1,val1)); + + final byte[] bad_key = + new byte[] { 0, 0, 0, 2, 0x50, (byte)0xE2, 0x35, 0x10, 0, 0, 0, 1, 0, 0, 0, 2 }; + span.addRow(new KeyValue(bad_key, TSDB.FAMILY(), qual2,val2)); + } + + @Test (expected = IllegalArgumentException.class) + public void addRowMissMatchedTagk() { + final byte[] qual1 = {0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = {0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }; + final byte[] val2 = Bytes.fromLong(8L); + + final Span span = new RollupSpan(tsdb, rollup_query); + span.addRow(new KeyValue(HOUR1, TSDB.FAMILY(), qual1, val1)); + + final byte[] bad_key = + new byte[] { 0, 0, 0, 1, 0x50, (byte)0xE2, 0x35, 0x10, 0, 0, 0, 2, 0, 0, 0, 2 }; + span.addRow(new KeyValue(bad_key, TSDB.FAMILY(), qual2, val2)); + } + + @Test (expected = IllegalArgumentException.class) + public void addRowMissMatchedTagv() { + final byte[] qual1 = {0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = {0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }; + final byte[] val2 = Bytes.fromLong(8L); + + final Span span = new RollupSpan(tsdb, rollup_query); + span.addRow(new KeyValue(HOUR1, TSDB.FAMILY(), qual1, val1)); + + final byte[] bad_key = + new byte[] { 0, 0, 0, 1, 0x50, (byte)0xE2, 0x35, 0x10, 0, 0, 0, 1, 0, 0, 0, 3 }; + span.addRow(new KeyValue(bad_key, TSDB.FAMILY(), qual2, val2)); + } + + @Test + public void addRowOutOfOrder() { + //2nd hour + final byte[] qual1 = {0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + //1st hour + final byte[] qual2 = {0x73, 0x75, 0x6D, 0x3A, 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + + final Span span = new RollupSpan(tsdb, rollup_query); + span.addRow(new KeyValue(HOUR2, TSDB.FAMILY(), qual1, val1)); + span.addRow(new KeyValue(HOUR1, TSDB.FAMILY(), qual2, val2)); + assertEquals(2, span.size()); + + assertEquals(1356998402000L, span.timestamp(0)); + assertEquals(5, span.longValue(0)); + assertEquals(1357002000000L, span.timestamp(1)); + assertEquals(4, span.longValue(1)); + } + + @Test + public void addDifferentSalt() throws Exception { + List<byte[]> qualifiers = new ArrayList<byte[]>(); + List<byte[]> values = new ArrayList<byte[]>(); + + qualifiers.add(new byte[]{ 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }); + qualifiers.add(new byte[]{ 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x27 }); + qualifiers.add(new byte[]{ 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x37 }); + qualifiers.add(new byte[]{ 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x47 }); + values.add(Bytes.fromLong(4L)); + values.add(Bytes.fromLong(5L)); + + final Span span = new RollupSpan(tsdb, rollup_query); + final byte[] hour1 = Arrays.copyOf(HOUR1, HOUR1.length); + hour1[0] = 2; + span.addRow(new KeyValue(HOUR1, TSDB.FAMILY(), qualifiers.get(0), values.get(0))); + span.addRow(new KeyValue(HOUR1, TSDB.FAMILY(), qualifiers.get(1), values.get(1))); + span.addRow(new KeyValue(hour1, TSDB.FAMILY(), qualifiers.get(2), values.get(0))); + span.addRow(new KeyValue(hour1, TSDB.FAMILY(), qualifiers.get(3), values.get(1))); + assertEquals(4, span.size()); + assertEquals(1356998400000L, span.timestamp(0)); + assertEquals(4, span.longValue(0)); + assertEquals(1356998402000L, span.timestamp(1)); + assertEquals(5, span.longValue(1)); + assertEquals(1356998403000L, span.timestamp(2)); + assertEquals(4, span.longValue(2)); + assertEquals(1356998404000L, span.timestamp(3)); + assertEquals(5, span.longValue(3)); + } + + @Test + public void addDifferentSaltDiffHour() throws Exception { + List<byte[]> qualifiers = new ArrayList<byte[]>(); + List<byte[]> values = new ArrayList<byte[]>(); + + qualifiers.add(new byte[]{ 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }); + values.add(Bytes.fromLong(4L)); + qualifiers.add(new byte[]{ 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x27 }); + values.add(Bytes.fromLong(5L)); + + final Span span = new RollupSpan(tsdb, rollup_query); + final byte[] hour2 = Arrays.copyOf(HOUR2, HOUR2.length); + hour2[0] = 2; + span.addRow(new KeyValue(HOUR1, TSDB.FAMILY(), qualifiers.get(0), values.get(0))); + span.addRow(new KeyValue(HOUR1, TSDB.FAMILY(), qualifiers.get(1), values.get(1))); + span.addRow(new KeyValue(hour2, TSDB.FAMILY(), qualifiers.get(0), values.get(0))); + span.addRow(new KeyValue(hour2, TSDB.FAMILY(), qualifiers.get(1), values.get(1))); + assertEquals(4, span.size()); + assertEquals(1356998400000L, span.timestamp(0)); + assertEquals(4, span.longValue(0)); + assertEquals(1356998402000L, span.timestamp(1)); + assertEquals(5, span.longValue(1)); + assertEquals(1357002000000L, span.timestamp(2)); + assertEquals(4, span.longValue(2)); + assertEquals(1357002002000L, span.timestamp(3)); + assertEquals(5, span.longValue(3)); + } + + @Test + public void addDifferentSaltDiffHourOO() throws Exception { + when(tsdb.followAppendRowLogic()).thenReturn(true); + List<byte[]> qualifiers = new ArrayList<byte[]>(); + List<byte[]> values = new ArrayList<byte[]>(); + + + qualifiers.add(new byte[]{ 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }); + values.add(Bytes.fromLong(4L)); + qualifiers.add(new byte[]{ 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x27 }); + values.add(Bytes.fromLong(5L)); + + final Span span = new RollupSpan(tsdb, rollup_query); + final byte[] hour2 = Arrays.copyOf(HOUR2, HOUR2.length); + hour2[0] = 2; + span.addRow(new KeyValue(hour2, TSDB.FAMILY(), qualifiers.get(0), values.get(0))); + span.addRow(new KeyValue(hour2, TSDB.FAMILY(), qualifiers.get(1), values.get(1))); + span.addRow(new KeyValue(HOUR1, TSDB.FAMILY(), qualifiers.get(0), values.get(0))); + span.addRow(new KeyValue(HOUR1, TSDB.FAMILY(), qualifiers.get(1), values.get(1))); + + assertEquals(4, span.size()); + assertEquals(1356998400000L, span.timestamp(0)); + assertEquals(4, span.longValue(0)); + assertEquals(1356998402000L, span.timestamp(1)); + assertEquals(5, span.longValue(1)); + assertEquals(1357002000000L, span.timestamp(2)); + assertEquals(4, span.longValue(2)); + assertEquals(1357002002000L, span.timestamp(3)); + assertEquals(5, span.longValue(3)); + } + + @Test (expected = IllegalArgumentException.class) + public void addDifferentKey() throws Exception { + when(tsdb.followAppendRowLogic()).thenReturn(true); + List<byte[]> qualifiers = new ArrayList<byte[]>(); + List<byte[]> values = new ArrayList<byte[]>(); + + qualifiers.add(new byte[]{ 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }); + values.add(Bytes.fromLong(4L)); + qualifiers.add(new byte[]{ 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x27 }); + values.add(Bytes.fromLong(5L)); + qualifiers.add(new byte[]{ 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x37 }); + qualifiers.add(new byte[]{ 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x47 }); + + final Span span = new RollupSpan(tsdb, rollup_query); + final byte[] hour1 = Arrays.copyOf(HOUR1, HOUR1.length); + hour1[hour1.length - 1] = 3; + span.addRow(new KeyValue(HOUR1, TSDB.FAMILY(), qualifiers.get(0), values.get(0))); + span.addRow(new KeyValue(HOUR1, TSDB.FAMILY(), qualifiers.get(1), values.get(1))); + span.addRow(new KeyValue(hour1, TSDB.FAMILY(), qualifiers.get(2), values.get(0))); + span.addRow(new KeyValue(hour1, TSDB.FAMILY(), qualifiers.get(3), values.get(1))); + } + + @Test (expected = IllegalArgumentException.class) + public void addDifferentSaltAndKey() throws Exception { + when(tsdb.followAppendRowLogic()).thenReturn(true); + List<byte[]> qualifiers = new ArrayList<byte[]>(); + List<byte[]> values = new ArrayList<byte[]>(); + + + qualifiers.add(new byte[]{ 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }); + values.add(Bytes.fromLong(4L)); + qualifiers.add(new byte[]{ 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x27 }); + values.add(Bytes.fromLong(5L)); + qualifiers.add(new byte[]{ 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x37 }); + qualifiers.add(new byte[]{ 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x47 }); + + final Span span = new RollupSpan(tsdb, rollup_query); + final byte[] hour1 = Arrays.copyOf(HOUR1, HOUR1.length); + hour1[0] = 2; + hour1[hour1.length - 1] = 3; + span.addRow(new KeyValue(HOUR1, TSDB.FAMILY(), qualifiers.get(0), values.get(0))); + span.addRow(new KeyValue(HOUR1, TSDB.FAMILY(), qualifiers.get(1), values.get(1))); + span.addRow(new KeyValue(hour1, TSDB.FAMILY(), qualifiers.get(2), values.get(0))); + span.addRow(new KeyValue(hour1, TSDB.FAMILY(), qualifiers.get(3), values.get(1))); + } + */ + @Test + public void timestampNormalized() throws Exception { + final byte[] qual1 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + + final Span span = new RollupSpan(tsdb, rollup_query); + span.addRow(new KeyValue(hour1, TSDB.FAMILY(), qual1, val1)); + span.addRow(new KeyValue(hour1, TSDB.FAMILY(), qual2, val2)); + span.addRow(new KeyValue(hour2, TSDB.FAMILY(), qual1, val1)); + span.addRow(new KeyValue(hour2, TSDB.FAMILY(), qual2, val2)); + span.addRow(new KeyValue(hour3, TSDB.FAMILY(), qual1, val1)); + span.addRow(new KeyValue(hour3, TSDB.FAMILY(), qual2, val2)); + + assertEquals(6, span.size()); + assertEquals(1356998400000L, span.timestamp(0)); + assertEquals(1356998402000L, span.timestamp(1)); + assertEquals(1357002000000L, span.timestamp(2)); + assertEquals(1357002002000L, span.timestamp(3)); + assertEquals(1357005600000L, span.timestamp(4)); + assertEquals(1357005602000L, span.timestamp(5)); + } + +// @Test + public void timestampFullSeconds() throws Exception { + //6 = num of bytes ("sum) + num of bytes (":") + num of bytes in actual qual + final byte[] agg = (aggr_sum.toString() + ROLLUP_QUAL_DELIM). + getBytes(Const.ASCII_CHARSET); + final byte[] qualifiers = new byte[agg.length + 2]; + System.arraycopy(agg, 0, qualifiers, 0, agg.length); + final Span span = new RollupSpan(tsdb, rollup_query); + + for (int i = 0; i < 100; i++) { + final short qualifier = (short) (i << Const.FLAG_BITS | 0x07); + System.arraycopy(Bytes.fromShort(qualifier), 0, qualifiers, agg.length, 2); + span.addRow(new KeyValue(hour1, TSDB.FAMILY(), qualifiers, Bytes.fromLong(i))); + span.addRow(new KeyValue(hour2, TSDB.FAMILY(), qualifiers, Bytes.fromLong(i))); + span.addRow(new KeyValue(hour3, TSDB.FAMILY(), qualifiers, Bytes.fromLong(i))); + } + + + assertEquals(3600 * 3, span.size()); + } + + @Test + public void timestampMS() throws Exception { + final byte[] qual1 = { 0x73, 0x75, 0x6D, 0x3A, (byte) 0xF0, 0x00, 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x73, 0x75, 0x6D, 0x3A, (byte) 0xF0, 0x00, 0x02, 0x07 }; + final byte[] val2 = Bytes.fromLong(5L); + + final Span span = new RollupSpan(tsdb, rollup_query); + span.addRow(new KeyValue(hour1, TSDB.FAMILY(), qual1, val1)); + span.addRow(new KeyValue(hour1, TSDB.FAMILY(), qual2, val2)); + span.addRow(new KeyValue(hour2, TSDB.FAMILY(), qual1, val1)); + span.addRow(new KeyValue(hour2, TSDB.FAMILY(), qual2, val2)); + span.addRow(new KeyValue(hour3, TSDB.FAMILY(), qual1, val1)); + span.addRow(new KeyValue(hour3, TSDB.FAMILY(), qual2, val2)); + + assertEquals(6, span.size()); + assertEquals(1356998400000L, span.timestamp(0)); + assertEquals(1356998400000L, span.timestamp(1)); + assertEquals(1357002000000L, span.timestamp(2)); + assertEquals(1357002000000L, span.timestamp(3)); + assertEquals(1357005600000L, span.timestamp(4)); + assertEquals(1357005600000L, span.timestamp(5)); + } + + @Test + public void iterateNormalizedMS() throws Exception { + final byte[] qual1 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + + final Span span = new RollupSpan(tsdb, rollup_query); + span.addRow(new KeyValue(hour1, TSDB.FAMILY(), qual1,val1)); + span.addRow(new KeyValue(hour1, TSDB.FAMILY(), qual2,val2)); + span.addRow(new KeyValue(hour2, TSDB.FAMILY(), qual1,val1)); + span.addRow(new KeyValue(hour2, TSDB.FAMILY(), qual2,val2)); + span.addRow(new KeyValue(hour3, TSDB.FAMILY(), qual1,val1)); + span.addRow(new KeyValue(hour3, TSDB.FAMILY(), qual2,val2)); + + assertEquals(6, span.size()); + final SeekableView it = span.iterator(); + DataPoint dp = it.next(); + + assertEquals(1356998400000L, dp.timestamp()); + assertEquals(4, dp.longValue()); + + dp = it.next(); + assertEquals(1356998402000L, dp.timestamp()); + assertEquals(5, dp.longValue()); + + dp = it.next(); + assertEquals(1357002000000L, dp.timestamp()); + assertEquals(4, dp.longValue()); + + dp = it.next(); + assertEquals(1357002002000L, dp.timestamp()); + assertEquals(5, dp.longValue()); + + dp = it.next(); + assertEquals(1357005600000L, dp.timestamp()); + assertEquals(4, dp.longValue()); + + dp = it.next(); + assertEquals(1357005602000L, dp.timestamp()); + assertEquals(5, dp.longValue()); + + assertFalse(it.hasNext()); + } + + @Test + public void lastTimestampInRow() throws Exception { + final byte[] qual2 = { 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + + final KeyValue kv = new KeyValue(hour1, TSDB.FAMILY(), qual2, val2); + + assertEquals(1356998402L, Span.lastTimestampInRow((short) 3, kv)); + } + + @Test + public void lastTimestampInRowMs() throws Exception { + final byte[] qual2 = { (byte) 0xF0, 0x00, 0x02, 0x07 }; + final byte[] val2 = Bytes.fromLong(5L); + + final KeyValue kv = new KeyValue(hour1, TSDB.FAMILY(), qual2, val2); + + assertEquals(1356998400008L, Span.lastTimestampInRow((short) 3, kv)); + } +} diff --git a/test/core/TestRollupSpanSalted.java b/test/core/TestRollupSpanSalted.java new file mode 100644 index 0000000000..49e602bfba --- /dev/null +++ b/test/core/TestRollupSpanSalted.java @@ -0,0 +1,46 @@ +package net.opentsdb.core; + +import org.junit.Before; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.reflect.Whitebox; + +import net.opentsdb.rollup.RollupConfig; +import net.opentsdb.rollup.RollupInterval; + +public class TestRollupSpanSalted extends TestRollupSpan { + + @Before + public void beforeLocal() throws Exception { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(2); + PowerMockito.when(Const.MAX_NUM_TAGS()).thenReturn((short) 8); + + hour1 = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_STRING); + hour2 = getRowKey(METRIC_STRING, 1357002000, TAGK_STRING, TAGV_STRING); + hour3 = getRowKey(METRIC_STRING, 1357005600, TAGK_STRING, TAGV_STRING); + + rollup_config = RollupConfig.builder() + .addAggregationId("sum", 0) + .addAggregationId("count", 1) + .addAggregationId("max", 2) + .addAggregationId("min", 3) + .addInterval(RollupInterval.builder() + .setTable("tsdb-rollup-10m") + .setPreAggregationTable("tsdb-rollup-agg-10m") + .setInterval("10m") + .setRowSpan("6h")) + .addInterval(RollupInterval.builder() + .setTable("tsdb-rollup-1h") + .setPreAggregationTable("tsdb-rollup-agg-1h") + .setInterval("1h") + .setRowSpan("1d")) + .addInterval(RollupInterval.builder() + .setTable("tsdb-rollup-1d") + .setPreAggregationTable("tsdb-rollup-agg-1d") + .setInterval("1d") + .setRowSpan("1n")) + .build(); + Whitebox.setInternalState(tsdb, "rollup_config", rollup_config); + } +} diff --git a/test/core/TestRowKey.java b/test/core/TestRowKey.java new file mode 100644 index 0000000000..59032a9f0c --- /dev/null +++ b/test/core/TestRowKey.java @@ -0,0 +1,669 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import java.util.Arrays; + +import net.opentsdb.uid.NoSuchUniqueId; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +public class TestRowKey extends BaseTsdbTest { + + @Test + public void metricNameAsync() throws Exception { + final byte[] key = { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, 0, 0, 1, 0, 0, 1}; + assertEquals(METRIC_STRING, RowKey.metricNameAsync(tsdb, key) + .joinUninterruptibly()); + } + + @Test + public void metricNameAsyncSalted() throws Exception { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + + byte[] key = { 0, 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, 0, 0, 1, 0, 0, 1}; + assertEquals(METRIC_STRING, RowKey.metricNameAsync(tsdb, key) + .joinUninterruptibly()); + + key = new byte[] { 1, 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, 0, 0, 1, 0, 0, 1}; + assertEquals(METRIC_STRING, RowKey.metricNameAsync(tsdb, key) + .joinUninterruptibly()); + + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(2); + key = new byte[] { 0, 0, 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, 0, 0, 1, 0, 0, 1}; + assertEquals(METRIC_STRING, RowKey.metricNameAsync(tsdb, key) + .joinUninterruptibly()); + + key = new byte[] { 0, 1, 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, 0, 0, 1, 0, 0, 1}; + assertEquals(METRIC_STRING, RowKey.metricNameAsync(tsdb, key) + .joinUninterruptibly()); + } + + @Test (expected = IllegalArgumentException.class) + public void metricNameAsyncRowNull() throws Exception { + RowKey.metricNameAsync(tsdb, null).joinUninterruptibly(); + } + + @Test (expected = IllegalArgumentException.class) + public void metricNameAsyncRowEmpty() throws Exception { + RowKey.metricNameAsync(tsdb, new byte[] { }).joinUninterruptibly(); + } + + @Test (expected = IllegalArgumentException.class) + public void metricNameAsyncRowTooShort() throws Exception { + RowKey.metricNameAsync(tsdb, new byte[] { 0, 1 }).joinUninterruptibly(); + } + + @Test (expected = IllegalArgumentException.class) + public void metricNameAsyncMissingSalt() throws Exception { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + + RowKey.metricNameAsync(tsdb, new byte[] { 0, 0, 1 }).joinUninterruptibly(); + } + + @Test (expected = NoSuchUniqueId.class) + public void metricNameAsyncNoSuchUniqueId() throws Exception { + final byte[] key = { 0, 0, 3, 0x50, (byte) 0xE2, 0x27, 0, 0, 0, 1, 0, 0, 1}; + RowKey.metricNameAsync(tsdb, key).joinUninterruptibly(); + } + + @Test (expected = NullPointerException.class) + public void metricNameAsyncNullTsdb() throws Exception { + final byte[] key = { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, 0, 0, 1, 0, 0, 1}; + RowKey.metricNameAsync(null, key).joinUninterruptibly(); + } + + @Test + public void rowKeyFromTSUID() throws Exception { + final byte[] tsuid = { 0, 0, 1, 0, 0, 1, 0, 0, 2 }; + byte[] key = { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, 0, 0, 1, 0, 0, 2 }; + assertArrayEquals(key, RowKey.rowKeyFromTSUID(tsdb, tsuid, 1356998400)); + + // zero timestamp + key = new byte[] { 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2 }; + assertArrayEquals(key, RowKey.rowKeyFromTSUID(tsdb, tsuid, 0)); + + // negative timestamp; honey badger don't care + key = new byte[] { 0, 0, 1, -1, -21, 88, -128, 0, 0, 1, 0, 0, 2 }; + assertArrayEquals(key, RowKey.rowKeyFromTSUID(tsdb, tsuid, -1356998400)); + } + + @Test (expected = NullPointerException.class) + public void rowKeyFromTSUIDNullTsuid() throws Exception { + RowKey.rowKeyFromTSUID(tsdb, null, 1356998400); + } + + @Test (expected = NullPointerException.class) + public void rowKeyFromTSUIDNullTsdb() throws Exception { + final byte[] tsuid = { 0, 0, 1, 0, 0, 1, 0, 0, 2 }; + RowKey.rowKeyFromTSUID(null, tsuid, 1356998400); + } + + @Test (expected = IllegalArgumentException.class) + public void rowKeyFromTSUIDShortTsuid() throws Exception { + final byte[] tsuid = { 0, 1 }; + RowKey.rowKeyFromTSUID(tsdb, tsuid, 1356998400); + } + + @Test (expected = IllegalArgumentException.class) + public void rowKeyFromTSUIDEmptyTsuid() throws Exception { + final byte[] tsuid = { }; + RowKey.rowKeyFromTSUID(tsdb, tsuid, 1356998400); + } + + @Test + public void rowKeyFromTSUIDNoTags() throws Exception { + final byte[] tsuid = { 0, 0, 1 }; + byte[] key = { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0 }; + assertArrayEquals(key, RowKey.rowKeyFromTSUID(tsdb, tsuid, 1356998400)); + } + + @Test + public void rowKeyFromTSUIDMillis() throws Exception { + final byte[] tsuid = { 0, 0, 1, 0, 0, 1, 0, 0, 2 }; + byte[] key = { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, 0, 0, 1, 0, 0, 2 }; + assertArrayEquals(key, RowKey.rowKeyFromTSUID(tsdb, tsuid, 1356998400123L)); + } + + @Test + public void rowKeyFromTSUIDSalted() throws Exception { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(2); + + final byte[] tsuid = { 0, 0, 1, 0, 0, 1, 0, 0, 2 }; + byte[] key = { 1, 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, 0, 0, 1, 0, 0, 2 }; + assertArrayEquals(key, RowKey.rowKeyFromTSUID(tsdb, tsuid, 1356998400)); + + // zero timestamp + key = new byte[] { 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2 }; + assertArrayEquals(key, RowKey.rowKeyFromTSUID(tsdb, tsuid, 0)); + + // negative timestamp; honey badger don't care + key = new byte[] { 1, 0, 0, 1, -1, -21, 88, -128, 0, 0, 1, 0, 0, 2 }; + assertArrayEquals(key, RowKey.rowKeyFromTSUID(tsdb, tsuid, -1356998400)); + + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(4); + key = new byte[] { 0, 0, 0, 1, 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, + 0, 0, 1, 0, 0, 2 }; + assertArrayEquals(key, RowKey.rowKeyFromTSUID(tsdb, tsuid, 1356998400)); + } + + + @Test + public void getSalt() { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(0); + assertArrayEquals(new byte[] {}, RowKey.getSaltBytes(2)); + + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + assertArrayEquals(new byte[] { 2 }, RowKey.getSaltBytes(2)); + assertArrayEquals(new byte[] { 20 }, RowKey.getSaltBytes(20)); + + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(2); + assertArrayEquals(new byte[] { 0, 20 }, RowKey.getSaltBytes(20)); + + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(4); + assertArrayEquals(new byte[] { 0, 0, 0, 20 }, RowKey.getSaltBytes(20)); + + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + assertArrayEquals(new byte[] { -2 }, RowKey.getSaltBytes(-2)); + } + + @Test (expected = NegativeArraySizeException.class) + public void getSaltNegativeWidth() { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(-1); + RowKey.getSaltBytes(2); + } + + @Test + public void prefixKeyWithSaltGlobalAndNoOps() { + setupSalt(); + // short rows + byte[] key = new byte[1]; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(new byte[1], key); + + key = new byte[2]; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(new byte[2], key); + + key = new byte[3]; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(new byte[3], key); + + key = new byte[4]; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(new byte[4], key); + + key = new byte[5]; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(new byte[5], key); + + key = new byte[6]; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(new byte[6], key); + + key = new byte[7]; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(new byte[7], key); + + key = new byte[8]; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(new byte[8], key); + + key = new byte[] { 0x00, 0x00, 0x00, 0x00, 0x50, (byte) 0xE2, 0x27, 0x00}; + byte[] compare = Arrays.copyOf(key, key.length); + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(compare, key); + + key = new byte[] { 0x00, 0x00, 0x01, 0x50, (byte) 0xE2, 0x27, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01}; + compare = Arrays.copyOf(key, key.length); + + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(0); + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(compare, key); + } + + @Test + public void prefixKeyWithSaltSameMetricDifferentTags() { + setupSalt(); + byte[] key = new byte[] { 0x00, 0x00, 0x00, 0x01, + 0x50, (byte) 0xE2, 0x27, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01}; + byte[] compare = Arrays.copyOf(key, key.length); + compare[0] = 0x08; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(compare, key); + + key = new byte[] { 0x00, 0x00, 0x00, 0x01, + 0x50, (byte) 0xE2, 0x27, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x01}; + compare = Arrays.copyOf(key, key.length); + compare[0] = 0x10; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(compare, key); + + key = new byte[] { 0x00, 0x00, 0x00, 0x01, + 0x50, (byte) 0xE2, 0x27, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01}; + compare = Arrays.copyOf(key, key.length); + compare[0] = 0x00; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(compare, key); + + key = new byte[] { 0x00, 0x00, 0x00, 0x01, + 0x50, (byte) 0xE2, 0x27, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x02}; + compare = Arrays.copyOf(key, key.length); + compare[0] = 0x09; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(compare, key); + + key = new byte[] { 0x00, 0x00, 0x00, 0x01, + 0x50, (byte) 0xE2, 0x27, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x02, 0x00, 0x00, 0x03}; + compare = Arrays.copyOf(key, key.length); + compare[0] = 0x03; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(compare, key); + + // no tags. Shouldn't happen, but *shrug* + key = new byte[] { 0x00, 0x00, 0x00, 0x01, 0x50, (byte) 0xE2, 0x27, 0x00 }; + compare = Arrays.copyOf(key, key.length); + compare[0] = 0x0C; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(compare, key); + } + + @Test + public void prefixKeyWithSaltDifferentMetricSameTags() { + setupSalt(); + byte[] key = new byte[] { 0x00, 0x00, 0x00, 0x01, + 0x50, (byte) 0xE2, 0x27, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01}; + byte[] compare = Arrays.copyOf(key, key.length); + compare[0] = 0x08; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(compare, key); + + key = new byte[] { 0x00, 0x00, 0x00, 0x02, + 0x50, (byte) 0xE2, 0x27, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01}; + compare = Arrays.copyOf(key, key.length); + compare[0] = 0x09; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(compare, key); + + key = new byte[] { 0x00, 0x00, 0x00, 0x03, + 0x50, (byte) 0xE2, 0x27, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01}; + compare = Arrays.copyOf(key, key.length); + compare[0] = 0x06; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(compare, key); + + key = new byte[] { 0x00, 0x0B, 0x14, 0x20, + 0x50, (byte) 0xE2, 0x27, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01}; + compare = Arrays.copyOf(key, key.length); + compare[0] = 0x06; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(compare, key); + } + + @Test + public void prefixKeyWithSaltSameMetricSameTagsDifferentTimestamp() { + setupSalt(); + byte[] key = new byte[] { 0x00, 0x00, 0x00, 0x01, + 0x50, (byte) 0xE2, 0x27, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01}; + byte[] compare = Arrays.copyOf(key, key.length); + compare[0] = 0x08; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(compare, key); + + key = new byte[] { 0x00, 0x00, 0x00, 0x01, + 0x50, (byte) 0xE2, 0x35, 0x10, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01}; + compare = Arrays.copyOf(key, key.length); + compare[0] = 0x08; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(compare, key); + + key = new byte[] { 0x00, 0x00, 0x00, 0x01, + 0x51, (byte) 0x0B, 0x13, (byte) 0x90, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01}; + compare = Arrays.copyOf(key, key.length); + compare[0] = 0x08; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(compare, key); + + key = new byte[] { 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01}; + compare = Arrays.copyOf(key, key.length); + compare[0] = 0x08; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(compare, key); + } + + @Test + public void prefixKeyWithSaltOverwrite() { + setupSalt(); + // makes sure we ignore and overwrite anything in the salt position + byte[] key = new byte[] { 0x02, 0x00, 0x00, 0x01, + 0x50, (byte) 0xE2, 0x27, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01}; + byte[] compare = Arrays.copyOf(key, key.length); + compare[0] = 0x08; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(compare, key); + + key = new byte[] { (byte) 0xFF, 0x00, 0x00, 0x01, + 0x50, (byte) 0xE2, 0x27, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01}; + compare = Arrays.copyOf(key, key.length); + compare[0] = 0x08; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(compare, key); + + key = new byte[] { (byte) 0x0E, 0x00, 0x00, 0x01, + 0x50, (byte) 0xE2, 0x27, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01}; + compare = Arrays.copyOf(key, key.length); + compare[0] = 0x08; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(compare, key); + } + + @Test (expected = ArithmeticException.class) + public void prefixKeyWithSaltZeroBucket() { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(0); + + final byte[] key = new byte[] { 0x02, 0x00, 0x00, 0x01, + 0x50, (byte) 0xE2, 0x27, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01}; + RowKey.prefixKeyWithSalt(key); + } + + // This actually works, but PLEASE don't do it! + @Test + public void prefixKeyWithSaltNegativeBucket() { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(-20); + + final byte[] key = new byte[] { 0x02, 0x00, 0x00, 0x01, + 0x50, (byte) 0xE2, 0x27, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01}; + final byte[] compare = Arrays.copyOf(key, key.length); + compare[0] = 0x08; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(compare, key); + } + + @Test + public void prefixKeyWithSaltNegativeWidth() { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(-1); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(20); + + final byte[] key = new byte[] { 0x00, 0x00, 0x01, + 0x50, (byte) 0xE2, 0x27, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01}; + final byte[] compare = Arrays.copyOf(key, key.length); + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(compare, key); + } + + @Test + public void prefixKeyWithSaltMissingTagV() { + setupSalt(); + // Honey badger don't care + final byte[] key = new byte[] { 0x02, 0x00, 0x00, 0x01, + 0x50, (byte) 0xE2, 0x27, 0x00, 0x00, 0x00, 0x01}; + final byte[] compare = Arrays.copyOf(key, key.length); + compare[0] = 0x0D; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(compare, key); + } + + @Test + public void prefixKeyWithSaltPartialTagK() { + setupSalt(); + // Honey badger don't care + final byte[] key = new byte[] { 0x02, 0x00, 0x00, 0x01, + 0x50, (byte) 0xE2, 0x27, 0x00 }; + final byte[] compare = Arrays.copyOf(key, key.length); + compare[0] = 0x0C; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(compare, key); + } + + @Test (expected = ArrayIndexOutOfBoundsException.class) + public void prefixKeyWithSaltMissingTags() { + setupSalt(); + final byte[] key = new byte[] { 0x02, 0x00, 0x00, 0x01, + 0x50, (byte) 0xE2, 0x27 }; + RowKey.prefixKeyWithSalt(key); + } + + @Test (expected = NullPointerException.class) + public void prefixKeyWithSaltNullKey() { + setupSalt(); + RowKey.prefixKeyWithSalt(null); + } + + @Test + public void prefixKeyWithSaltEmptyKey() { + setupSalt(); + final byte[] key = new byte[] {}; + RowKey.prefixKeyWithSalt(key); + assertArrayEquals(new byte[] {}, key); + } + + @Test + public void rowKeyContainsMetric() { + setupSalt(); + final byte[] metric = new byte[] { 0, 0, 1 }; + + assertEquals(0, RowKey.rowKeyContainsMetric(metric, + new byte[] { 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 })); + + // not there! + assertEquals(-1, RowKey.rowKeyContainsMetric(metric, + new byte[] { 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 })); + + // double salt bytes + assertEquals(-1, RowKey.rowKeyContainsMetric(metric, + new byte[] { 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 })); + + // missing salt but expecting it + assertEquals(-1, RowKey.rowKeyContainsMetric(metric, + new byte[] {0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 })); + + // both empty + assertEquals(0, RowKey.rowKeyContainsMetric(new byte[] { }, + new byte[] { })); + + // empty metric returns 0 TODO(clarsen) see if we really want that + assertEquals(0, RowKey.rowKeyContainsMetric(new byte[] { }, + new byte[] {0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 })); + } + + @Test (expected = NullPointerException.class) + public void rowKeyContainsMetricNullMetric() { + setupSalt(); + RowKey.rowKeyContainsMetric(null, + new byte[] { 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 }); + } + + @Test (expected = NullPointerException.class) + public void rowKeyContainsMetricNullKey() { + setupSalt(); + RowKey.rowKeyContainsMetric(new byte[] { 0, 0, 1 }, null); + } + + @Test (expected = IndexOutOfBoundsException.class) + public void rowKeyContainsMetricKeyTooShortSalted() { + setupSalt(); + RowKey.rowKeyContainsMetric(new byte[] { 0, 0, 1 }, + new byte[] { 1, 0, 0 }); + } + + @Test (expected = IndexOutOfBoundsException.class) + public void rowKeyContainsMetricKeyTooShortNoSalt() { + RowKey.rowKeyContainsMetric(new byte[] { 0, 0, 1 }, + new byte[] { 0, 0 }); + } + + @Test + public void rowKeyContainsMetricNoSalt() { + final byte[] metric = new byte[] { 0, 0, 1 }; + + assertEquals(0, RowKey.rowKeyContainsMetric(metric, + new byte[] { 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 })); + + // not there! + assertEquals(1, RowKey.rowKeyContainsMetric(metric, + new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 })); + + // salted but shouldn't be + assertEquals(-1, RowKey.rowKeyContainsMetric(metric, + new byte[] { 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 })); + } + + @Test + public void saltCmp() { + setupSalt(); + + // diff salt, same keys + assertEquals(0, new RowKey.SaltCmp().compare( + new byte[] { 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 }, + new byte[] { 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 })); + + // same addy + final byte[] key = new byte[] { 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 }; + assertEquals(0, new RowKey.SaltCmp().compare(key, key)); + + assertEquals(1, new RowKey.SaltCmp().compare( + new byte[] { 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4 }, + new byte[] { 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 })); + + assertEquals(-1, new RowKey.SaltCmp().compare( + new byte[] { 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 }, + new byte[] { 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4 })); + + assertEquals(-1, new RowKey.SaltCmp().compare( + new byte[] { 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0 }, + new byte[] { 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 })); + + assertEquals(1, new RowKey.SaltCmp().compare( + new byte[] { 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 }, + new byte[] { 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0 })); + + // nothing after the salt + assertEquals(0, new RowKey.SaltCmp().compare( + new byte[] { 1 }, + new byte[] { 2 })); + + // empty + assertEquals(0, new RowKey.SaltCmp().compare( + new byte[] { }, + new byte[] { })); + + // wider salt + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(3); + + assertEquals(0, new RowKey.SaltCmp().compare( + new byte[] { 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 }, + new byte[] { 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 })); + + assertEquals(0, new RowKey.SaltCmp().compare( + new byte[] { 1, 3, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 }, + new byte[] { 2, 4, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 })); + + assertEquals(0, new RowKey.SaltCmp().compare( + new byte[] { 1, 3, 5, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 }, + new byte[] { 2, 4, 6, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 })); + + assertEquals(-1, new RowKey.SaltCmp().compare( + new byte[] { 1, 3, 5, 7, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 }, + new byte[] { 2, 4, 6, 8, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 })); + } + + @Test (expected = NullPointerException.class) + public void saltCmpNullA() throws Exception { + setupSalt(); + new RowKey.SaltCmp().compare(null, + new byte[] { 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 }); + } + + @Test (expected = NullPointerException.class) + public void saltCmpNullB() throws Exception { + setupSalt(); + new RowKey.SaltCmp().compare( + new byte[] { 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 }, + null); + } + + @Test + public void saltCmpNoSalt() { + assertEquals(0, new RowKey.SaltCmp().compare( + new byte[] { 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 }, + new byte[] { 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 })); + + // diff salt, same keys + assertEquals(-1, new RowKey.SaltCmp().compare( + new byte[] { 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 }, + new byte[] { 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 })); + + // same addy + final byte[] key = new byte[] { 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 }; + assertEquals(0, new RowKey.SaltCmp().compare(key, key)); + + assertEquals(-1, new RowKey.SaltCmp().compare( + new byte[] { 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4 }, + new byte[] { 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 })); + + assertEquals(-1, new RowKey.SaltCmp().compare( + new byte[] { 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 }, + new byte[] { 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4 })); + + assertEquals(-1, new RowKey.SaltCmp().compare( + new byte[] { 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0 }, + new byte[] { 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 })); + + assertEquals(-1, new RowKey.SaltCmp().compare( + new byte[] { 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3 }, + new byte[] { 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0 })); + + // nothing after the salt + assertEquals(-1, new RowKey.SaltCmp().compare( + new byte[] { 1 }, + new byte[] { 2 })); + + // empty + assertEquals(0, new RowKey.SaltCmp().compare( + new byte[] { }, + new byte[] { })); + } + + /** + * Mocks out the static Const class for a single salt byte with 20 buckets + */ + private static void setupSalt() { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(20); + } + +} diff --git a/test/core/TestRowSeq.java b/test/core/TestRowSeq.java index 9c00b3baf3..fe8d1a932e 100644 --- a/test/core/TestRowSeq.java +++ b/test/core/TestRowSeq.java @@ -12,12 +12,14 @@ // see <http://www.gnu.org/licenses/>. package net.opentsdb.core; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.when; import static org.powermock.api.mockito.PowerMockito.mock; +import java.util.Arrays; import java.util.NoSuchElementException; import net.opentsdb.storage.MockBase; @@ -26,9 +28,11 @@ import org.hbase.async.Bytes; import org.hbase.async.KeyValue; +import org.hbase.async.Bytes.ByteMap; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; @@ -43,16 +47,18 @@ "ch.qos.*", "org.slf4j.*", "com.sum.*", "org.xml.*"}) @PrepareForTest({ RowSeq.class, TSDB.class, UniqueId.class, KeyValue.class, - Config.class, RowKey.class }) + Config.class, RowKey.class, Const.class }) public final class TestRowSeq { private TSDB tsdb = mock(TSDB.class); private Config config = mock(Config.class); private UniqueId metrics = mock(UniqueId.class); - private static final byte[] TABLE = { 't', 'a', 'b', 'l', 'e' }; - private static final byte[] KEY = + public static final byte[] TABLE = { 't', 'a', 'b', 'l', 'e' }; + public static final byte[] KEY = { 0, 0, 1, 0x50, (byte)0xE2, 0x27, 0, 0, 0, 1, 0, 0, 2 }; - private static final byte[] FAMILY = { 't' }; - private static final byte[] ZERO = { 0 }; + public static final byte[] SALTED_KEY = + { 0, 0, 0, 1, 0x50, (byte)0xE2, 0x27, 0, 0, 0, 1, 0, 0, 2 }; + public static final byte[] FAMILY = { 't' }; + public static final byte[] ZERO = { 0 }; @Before public void before() throws Exception { @@ -73,7 +79,23 @@ public void setRow() throws Exception { final byte[] qual2 = { 0x00, 0x27 }; final byte[] val2 = Bytes.fromLong(5L); final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); - final KeyValue kv = makekv(qual12, + final KeyValue kv = makekv(KEY, qual12, + MockBase.concatByteArrays(val1, val2, ZERO)); + + final RowSeq rs = new RowSeq(tsdb); + rs.setRow(kv); + assertEquals(2, rs.size()); + } + + @Test + public void setRowSalted() throws Exception { + setupSalt(); + final byte[] qual1 = { 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); + final KeyValue kv = makekv(SALTED_KEY, qual12, MockBase.concatByteArrays(val1, val2, ZERO)); final RowSeq rs = new RowSeq(tsdb); @@ -88,7 +110,7 @@ public void setRowAlreadySet() throws Exception { final byte[] qual2 = { 0x00, 0x27 }; final byte[] val2 = Bytes.fromLong(5L); final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); - final KeyValue kv = makekv(qual12, + final KeyValue kv = makekv(KEY, qual12, MockBase.concatByteArrays(val1, val2, ZERO)); final RowSeq rs = new RowSeq(tsdb); @@ -106,7 +128,39 @@ public void addRowMergeLater() throws Exception { final byte[] val2 = Bytes.fromLong(5L); final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); final RowSeq rs = new RowSeq(tsdb); - rs.setRow(makekv(qual12, MockBase.concatByteArrays(val1, val2, ZERO))); + rs.setRow(makekv(KEY, qual12, MockBase.concatByteArrays(val1, val2, ZERO))); + assertEquals(2, rs.size()); + + final byte[] qual3 = { 0x00, 0x37 }; + final byte[] val3 = Bytes.fromLong(6L); + final byte[] qual4 = { 0x00, 0x47 }; + final byte[] val4 = Bytes.fromLong(7L); + final byte[] qual34 = MockBase.concatByteArrays(qual3, qual4); + rs.addRow(makekv(KEY, qual34, MockBase.concatByteArrays(val3, val4, ZERO))); + + assertEquals(4, rs.size()); + assertEquals(1356998400000L, rs.timestamp(0)); + assertEquals(4, rs.longValue(0)); + assertEquals(1356998402000L, rs.timestamp(1)); + assertEquals(5, rs.longValue(1)); + assertEquals(1356998403000L, rs.timestamp(2)); + assertEquals(6, rs.longValue(2)); + assertEquals(1356998404000L, rs.timestamp(3)); + assertEquals(7, rs.longValue(3)); + } + + @Test + public void addRowMergeLaterSalted() throws Exception { + setupSalt(); + // this happens if the same row key is used for the addRow call + final byte[] qual1 = { 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); + final RowSeq rs = new RowSeq(tsdb); + rs.setRow(makekv(SALTED_KEY, qual12, + MockBase.concatByteArrays(val1, val2, ZERO))); assertEquals(2, rs.size()); final byte[] qual3 = { 0x00, 0x37 }; @@ -114,7 +168,10 @@ public void addRowMergeLater() throws Exception { final byte[] qual4 = { 0x00, 0x47 }; final byte[] val4 = Bytes.fromLong(7L); final byte[] qual34 = MockBase.concatByteArrays(qual3, qual4); - rs.addRow(makekv(qual34, MockBase.concatByteArrays(val3, val4, ZERO))); + final byte[] salted_key2 = Arrays.copyOf(SALTED_KEY, SALTED_KEY.length); + salted_key2[0] = 1; + rs.addRow(makekv(salted_key2, qual34, + MockBase.concatByteArrays(val3, val4, ZERO))); assertEquals(4, rs.size()); assertEquals(1356998400000L, rs.timestamp(0)); @@ -136,7 +193,7 @@ public void addRowMergeEarlier() throws Exception { final byte[] val2 = Bytes.fromLong(7L); final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); final RowSeq rs = new RowSeq(tsdb); - rs.setRow(makekv(qual12, MockBase.concatByteArrays(val1, val2, ZERO))); + rs.setRow(makekv(KEY, qual12, MockBase.concatByteArrays(val1, val2, ZERO))); assertEquals(2, rs.size()); final byte[] qual3 = { 0x00, 0x07 }; @@ -144,7 +201,42 @@ public void addRowMergeEarlier() throws Exception { final byte[] qual4 = { 0x00, 0x27 }; final byte[] val4 = Bytes.fromLong(5L); final byte[] qual34 = MockBase.concatByteArrays(qual3, qual4); - rs.addRow(makekv(qual34, MockBase.concatByteArrays(val3, val4, ZERO))); + rs.addRow(makekv(KEY, qual34, MockBase.concatByteArrays(val3, val4, ZERO))); + + assertEquals(4, rs.size()); + assertEquals(1356998400000L, rs.timestamp(0)); + assertEquals(4, rs.longValue(0)); + assertEquals(1356998402000L, rs.timestamp(1)); + assertEquals(5, rs.longValue(1)); + assertEquals(1356998403000L, rs.timestamp(2)); + assertEquals(6, rs.longValue(2)); + assertEquals(1356998404000L, rs.timestamp(3)); + assertEquals(7, rs.longValue(3)); + } + + @Test + public void addRowMergeEarlierSalted() throws Exception { + setupSalt(); + // this happens if the same row key is used for the addRow call + final byte[] qual1 = { 0x00, 0x37 }; + final byte[] val1 = Bytes.fromLong(6L); + final byte[] qual2 = { 0x00, 0x47 }; + final byte[] val2 = Bytes.fromLong(7L); + final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); + final RowSeq rs = new RowSeq(tsdb); + rs.setRow(makekv(SALTED_KEY, qual12, + MockBase.concatByteArrays(val1, val2, ZERO))); + assertEquals(2, rs.size()); + + final byte[] qual3 = { 0x00, 0x07 }; + final byte[] val3 = Bytes.fromLong(4L); + final byte[] qual4 = { 0x00, 0x27 }; + final byte[] val4 = Bytes.fromLong(5L); + final byte[] qual34 = MockBase.concatByteArrays(qual3, qual4); + final byte[] salted_key2 = Arrays.copyOf(SALTED_KEY, SALTED_KEY.length); + salted_key2[0] = 1; + rs.addRow(makekv(salted_key2, qual34, + MockBase.concatByteArrays(val3, val4, ZERO))); assertEquals(4, rs.size()); assertEquals(1356998400000L, rs.timestamp(0)); @@ -166,7 +258,7 @@ public void addRowMergeMiddle() throws Exception { final byte[] val2 = Bytes.fromLong(5L); final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); final RowSeq rs = new RowSeq(tsdb); - rs.setRow(makekv(qual12, MockBase.concatByteArrays(val1, val2, ZERO))); + rs.setRow(makekv(KEY, qual12, MockBase.concatByteArrays(val1, val2, ZERO))); assertEquals(2, rs.size()); final byte[] qual3 = { 0x00, 0x57 }; @@ -174,7 +266,7 @@ public void addRowMergeMiddle() throws Exception { final byte[] qual4 = { 0x00, 0x67 }; final byte[] val4 = Bytes.fromLong(9L); final byte[] qual34 = MockBase.concatByteArrays(qual3, qual4); - rs.addRow(makekv(qual34, MockBase.concatByteArrays(val3, val4, ZERO))); + rs.addRow(makekv(KEY, qual34, MockBase.concatByteArrays(val3, val4, ZERO))); assertEquals(4, rs.size()); final byte[] qual5 = { 0x00, 0x37 }; @@ -182,7 +274,55 @@ public void addRowMergeMiddle() throws Exception { final byte[] qual6 = { 0x00, 0x47 }; final byte[] val6 = Bytes.fromLong(7L); final byte[] qual56 = MockBase.concatByteArrays(qual5, qual6); - rs.addRow(makekv(qual56, MockBase.concatByteArrays(val5, val6, ZERO))); + rs.addRow(makekv(KEY, qual56, MockBase.concatByteArrays(val5, val6, ZERO))); + + assertEquals(6, rs.size()); + assertEquals(1356998400000L, rs.timestamp(0)); + assertEquals(4, rs.longValue(0)); + assertEquals(1356998402000L, rs.timestamp(1)); + assertEquals(5, rs.longValue(1)); + assertEquals(1356998403000L, rs.timestamp(2)); + assertEquals(6, rs.longValue(2)); + assertEquals(1356998404000L, rs.timestamp(3)); + assertEquals(7, rs.longValue(3)); + assertEquals(1356998405000L, rs.timestamp(4)); + assertEquals(8, rs.longValue(4)); + assertEquals(1356998406000L, rs.timestamp(5)); + assertEquals(9, rs.longValue(5)); + } + + @Test + public void addRowMergeMiddleSalted() throws Exception { + setupSalt(); + // this happens if the same row key is used for the addRow call + final byte[] qual1 = { 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); + final RowSeq rs = new RowSeq(tsdb); + rs.setRow(makekv(SALTED_KEY, qual12, + MockBase.concatByteArrays(val1, val2, ZERO))); + assertEquals(2, rs.size()); + + final byte[] qual3 = { 0x00, 0x57 }; + final byte[] val3 = Bytes.fromLong(8L); + final byte[] qual4 = { 0x00, 0x67 }; + final byte[] val4 = Bytes.fromLong(9L); + final byte[] qual34 = MockBase.concatByteArrays(qual3, qual4); + rs.addRow(makekv(SALTED_KEY, qual34, + MockBase.concatByteArrays(val3, val4, ZERO))); + assertEquals(4, rs.size()); + + final byte[] qual5 = { 0x00, 0x37 }; + final byte[] val5 = Bytes.fromLong(6L); + final byte[] qual6 = { 0x00, 0x47 }; + final byte[] val6 = Bytes.fromLong(7L); + final byte[] qual56 = MockBase.concatByteArrays(qual5, qual6); + final byte[] salted_key2 = Arrays.copyOf(SALTED_KEY, SALTED_KEY.length); + salted_key2[0] = 1; + rs.addRow(makekv(salted_key2, qual56, + MockBase.concatByteArrays(val5, val6, ZERO))); assertEquals(6, rs.size()); assertEquals(1356998400000L, rs.timestamp(0)); @@ -210,13 +350,13 @@ public void addRowMergeDuplicateLater() throws Exception { final byte[] val3 = Bytes.fromLong(6L); final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2, qual3); final RowSeq rs = new RowSeq(tsdb); - rs.setRow(makekv(qual12, MockBase.concatByteArrays(val1, val2, val3, ZERO))); + rs.setRow(makekv(KEY, qual12, MockBase.concatByteArrays(val1, val2, val3, ZERO))); assertEquals(3, rs.size()); final byte[] qual4 = { 0x00, 0x47 }; final byte[] val4 = Bytes.fromLong(7L); final byte[] qual34 = MockBase.concatByteArrays(qual3, qual4); - rs.addRow(makekv(qual34, MockBase.concatByteArrays(val3, val4, ZERO))); + rs.addRow(makekv(KEY, qual34, MockBase.concatByteArrays(val3, val4, ZERO))); assertEquals(4, rs.size()); assertEquals(1356998400000L, rs.timestamp(0)); @@ -240,13 +380,13 @@ public void addRowMergeDuplicateEarlier() throws Exception { final byte[] val2 = Bytes.fromLong(7L); final byte[] qual12 = MockBase.concatByteArrays(qual4, qual1, qual2); final RowSeq rs = new RowSeq(tsdb); - rs.setRow(makekv(qual12, MockBase.concatByteArrays(val4, val1, val2, ZERO))); + rs.setRow(makekv(KEY, qual12, MockBase.concatByteArrays(val4, val1, val2, ZERO))); assertEquals(3, rs.size()); final byte[] qual3 = { 0x00, 0x07 }; final byte[] val3 = Bytes.fromLong(4L); final byte[] qual34 = MockBase.concatByteArrays(qual3, qual4); - rs.addRow(makekv(qual34, MockBase.concatByteArrays(val3, val4, ZERO))); + rs.addRow(makekv(KEY, qual34, MockBase.concatByteArrays(val3, val4, ZERO))); assertEquals(4, rs.size()); assertEquals(1356998400000L, rs.timestamp(0)); @@ -267,7 +407,7 @@ public void addRowDiffBaseTime() throws Exception { final byte[] val2 = Bytes.fromLong(5L); final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); final RowSeq rs = new RowSeq(tsdb); - rs.setRow(makekv(qual12, MockBase.concatByteArrays(val1, val2, ZERO))); + rs.setRow(makekv(KEY, qual12, MockBase.concatByteArrays(val1, val2, ZERO))); assertEquals(2, rs.size()); final byte[] qual3 = { 0x00, 0x37 }; @@ -280,6 +420,30 @@ public void addRowDiffBaseTime() throws Exception { MockBase.concatByteArrays(val3, val4, ZERO))); } + @Test (expected = IllegalDataException.class) + public void addRowDiffBaseTimeSalt() throws Exception { + setupSalt(); + final byte[] qual1 = { 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); + final RowSeq rs = new RowSeq(tsdb); + rs.setRow(makekv(SALTED_KEY, qual12, + MockBase.concatByteArrays(val1, val2, ZERO))); + assertEquals(2, rs.size()); + + final byte[] qual3 = { 0x00, 0x37 }; + final byte[] val3 = Bytes.fromLong(6L); + final byte[] qual4 = { 0x00, 0x47 }; + final byte[] val4 = Bytes.fromLong(7L); + final byte[] qual34 = MockBase.concatByteArrays(qual3, qual4); + final byte[] row2 = { 1, 0, 0, 1, 0x50, (byte)0xE2, 0x35, 0x10, + 0, 0, 1, 0, 0, 2 }; + rs.addRow(new KeyValue(row2, FAMILY, qual34, + MockBase.concatByteArrays(val3, val4, ZERO))); + } + @Test public void addRowMergeMs() throws Exception { // this happens if the same row key is used for the addRow call @@ -289,7 +453,7 @@ public void addRowMergeMs() throws Exception { final byte[] val2 = Bytes.fromLong(5L); final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); final RowSeq rs = new RowSeq(tsdb); - rs.setRow(makekv(qual12, MockBase.concatByteArrays(val1, val2, ZERO))); + rs.setRow(makekv(KEY, qual12, MockBase.concatByteArrays(val1, val2, ZERO))); assertEquals(2, rs.size()); final byte[] qual3 = { (byte) 0xF0, 0x00, 0x07, 0x07 }; @@ -297,7 +461,7 @@ public void addRowMergeMs() throws Exception { final byte[] qual4 = { (byte) 0xF0, 0x00, 0x09, 0x07 }; final byte[] val4 = Bytes.fromLong(7L); final byte[] qual34 = MockBase.concatByteArrays(qual3, qual4); - rs.addRow(makekv(qual34, MockBase.concatByteArrays(val3, val4, ZERO))); + rs.addRow(makekv(KEY, qual34, MockBase.concatByteArrays(val3, val4, ZERO))); assertEquals(4, rs.size()); assertEquals(1356998400000L, rs.timestamp(0)); @@ -319,7 +483,7 @@ public void addRowMergeSecAndMs() throws Exception { final byte[] val2 = Bytes.fromLong(5L); final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); final RowSeq rs = new RowSeq(tsdb); - rs.setRow(makekv(qual12, MockBase.concatByteArrays(val1, val2, + rs.setRow(makekv(KEY, qual12, MockBase.concatByteArrays(val1, val2, new byte[] { 1 }))); assertEquals(2, rs.size()); @@ -328,7 +492,7 @@ public void addRowMergeSecAndMs() throws Exception { final byte[] qual4 = { (byte) 0xF0, 0x01, 0x09, 0x07 }; final byte[] val4 = Bytes.fromLong(7L); final byte[] qual34 = MockBase.concatByteArrays(qual3, qual4); - rs.addRow(makekv(qual34, MockBase.concatByteArrays(val3, val4, + rs.addRow(makekv(KEY, qual34, MockBase.concatByteArrays(val3, val4, new byte[] { 1 }))); assertEquals(4, rs.size()); @@ -349,7 +513,7 @@ public void addRowNotSet() throws Exception { final byte[] qual2 = { 0x00, 0x27 }; final byte[] val2 = Bytes.fromLong(5L); final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); - final KeyValue kv = makekv(qual12, + final KeyValue kv = makekv(KEY, qual12, MockBase.concatByteArrays(val1, val2, ZERO)); final RowSeq rs = new RowSeq(tsdb); @@ -363,7 +527,25 @@ public void timestamp() throws Exception { final byte[] qual2 = { 0x00, 0x27 }; final byte[] val2 = Bytes.fromLong(5L); final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); - final KeyValue kv = makekv(qual12, + final KeyValue kv = makekv(KEY, qual12, + MockBase.concatByteArrays(val1, val2, ZERO)); + + final RowSeq rs = new RowSeq(tsdb); + rs.setRow(kv); + + assertEquals(1356998400000L, rs.timestamp(0)); + assertEquals(1356998402000L, rs.timestamp(1)); + } + + @Test + public void timestampSalted() throws Exception { + setupSalt(); + final byte[] qual1 = { 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); + final KeyValue kv = makekv(SALTED_KEY, qual12, MockBase.concatByteArrays(val1, val2, ZERO)); final RowSeq rs = new RowSeq(tsdb); @@ -380,7 +562,7 @@ public void timestampNormalizeMS() throws Exception { final byte[] qual2 = { 0x00, 0x27 }; final byte[] val2 = Bytes.fromLong(5L); final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); - final KeyValue kv = makekv(qual12, + final KeyValue kv = makekv(KEY, qual12, MockBase.concatByteArrays(val1, val2, ZERO)); final RowSeq rs = new RowSeq(tsdb); @@ -397,7 +579,7 @@ public void timestampMs() throws Exception { final byte[] qual2 = { (byte) 0xF0, 0x00, 0x02, 0x07 }; final byte[] val2 = Bytes.fromLong(5L); final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); - final KeyValue kv = makekv(qual12, + final KeyValue kv = makekv(KEY, qual12, MockBase.concatByteArrays(val1, val2, ZERO)); final RowSeq rs = new RowSeq(tsdb); @@ -414,7 +596,7 @@ public void timestampMixedNormalized() throws Exception { final byte[] qual2 = { (byte) 0xF0, 0x00, 0x02, 0x07 }; final byte[] val2 = Bytes.fromLong(5L); final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); - final KeyValue kv = makekv(qual12, + final KeyValue kv = makekv(KEY, qual12, MockBase.concatByteArrays(val1, val2, ZERO)); final RowSeq rs = new RowSeq(tsdb); @@ -431,7 +613,7 @@ public void timestampMixedNonNormalized() throws Exception { final byte[] qual2 = { (byte) 0xF0, 0x00, 0x02, 0x07 }; final byte[] val2 = Bytes.fromLong(5L); final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); - final KeyValue kv = makekv(qual12, + final KeyValue kv = makekv(KEY, qual12, MockBase.concatByteArrays(val1, val2, ZERO)); final RowSeq rs = new RowSeq(tsdb); @@ -448,7 +630,7 @@ public void timestampOutofBounds() throws Exception { final byte[] qual2 = { (byte) 0xF0, 0x00, 0x02, 0x07 }; final byte[] val2 = Bytes.fromLong(5L); final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); - final KeyValue kv = makekv(qual12, + final KeyValue kv = makekv(KEY, qual12, MockBase.concatByteArrays(val1, val2, ZERO)); final RowSeq rs = new RowSeq(tsdb); @@ -466,7 +648,7 @@ public void iterateNormalizedMS() throws Exception { final byte[] qual2 = { 0x00, 0x27 }; final byte[] val2 = Bytes.fromLong(5L); final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); - final KeyValue kv = makekv(qual12, + final KeyValue kv = makekv(KEY, qual12, MockBase.concatByteArrays(val1, val2, ZERO)); final RowSeq rs = new RowSeq(tsdb); @@ -494,7 +676,7 @@ public void iterateMs() throws Exception { final byte[] qual2 = { (byte) 0xF0, 0x00, 0x02, 0x07 }; final byte[] val2 = Bytes.fromLong(5L); final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); - final KeyValue kv = makekv(qual12, + final KeyValue kv = makekv(KEY, qual12, MockBase.concatByteArrays(val1, val2, ZERO)); final RowSeq rs = new RowSeq(tsdb); @@ -525,7 +707,7 @@ public void iterateMsLarge() throws Exception { ts += 50; } final byte[] values = new byte[(4 * limit) + 1]; - final KeyValue kv = makekv(qualifier, values); + final KeyValue kv = makekv(KEY, qualifier, values); final RowSeq rs = new RowSeq(tsdb); rs.setRow(kv); @@ -542,7 +724,22 @@ public void iterateMsLarge() throws Exception { @Test public void seekMs() throws Exception { final RowSeq rs = new RowSeq(tsdb); - rs.setRow(getMs()); + rs.setRow(getMs(false)); + + final SeekableView it = rs.iterator(); + it.seek(1356998400008L); + DataPoint dp = it.next(); + assertEquals(1356998400008L, dp.timestamp()); + assertEquals(5, dp.longValue()); + + assertTrue(it.hasNext()); + } + + @Test + public void seekMsSalted() throws Exception { + setupSalt(); + final RowSeq rs = new RowSeq(tsdb); + rs.setRow(getMs(true)); final SeekableView it = rs.iterator(); it.seek(1356998400008L); @@ -556,7 +753,7 @@ public void seekMs() throws Exception { @Test public void seekMsStart() throws Exception { final RowSeq rs = new RowSeq(tsdb); - rs.setRow(getMs()); + rs.setRow(getMs(false)); final SeekableView it = rs.iterator(); it.seek(1356998400000L); @@ -570,7 +767,7 @@ public void seekMsStart() throws Exception { @Test public void seekMsBetween() throws Exception { final RowSeq rs = new RowSeq(tsdb); - rs.setRow(getMs()); + rs.setRow(getMs(false)); final SeekableView it = rs.iterator(); it.seek(1356998400005L); @@ -584,7 +781,7 @@ public void seekMsBetween() throws Exception { @Test public void seekMsEnd() throws Exception { final RowSeq rs = new RowSeq(tsdb); - rs.setRow(getMs()); + rs.setRow(getMs(false)); final SeekableView it = rs.iterator(); it.seek(1356998400016L); @@ -598,7 +795,7 @@ public void seekMsEnd() throws Exception { @Test public void seekMsTooEarly() throws Exception { final RowSeq rs = new RowSeq(tsdb); - rs.setRow(getMs()); + rs.setRow(getMs(false)); final SeekableView it = rs.iterator(); it.seek(1356998300000L); @@ -612,19 +809,131 @@ public void seekMsTooEarly() throws Exception { @Test (expected = NoSuchElementException.class) public void seekMsPastLastDp() throws Exception { final RowSeq rs = new RowSeq(tsdb); - rs.setRow(getMs()); + rs.setRow(getMs(false)); final SeekableView it = rs.iterator(); it.seek(1356998400032L); it.next(); } + @Test + public void metricUID() throws Exception { + final byte[] qual1 = { 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); + final KeyValue kv = makekv(qual12, + MockBase.concatByteArrays(val1, val2, ZERO)); + + final RowSeq rs = new RowSeq(tsdb); + rs.setRow(kv); + + assertArrayEquals(new byte[] { 0, 0, 1 }, rs.metricUID()); + } + + @Test + public void metricUIDSalted() throws Exception { + setupSalt(); + final byte[] qual1 = { 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); + final KeyValue kv = makekv(qual12, + MockBase.concatByteArrays(val1, val2, ZERO)); + + final RowSeq rs = new RowSeq(tsdb); + rs.setRow(kv); + + assertArrayEquals(new byte[] { 0, 0, 1 }, rs.metricUID()); + } + + @Test (expected = NullPointerException.class) + public void metricUIDKeyNotSet() throws Exception { + final RowSeq rs = new RowSeq(tsdb); + rs.metricUID(); + } + + @Test + public void getTagUids() throws Exception { + final byte[] qual1 = { 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); + final KeyValue kv = makekv(qual12, + MockBase.concatByteArrays(val1, val2, ZERO)); + + final RowSeq rs = new RowSeq(tsdb); + rs.setRow(kv); + + final ByteMap<byte[]> uids = rs.getTagUids(); + assertEquals(1, uids.size()); + assertArrayEquals(new byte[] { 0, 0, 1 }, uids.firstKey()); + assertArrayEquals(new byte[] { 0, 0, 2 }, + uids.firstEntry().getValue()); + } + + @Test + public void getTagUidsSalted() throws Exception { + setupSalt(); + final byte[] qual1 = { 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); + final KeyValue kv = makekv(qual12, + MockBase.concatByteArrays(val1, val2, ZERO)); + + final RowSeq rs = new RowSeq(tsdb); + rs.setRow(kv); + + final ByteMap<byte[]> uids = rs.getTagUids(); + assertEquals(1, uids.size()); + assertEquals(0, Bytes.memcmp(new byte[] { 0, 0, 1 }, uids.firstKey())); + assertEquals(0, Bytes.memcmp(new byte[] { 0, 0, 2 }, + uids.firstEntry().getValue())); + } + + @Test (expected = NullPointerException.class) + public void getTagUidsNotSet() throws Exception { + final RowSeq rs = new RowSeq(tsdb); + rs.getTagUids(); + } + + @Test + public void getAggregatedTagUids() throws Exception { + final byte[] qual1 = { 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); + final KeyValue kv = makekv(qual12, + MockBase.concatByteArrays(val1, val2, ZERO)); + + final RowSeq rs = new RowSeq(tsdb); + rs.setRow(kv); + + assertEquals(0, rs.getAggregatedTagUids().size()); + } + /** Shorthand to create a {@link KeyValue}. */ - private static KeyValue makekv(final byte[] qualifier, final byte[] value) { + public static KeyValue makekv(final byte[] qualifier, final byte[] value) { + if (Const.SALT_WIDTH() > 0) { + return new KeyValue(SALTED_KEY, FAMILY, qualifier, value); + } return new KeyValue(KEY, FAMILY, qualifier, value); } - private static KeyValue getMs() { + /** Shorthand to create a {@link KeyValue}. */ + public static KeyValue makekv(final byte[] key, final byte[] qualifier, + final byte[] value) { + return new KeyValue(key, FAMILY, qualifier, value); + } + + /** Helper that builds a KeyValue with millisecond timestamps */ + private static KeyValue getMs(final boolean salted) { final byte[] qual1 = { (byte) 0xF0, 0x00, 0x00, 0x07 }; final byte[] val1 = Bytes.fromLong(4L); final byte[] qual2 = { (byte) 0xF0, 0x00, 0x02, 0x07 }; @@ -632,8 +941,15 @@ private static KeyValue getMs() { final byte[] qual3 = { (byte) 0xF0, 0x00, 0x04, 0x07 }; final byte[] val3 = Bytes.fromLong(6L); final byte[] qual123 = MockBase.concatByteArrays(qual1, qual2, qual3); - final KeyValue kv = makekv(qual123, - MockBase.concatByteArrays(val1, val2, val3, ZERO)); + final KeyValue kv = makekv((salted ? SALTED_KEY : KEY), + qual123, MockBase.concatByteArrays(val1, val2, val3, ZERO)); return kv; } + + /** Helper to mockout the salt configuration */ + private void setupSalt() { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(2); + } } diff --git a/test/core/TestRpcResponsder.java b/test/core/TestRpcResponsder.java new file mode 100644 index 0000000000..5ddbf73baf --- /dev/null +++ b/test/core/TestRpcResponsder.java @@ -0,0 +1,65 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2010-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + + +import net.opentsdb.utils.Config; +import org.jboss.netty.util.internal.ThreadLocalRandom; +import org.junit.Assert; +import org.junit.Test; + +import java.util.concurrent.atomic.AtomicInteger; + +public class TestRpcResponsder { + + private final AtomicInteger complete_counter = new AtomicInteger(0); + + @Test(timeout = 60000) + public void testGracefulShutdown() throws InterruptedException { + RpcResponder rpcResponder = new RpcResponder(new Config()); + + final int n = 100; + for (int i = 0; i < n; i++) { + rpcResponder.response(new MockResponseProcess()); + } + + Thread.sleep(500); + rpcResponder.close(); + + try { + rpcResponder.response(new MockResponseProcess()); + Assert.fail("Expect an IllegalStateException"); + } catch (IllegalStateException ignore) { + } + + Assert.assertEquals(n, complete_counter.get()); + } + + private class MockResponseProcess implements Runnable { + + @Override + public void run() { + long duration = ThreadLocalRandom.current().nextInt(5000); + while (duration > 0) { + try { + Thread.sleep(100); + } catch (InterruptedException ignore) { + } + duration -= 100; + } + complete_counter.incrementAndGet(); + } + } + + +} diff --git a/test/core/TestSaltScanner.java b/test/core/TestSaltScanner.java new file mode 100644 index 0000000000..d3504d3b62 --- /dev/null +++ b/test/core/TestSaltScanner.java @@ -0,0 +1,556 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.TreeMap; + +import net.opentsdb.query.filter.TagVFilter; +import net.opentsdb.query.filter.TagVWildcardFilter; +import net.opentsdb.uid.UniqueId; + +import org.hbase.async.KeyValue; +import org.hbase.async.Scanner; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.stumbleupon.async.Deferred; +import com.google.common.collect.Maps; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ TSDB.class, Scanner.class, SaltScanner.class, Span.class, + Const.class, UniqueId.class }) +public class TestSaltScanner extends BaseTsdbTest { + protected byte[] KEY_A; + protected byte[] KEY_B; + protected byte[] KEY_C; + protected final static byte[] FAMILY = "t".getBytes(); + protected final static byte[] QUALIFIER_A = { 0x00, 0x00 }; + protected final static byte[] QUALIFIER_B = { 0x00, 0x10 }; + protected final static byte[] VALUE = { 0x42 }; + protected final static long VALUE_LONG = 66; + + protected List<Scanner> scanners; + protected TreeMap<byte[], Span> spans; + protected List<TagVFilter> filters; + + protected List<ArrayList<ArrayList<KeyValue>>> kvs_a; + protected List<ArrayList<ArrayList<KeyValue>>> kvs_b; + + protected Scanner scanner_a; + protected Scanner scanner_b; + + @Before + public void beforeLocal() throws Exception { + KEY_A = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_STRING); + KEY_B = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_B_STRING); + KEY_C = getRowKey(METRIC_STRING, 1359680400, TAGK_STRING, TAGV_STRING); + + filters = new ArrayList<TagVFilter>(); + + spans = new TreeMap<byte[], Span>(new RowKey.SaltCmp()); + setupMockScanners(true); + } + + @Test + public void ctor() { + assertNotNull(new SaltScanner(tsdb, METRIC_BYTES, scanners, spans, filters)); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNullTSDB() { + new SaltScanner(null, METRIC_BYTES, scanners, spans, filters); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNullMETRIC_BYTES() { + new SaltScanner(tsdb, null, scanners, spans, filters); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorShortMETRIC_BYTES() { + new SaltScanner(tsdb, new byte[] { 0, 1 }, scanners, spans, filters); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNullScanners() { + new SaltScanner(tsdb, METRIC_BYTES, null, spans, filters); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNotEnoughScanners() { + scanners.remove(0); + new SaltScanner(tsdb, METRIC_BYTES, scanners, spans, filters); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorTooManyScanners() { + scanners.add(mock(Scanner.class)); + new SaltScanner(tsdb, METRIC_BYTES, scanners, spans, filters); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNullSpans() { + new SaltScanner(tsdb, METRIC_BYTES, scanners, null, filters); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorSpansHaveData() { + spans.put(new byte[] { 0, 0, 0, 1 }, new Span(tsdb)); + new SaltScanner(tsdb, METRIC_BYTES, scanners, spans, filters); + } + + @Test + public void scanNoData() throws Exception { + final SaltScanner scanner = new SaltScanner(tsdb, METRIC_BYTES, scanners, + spans, filters); + assertTrue(Maps.difference(spans, scanner.scan().joinUninterruptibly()).areEqual()); + assertTrue(spans.isEmpty()); + } + + @Test + public void scan() throws Exception { + setupMockScanners(false); + final SaltScanner scanner = new SaltScanner(tsdb, METRIC_BYTES, scanners, + spans, filters); + assertTrue(Maps.difference(spans, scanner.scan().joinUninterruptibly()).areEqual()); + assertEquals(3, spans.size()); + + Span span = spans.get(KEY_A); + assertEquals(2, span.size()); + assertEquals(VALUE_LONG, span.longValue(0)); + assertEquals(1356998400000L, span.timestamp(0)); + assertEquals(VALUE_LONG, span.longValue(1)); + assertEquals(1356998401000L, span.timestamp(1)); + assertEquals(1, span.getAnnotations().size()); + + span = spans.get(KEY_B); + assertEquals(1, span.size()); + assertEquals(VALUE_LONG, span.longValue(0)); + assertEquals(1356998400000L, span.timestamp(0)); + assertEquals(0, span.getAnnotations().size()); + + span = spans.get(KEY_C); + assertEquals(2, span.size()); + assertEquals(VALUE_LONG, span.longValue(0)); + assertEquals(1359680400000L, span.timestamp(0)); + assertEquals(VALUE_LONG, span.longValue(1)); + assertEquals(1359680401000L, span.timestamp(1)); + assertEquals(0, span.getAnnotations().size()); + + verify(tag_values, never()).getNameAsync(TAGV_BYTES); + verify(tag_values, never()).getNameAsync(TAGV_B_BYTES); + } + + @Test + public void scanWithFilter() throws Exception { + setupMockScanners(false); + filters.add(TagVFilter.Builder().setType("regexp").setFilter("web.*") + .setTagk(TAGK_STRING).build()); + final SaltScanner scanner = new SaltScanner(tsdb, METRIC_BYTES, scanners, + spans, filters); + assertTrue(Maps.difference(spans, scanner.scan().joinUninterruptibly()).areEqual()); + assertEquals(3, spans.size()); + + Span span = spans.get(KEY_A); + assertEquals(2, span.size()); + assertEquals(VALUE_LONG, span.longValue(0)); + assertEquals(1356998400000L, span.timestamp(0)); + assertEquals(VALUE_LONG, span.longValue(1)); + assertEquals(1356998401000L, span.timestamp(1)); + assertEquals(1, span.getAnnotations().size()); + + span = spans.get(KEY_B); + assertEquals(1, span.size()); + assertEquals(VALUE_LONG, span.longValue(0)); + assertEquals(1356998400000L, span.timestamp(0)); + assertEquals(0, span.getAnnotations().size()); + + span = spans.get(KEY_C); + assertEquals(2, span.size()); + assertEquals(VALUE_LONG, span.longValue(0)); + assertEquals(1359680400000L, span.timestamp(0)); + assertEquals(VALUE_LONG, span.longValue(1)); + assertEquals(1359680401000L, span.timestamp(1)); + assertEquals(0, span.getAnnotations().size()); + + verify(tag_values, atLeast(1)).getNameAsync(TAGV_BYTES); + verify(tag_values, atLeast(1)).getNameAsync(TAGV_B_BYTES); + } + + @Test + public void scanWithTwoFilter() throws Exception { + setupMockScanners(false); + filters.add(TagVFilter.Builder().setType("regexp").setFilter("web.*") + .setTagk(TAGK_STRING).build()); + filters.add(TagVFilter.Builder().setType("wildcard").setFilter("web*") + .setTagk(TAGK_STRING).build()); + final SaltScanner scanner = new SaltScanner(tsdb, METRIC_BYTES, scanners, + spans, filters); + assertTrue(Maps.difference(spans, scanner.scan().joinUninterruptibly()).areEqual()); + assertEquals(3, spans.size()); + + Span span = spans.get(KEY_A); + assertEquals(2, span.size()); + assertEquals(VALUE_LONG, span.longValue(0)); + assertEquals(1356998400000L, span.timestamp(0)); + assertEquals(VALUE_LONG, span.longValue(1)); + assertEquals(1356998401000L, span.timestamp(1)); + assertEquals(1, span.getAnnotations().size()); + + span = spans.get(KEY_B); + assertEquals(1, span.size()); + assertEquals(VALUE_LONG, span.longValue(0)); + assertEquals(1356998400000L, span.timestamp(0)); + assertEquals(0, span.getAnnotations().size()); + + span = spans.get(KEY_C); + assertEquals(2, span.size()); + assertEquals(VALUE_LONG, span.longValue(0)); + assertEquals(1359680400000L, span.timestamp(0)); + assertEquals(VALUE_LONG, span.longValue(1)); + assertEquals(1359680401000L, span.timestamp(1)); + assertEquals(0, span.getAnnotations().size()); + + verify(tag_values, atLeast(1)).getNameAsync(TAGV_BYTES); + verify(tag_values, atLeast(1)).getNameAsync(TAGV_B_BYTES); + } + + @Test + public void scanWithFilterNoMatch() throws Exception { + setupMockScanners(false); + filters.add(TagVFilter.Builder().setType("regexp").setFilter("db.*") + .setTagk(TAGK_STRING).build()); + + final SaltScanner scanner = new SaltScanner(tsdb, METRIC_BYTES, scanners, + spans, filters); + assertTrue(Maps.difference(spans, scanner.scan().joinUninterruptibly()).areEqual()); + assertEquals(0, spans.size()); + + verify(tag_values, atLeast(1)).getNameAsync(TAGV_BYTES); + verify(tag_values, atLeast(1)).getNameAsync(TAGV_B_BYTES); + } + + @Test + public void scanWithTwoFiltersNoMatch() throws Exception { + setupMockScanners(false); + filters.add(TagVFilter.Builder().setType("regexp").setFilter("web.*") + .setTagk(TAGK_STRING).build()); + filters.add(TagVFilter.Builder().setType("wildcard").setFilter("db*") + .setTagk(TAGK_STRING).build()); + final SaltScanner scanner = new SaltScanner(tsdb, METRIC_BYTES, scanners, + spans, filters); + assertTrue(Maps.difference(spans, scanner.scan().joinUninterruptibly()).areEqual()); + assertEquals(0, spans.size()); + + verify(tag_values, atLeast(1)).getNameAsync(TAGV_BYTES); + verify(tag_values, atLeast(1)).getNameAsync(TAGV_B_BYTES); + } + + @Test + public void scanHBaseScannerFromDeferredA() throws Exception { + setupMockScanners(false); + // we can't instantiate an HBaseException so just throw a RuntimeException + final RuntimeException e = new RuntimeException("From HBase"); + when(scanner_a.nextRows()) + .thenReturn(Deferred.fromResult(kvs_a.get(0))) + .thenReturn(Deferred. + <ArrayList<ArrayList<KeyValue>>>fromError(e)); + + final SaltScanner scanner = new SaltScanner(tsdb, METRIC_BYTES, scanners, + spans, filters); + try { + scanner.scan().joinUninterruptibly(); + fail("Expected a runtime exception here"); + } catch (RuntimeException re) { + assertEquals(e, re); + } + } + + @Test + public void scanHBaseScannerFromDeferredB() throws Exception { + if (Const.SALT_WIDTH() > 0) { + setupMockScanners(false); + // we can't instantiate an HBaseException so just throw a RuntimeException + final RuntimeException e = new RuntimeException("From HBase"); + when(scanner_b.nextRows()) + .thenReturn(Deferred.fromResult(kvs_b.get(0))) + .thenReturn(Deferred.fromResult(kvs_b.get(1))) + .thenReturn(Deferred. + <ArrayList<ArrayList<KeyValue>>>fromError(e)); + + final SaltScanner scanner = new SaltScanner(tsdb, METRIC_BYTES, scanners, + spans, filters); + try { + scanner.scan().joinUninterruptibly(); + fail("Expected a runtime exception here"); + } catch (RuntimeException re) { + assertEquals(e, re); + } + } + } + + @Test + public void scanHBaseScannerThrownA() throws Exception { + setupMockScanners(false); + // we can't instantiate an HBaseException so just throw a RuntimeException + final RuntimeException e = new RuntimeException("From HBase"); + when(scanner_a.nextRows()) + .thenReturn(Deferred.fromResult(kvs_a.get(0))) + .thenThrow(e); + + final SaltScanner scanner = new SaltScanner(tsdb, METRIC_BYTES, scanners, + spans, filters); + try { + scanner.scan().joinUninterruptibly(); + fail("Expected a runtime exception here"); + } catch (RuntimeException re) { + assertEquals(e, re); + } + } + + @Test + public void scanHBaseScannerThrownB() throws Exception { + if (Const.SALT_WIDTH() > 0) { + setupMockScanners(false); + // we can't instantiate an HBaseException so just throw a RuntimeException + final RuntimeException e = new RuntimeException("From HBase"); + when(scanner_b.nextRows()) + .thenReturn(Deferred.fromResult(kvs_b.get(0))) + .thenReturn(Deferred.fromResult(kvs_b.get(1))) + .thenThrow(e); + + final SaltScanner scanner = new SaltScanner(tsdb, METRIC_BYTES, scanners, + spans, filters); + try { + scanner.scan().joinUninterruptibly(); + fail("Expected a runtime exception here"); + } catch (RuntimeException re) { + assertEquals(e, re); + } + } + } + + @Test (expected = IllegalDataException.class) + public void scanBadRowKey() throws Exception { + setupMockScanners(false); + + final ArrayList<ArrayList<KeyValue>> rows = + new ArrayList<ArrayList<KeyValue>>(1); + final ArrayList<KeyValue> row = new ArrayList<KeyValue>(1); + rows.add(row); + final byte[] key = { 0x00, 0x00, 0x00, 0x02, + 0x51, (byte) 0x0B, 0x13, (byte) 0x90, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01 }; + row.add(new KeyValue(key, FAMILY, QUALIFIER_A, 0, VALUE)); + kvs_a.set(2, rows); + + when(scanner_a.nextRows()) + .thenReturn(Deferred.fromResult(kvs_a.get(0))) + .thenReturn(Deferred.fromResult(kvs_a.get(1))) + .thenReturn(Deferred.fromResult(kvs_a.get(2))) + .thenReturn(Deferred.<ArrayList<ArrayList<KeyValue>>>fromResult(null)); + + final SaltScanner scanner = new SaltScanner(tsdb, METRIC_BYTES, scanners, + spans, filters); + scanner.scan().joinUninterruptibly(); + } + + @SuppressWarnings("unchecked") + @Test (expected = IllegalDataException.class) + public void scanCompactionDataException() throws Exception { + setupMockScanners(false); + + doThrow(new IllegalDataException("Boo!")).when( + tsdb).compact(any(ArrayList.class), any(List.class), any(List.class)); + + final SaltScanner scanner = new SaltScanner(tsdb, METRIC_BYTES, scanners, + spans, filters); + scanner.scan().joinUninterruptibly(); + } + + @SuppressWarnings("unchecked") + @Test (expected = RuntimeException.class) + public void scanCompactionRuntimeException() throws Exception { + setupMockScanners(false); + + doThrow(new RuntimeException("Boo!")).when( + tsdb).compact(any(ArrayList.class), any(List.class), any(List.class)); + + final SaltScanner scanner = new SaltScanner(tsdb, METRIC_BYTES, scanners, + spans, filters); + scanner.scan().joinUninterruptibly(); + } + + @Test + public void scanTooManyDps() throws Exception { + setupMockScanners(false); + List<TagVFilter> filters = new ArrayList<TagVFilter>(1); + filters.add(new TagVWildcardFilter(TAGK_STRING, "web*")); + + final SaltScanner scanner = new SaltScanner(tsdb, METRIC_BYTES, scanners, + spans, null, false, null, null, 0, null, 0, 1); + try { + scanner.scan().joinUninterruptibly(); + fail("Excpected a QueryException"); + } catch (QueryException e) { + assertEquals(HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE, e.getStatus()); + } + } + + @Test + public void scanTooManyBytes() throws Exception { + setupMockScanners(false); + List<TagVFilter> filters = new ArrayList<TagVFilter>(1); + filters.add(new TagVWildcardFilter(TAGK_STRING, "web*")); + + final SaltScanner scanner = new SaltScanner(tsdb, METRIC_BYTES, scanners, + spans, null, false, null, null, 0, null, 2, 0); + try { + scanner.scan().joinUninterruptibly(); + fail("Excpected a QueryException"); + } catch (QueryException e) { + assertEquals(HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE, e.getStatus()); + } + config.overrideConfig("tsd.core.scanner.max_bytes", "0"); + } + + + /** + * Sets up a pair of scanners with either a list of values or no data + * @param no_data Whether or not to return 0 data. + */ + protected void setupMockScanners(final boolean no_data) throws Exception { + if (Const.SALT_WIDTH() > 0) { + scanners = new ArrayList<Scanner>(Const.SALT_BUCKETS()); + scanner_a = mock(Scanner.class); + scanner_b = mock(Scanner.class); + if (no_data) { + when(scanner_a.nextRows()).thenReturn( + Deferred.<ArrayList<ArrayList<KeyValue>>>fromResult(null)); + when(scanner_b.nextRows()).thenReturn( + Deferred.<ArrayList<ArrayList<KeyValue>>>fromResult(null)); + } else { + setupValues(); + } + scanners.add(scanner_a); + scanners.add(scanner_b); + } else { + scanners = new ArrayList<Scanner>(1); + scanner_a = mock(Scanner.class); + if (no_data) { + when(scanner_a.nextRows()).thenReturn( + Deferred.<ArrayList<ArrayList<KeyValue>>>fromResult(null)); + } else { + setupValues(); + } + scanners.add(scanner_a); + } + } + + /** + * This method sets up some row keys and values to pass to the scanners. + * The values aren't exactly what would normally be passed to a salt scanner + * in that we have the same series salted across separate buckets. That would + * only happen if you add the timestamp to the salt calculation, which we + * may do in the future. We're testing now for future proofing. + */ + protected void setupValues() throws Exception { + setDataPointStorage(); + kvs_a = new ArrayList<ArrayList<ArrayList<KeyValue>>>(3); + kvs_b = new ArrayList<ArrayList<ArrayList<KeyValue>>>(2); + + final String note = "{\"tsuid\":\"000001000001000001\"," + + "\"startTime\":1356998490,\"endTime\":0,\"description\":" + + "\"The Great A'Tuin!\",\"notes\":\"Millenium hand and shrimp\"," + + "\"custom\":null}"; + + for (int i = 0; i < 5; i++) { + final ArrayList<ArrayList<KeyValue>> rows = + new ArrayList<ArrayList<KeyValue>>(1); + final ArrayList<KeyValue> row = new ArrayList<KeyValue>(2); + rows.add(row); + byte[] key = null; + + switch (i) { + case 0: + row.add(new KeyValue(KEY_A, FAMILY, QUALIFIER_A, 0, VALUE)); + kvs_a.add(rows); + break; + case 1: + row.add(new KeyValue(KEY_B, FAMILY, QUALIFIER_A, 0, VALUE)); + kvs_a.add(rows); + break; + case 2: + row.add(new KeyValue(KEY_C, FAMILY, QUALIFIER_A, 0, VALUE)); + kvs_a.add(rows); + break; + case 3: + key = Arrays.copyOf(KEY_A, KEY_A.length); + row.add(new KeyValue(key, FAMILY, QUALIFIER_B, 0, VALUE)); + row.add(new KeyValue(key, FAMILY, new byte[] { 1, 0, 0 }, 0, + note.getBytes(Charset.forName("UTF8")))); + kvs_b.add(rows); + break; + case 4: + key = Arrays.copyOf(KEY_C, KEY_C.length); + row.add(new KeyValue(key, FAMILY, QUALIFIER_B, 0, VALUE)); + kvs_b.add(rows); + break; + } + } + + if (Const.SALT_WIDTH() > 0) { + when(scanner_a.nextRows()) + .thenReturn(Deferred.fromResult(kvs_a.get(0))) + .thenReturn(Deferred.fromResult(kvs_a.get(1))) + .thenReturn(Deferred.fromResult(kvs_a.get(2))) + .thenReturn(Deferred.<ArrayList<ArrayList<KeyValue>>>fromResult(null)); + + when(scanner_b.nextRows()) + .thenReturn(Deferred.fromResult(kvs_b.get(0))) + .thenReturn(Deferred.fromResult(kvs_b.get(1))) + .thenReturn(Deferred.<ArrayList<ArrayList<KeyValue>>>fromResult(null)); + } else { + when(scanner_a.nextRows()) + .thenReturn(Deferred.fromResult(kvs_a.get(0))) + .thenReturn(Deferred.fromResult(kvs_a.get(1))) + .thenReturn(Deferred.fromResult(kvs_a.get(2))) + .thenReturn(Deferred.fromResult(kvs_b.get(0))) + .thenReturn(Deferred.fromResult(kvs_b.get(1))) + .thenReturn(Deferred.<ArrayList<ArrayList<KeyValue>>>fromResult(null)); + } + } +} diff --git a/test/core/TestSaltScannerHistogram.java b/test/core/TestSaltScannerHistogram.java new file mode 100644 index 0000000000..5925df891d --- /dev/null +++ b/test/core/TestSaltScannerHistogram.java @@ -0,0 +1,412 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import com.esotericsoftware.kryo.io.Output; +import com.google.common.collect.Maps; +import com.stumbleupon.async.Deferred; + +import net.opentsdb.query.filter.TagVFilter; +import net.opentsdb.query.filter.TagVRegexFilter; +import net.opentsdb.query.filter.TagVWildcardFilter; +import net.opentsdb.stats.QueryStats; +import net.opentsdb.uid.UniqueId; +import net.opentsdb.uid.UniqueId.UniqueIdType; +import net.opentsdb.utils.Config; +import net.opentsdb.utils.DateTime; +import net.opentsdb.utils.Threads; + +import org.hbase.async.HBaseClient; +import org.hbase.async.KeyValue; +import org.hbase.async.Scanner; +import org.jboss.netty.util.HashedWheelTimer; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +import com.google.common.collect.Maps; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.TreeMap; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ TSDB.class, Scanner.class, SaltScanner.class, Span.class, + Const.class, UniqueId.class, Tags.class, QueryStats.class, DateTime.class, + HistogramCodecManager.class, + SimpleHistogram.class, SimpleHistogramDecoder.class}) +public class TestSaltScannerHistogram extends BaseTsdbTest { + protected final static byte[] FAMILY = "t".getBytes(); + protected final static byte[] QUALIFIER_A = { 0x06, 0x00, 0x00}; + protected final static byte[] QUALIFIER_B = { 0x06, 0x10, 0x00 }; + + protected byte[] VALUE; + + protected List<Scanner> scanners; + protected TreeMap<byte[], HistogramSpan> spans; + + protected List<ArrayList<ArrayList<KeyValue>>> kvs_a; + protected List<ArrayList<ArrayList<KeyValue>>> kvs_b; + + protected Scanner scanner_a; + protected Scanner scanner_b; + protected QueryStats query_stats; + + protected byte[] key_a; + //different tagv + protected byte[] key_b; + //same as A bug different time + protected byte[] key_c; + + @Before + public void before() throws Exception { + // Copying the whole thing as the SPY in the base mucks up the references. + uid_map = Maps.newHashMap(); + PowerMockito.mockStatic(Threads.class); + timer = new FakeTaskTimer(); + PowerMockito.when(Threads.newTimer(anyString())).thenReturn(timer); + PowerMockito.when(Threads.newTimer(anyInt(), anyString())).thenReturn(timer); + + PowerMockito.whenNew(HashedWheelTimer.class).withNoArguments() + .thenReturn(timer); + PowerMockito.whenNew(HBaseClient.class).withAnyArguments() + .thenReturn(client); + + config = new Config(false); + config.overrideConfig("tsd.storage.enable_compaction", "false"); + tsdb = new TSDB(config); + + config.setAutoMetric(true); + + Whitebox.setInternalState(tsdb, "metrics", metrics); + Whitebox.setInternalState(tsdb, "tag_names", tag_names); + Whitebox.setInternalState(tsdb, "tag_values", tag_values); + + setupMetricMaps(); + setupTagkMaps(); + setupTagvMaps(); + + mockUID(UniqueIdType.METRIC, HISTOGRAM_METRIC_STRING, HISTOGRAM_METRIC_BYTES); + + // add metrics and tags to the UIDs list for other functions to share + uid_map.put(METRIC_STRING, METRIC_BYTES); + uid_map.put(METRIC_B_STRING, METRIC_B_BYTES); + uid_map.put(NSUN_METRIC, NSUI_METRIC); + uid_map.put(HISTOGRAM_METRIC_STRING, HISTOGRAM_METRIC_BYTES); + + uid_map.put(TAGK_STRING, TAGK_BYTES); + uid_map.put(TAGK_B_STRING, TAGK_B_BYTES); + uid_map.put(NSUN_TAGK, NSUI_TAGK); + + uid_map.put(TAGV_STRING, TAGV_BYTES); + uid_map.put(TAGV_B_STRING, TAGV_B_BYTES); + uid_map.put(NSUN_TAGV, NSUI_TAGV); + + uid_map.putAll(UIDS); + + when(metrics.width()).thenReturn((short)3); + when(tag_names.width()).thenReturn((short)3); + when(tag_values.width()).thenReturn((short)3); + + tags = new HashMap<String, String>(1); + tags.put(TAGK_STRING, TAGV_STRING); + config.overrideConfig("tsd.core.histograms.config", + "{\"net.opentsdb.core.LongHistogramDataPointForTestDecoder\": 0}"); + HistogramCodecManager manager = + new HistogramCodecManager(tsdb); + Whitebox.setInternalState(tsdb, "histogram_manager", manager); + + ByteArrayOutputStream outBuffer = new ByteArrayOutputStream(); + Output output = new Output(outBuffer); + output.writeByte(0 /*HistoType.SimpleHistogramType.ordinal()*/); + output.writeShort(8); + output.writeFloat(1.0f); + output.writeFloat(2.0f); + output.writeLong(5, true); + output.writeFloat(2.0f); + output.writeFloat(3.0f); + output.writeLong(5, true); + output.writeFloat(3.0f); + output.writeFloat(4.0f); + output.writeLong(5, true); + output.writeFloat(4.0f); + output.writeFloat(5.0f); + output.writeLong(0, true); + output.writeFloat(5.0f); + output.writeFloat(6.0f); + output.writeLong(0, true); + output.writeFloat(6.0f); + output.writeFloat(7.0f); + output.writeLong(0, true); + output.writeFloat(7.0f); + output.writeFloat(8.0f); + output.writeLong(0, true); + output.writeFloat(8.0f); + output.writeFloat(9.0f); + output.writeLong(0, true); + output.writeLong(0, true); + output.writeLong(2, true); + output.close(); + VALUE = outBuffer.toByteArray(); + + query_stats = mock(QueryStats.class); + spans = new TreeMap<byte[], HistogramSpan>(new RowKey.SaltCmp()); + setupMockScanners(true); + + key_a = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_STRING, + TAGK_B_STRING, TAGV_STRING); + key_b = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_B_STRING, + TAGK_B_STRING, TAGV_STRING); + key_c = getRowKey(METRIC_STRING, 1359680400, TAGK_STRING, TAGV_STRING, + TAGK_B_STRING, TAGV_STRING); + } + + @Test + public void scan() throws Exception { + setupMockScanners(false); + + SimpleHistogram y1Hist = mock(SimpleHistogram.class); + PowerMockito.whenNew(SimpleHistogram.class).withAnyArguments().thenReturn(y1Hist); + + final SaltScanner scanner = new SaltScanner(tsdb, METRIC_BYTES, scanners, + null, null, false, null, query_stats, 0, spans, 0, 0); + assertTrue(Maps.difference(spans, scanner.scanHistogram().joinUninterruptibly()).areEqual()); + assertEquals(3, spans.size()); + + HistogramSpan span = spans.get(key_a); + assertEquals(2, span.size()); + assertEquals(1356998400000L, span.timestamp(0)); + assertEquals(1357002496000L, span.timestamp(1)); + assertEquals(1, span.getAnnotations().size()); + + span = spans.get(key_b); + assertEquals(1, span.size()); + assertEquals(1356998400000L, span.timestamp(0)); + assertEquals(0, span.getAnnotations().size()); + + span = spans.get(key_c); + assertEquals(2, span.size()); + assertEquals(1359680400000L, span.timestamp(0)); + assertEquals(1359684496000L, span.timestamp(1)); + assertEquals(0, span.getAnnotations().size()); + } + + @Test + public void scanWithFilter() throws Exception { + setupMockScanners(false); + List<TagVFilter> filters = new ArrayList<TagVFilter>(1); + filters.add(new TagVWildcardFilter(TAGK_STRING, "web*")); + + SimpleHistogram y1Hist = mock(SimpleHistogram.class); + PowerMockito.whenNew(SimpleHistogram.class).withAnyArguments().thenReturn(y1Hist); + + final SaltScanner scanner = new SaltScanner(tsdb, METRIC_BYTES, scanners, + null, null, false, null, query_stats, 0, spans, 0, 0); + + assertTrue(Maps.difference(spans, scanner.scanHistogram().joinUninterruptibly()).areEqual()); + assertEquals(3, spans.size()); + + HistogramSpan span = spans.get(key_a); + assertEquals(2, span.size()); + assertEquals(1356998400000L, span.timestamp(0)); + assertEquals(1357002496000L, span.timestamp(1)); + assertEquals(1, span.getAnnotations().size()); + + span = spans.get(key_b); + assertEquals(1, span.size()); + assertEquals(1356998400000L, span.timestamp(0)); + assertEquals(0, span.getAnnotations().size()); + + span = spans.get(key_c); + assertEquals(2, span.size()); + assertEquals(1359680400000L, span.timestamp(0)); + assertEquals(1359684496000L, span.timestamp(1)); + assertEquals(0, span.getAnnotations().size()); + } + + @Test + public void scanWithFiltersOnSameTag() throws Exception { + setupMockScanners(false); + List<TagVFilter> filters = new ArrayList<TagVFilter>(1); + filters.add(new TagVWildcardFilter("host", "web*")); + filters.add(new TagVWildcardFilter("host", "w*b*")); + filters.add(new TagVRegexFilter("host", "w.*")); + + SimpleHistogram y1Hist = mock(SimpleHistogram.class); + PowerMockito.whenNew(SimpleHistogram.class).withAnyArguments().thenReturn(y1Hist); + + final SaltScanner scanner = new SaltScanner(tsdb, METRIC_BYTES, scanners, + null, null, false, null, query_stats, 0, spans, 0, 0); + + assertTrue(Maps.difference(spans, scanner.scanHistogram().joinUninterruptibly()).areEqual()); + assertEquals(3, spans.size()); + + HistogramSpan span = spans.get(key_a); + assertEquals(2, span.size()); + assertEquals(1356998400000L, span.timestamp(0)); + assertEquals(1357002496000L, span.timestamp(1)); + assertEquals(1, span.getAnnotations().size()); + + span = spans.get(key_b); + assertEquals(1, span.size()); + assertEquals(1356998400000L, span.timestamp(0)); + assertEquals(0, span.getAnnotations().size()); + + span = spans.get(key_c); + assertEquals(2, span.size()); + assertEquals(1359680400000L, span.timestamp(0)); + assertEquals(1359684496000L, span.timestamp(1)); + assertEquals(0, span.getAnnotations().size()); + } + + @Test + public void scanWithFiltersOnSameTagOneFail() throws Exception { + setupMockScanners(false); + List<TagVFilter> filters = new ArrayList<TagVFilter>(1); + filters.add(new TagVWildcardFilter("host", "web*")); + filters.add(new TagVWildcardFilter("host", "drood*")); + + SimpleHistogram y1Hist = mock(SimpleHistogram.class); + PowerMockito.whenNew(SimpleHistogram.class).withAnyArguments().thenReturn(y1Hist); + + final SaltScanner scanner = new SaltScanner(tsdb, METRIC_BYTES, scanners, + null, filters, false, null, query_stats, 0, spans, 0, 0); + + assertTrue(Maps.difference(spans, scanner.scan().joinUninterruptibly()).areEqual()); + assertTrue(Maps.difference(spans, scanner.scanHistogram().joinUninterruptibly()).areEqual()); + assertEquals(0, spans.size()); + } + + /** + * Sets up a pair of scanners with either a list of values or no data + * @param no_data Whether or not to return 0 data. + */ + protected void setupMockScanners(final boolean no_data) { + if (Const.SALT_WIDTH() > 0) { + scanners = new ArrayList<Scanner>(Const.SALT_BUCKETS()); + scanner_a = mock(Scanner.class); + scanner_b = mock(Scanner.class); + if (no_data) { + when(scanner_a.nextRows()).thenReturn( + Deferred.<ArrayList<ArrayList<KeyValue>>>fromResult(null)); + when(scanner_b.nextRows()).thenReturn( + Deferred.<ArrayList<ArrayList<KeyValue>>>fromResult(null)); + } else { + setupValues(); + } + scanners.add(scanner_a); + scanners.add(scanner_b); + } else { + scanners = new ArrayList<Scanner>(1); + scanner_a = mock(Scanner.class); + if (no_data) { + when(scanner_a.nextRows()).thenReturn( + Deferred.<ArrayList<ArrayList<KeyValue>>>fromResult(null)); + } else { + setupValues(); + } + scanners.add(scanner_a); + } + } + + /** + * This method sets up some row keys and values to pass to the scanners. + * The values aren't exactly what would normally be passed to a salt scanner + * in that we have the same series salted across separate buckets. That would + * only happen if you add the timestamp to the salt calculation, which we + * may do in the future. We're testing now for future proofing. + */ + protected void setupValues() { + kvs_a = new ArrayList<ArrayList<ArrayList<KeyValue>>>(3); + kvs_b = new ArrayList<ArrayList<ArrayList<KeyValue>>>(2); + + final String note = "{\"tsuid\":\"000000010000000100000001\"," + + "\"startTime\":1356998490,\"endTime\":0,\"description\":" + + "\"The Great A'Tuin!\",\"notes\":\"Millenium hand and shrimp\"," + + "\"custom\":null}"; + + for (int i = 0; i < 5; i++) { + final ArrayList<ArrayList<KeyValue>> rows = + new ArrayList<ArrayList<KeyValue>>(1); + final ArrayList<KeyValue> row = new ArrayList<KeyValue>(2); + rows.add(row); + byte[] key = null; + + switch (i) { + case 0: + row.add(new KeyValue(key_a, FAMILY, QUALIFIER_A, 0, VALUE)); + kvs_a.add(rows); + break; + case 1: + row.add(new KeyValue(key_b, FAMILY, QUALIFIER_A, 0, VALUE)); + kvs_a.add(rows); + break; + case 2: + row.add(new KeyValue(key_c, FAMILY, QUALIFIER_A, 0, VALUE)); + kvs_a.add(rows); + break; + case 3: + key = Arrays.copyOf(key_a, key_a.length); + row.add(new KeyValue(key, FAMILY, QUALIFIER_B, 0, VALUE)); + row.add(new KeyValue(key, FAMILY, new byte[] { 1, 0, 0 }, 0, + note.getBytes(Charset.forName("UTF8")))); + kvs_b.add(rows); + break; + case 4: + key = Arrays.copyOf(key_c, key_c.length); + row.add(new KeyValue(key, FAMILY, QUALIFIER_B, 0, VALUE)); + kvs_b.add(rows); + break; + } + } + + if (Const.SALT_WIDTH() > 0) { + when(scanner_a.nextRows()) + .thenReturn(Deferred.fromResult(kvs_a.get(0))) + .thenReturn(Deferred.fromResult(kvs_a.get(1))) + .thenReturn(Deferred.fromResult(kvs_a.get(2))) + .thenReturn(Deferred.<ArrayList<ArrayList<KeyValue>>>fromResult(null)); + when(scanner_b.nextRows()) + .thenReturn(Deferred.fromResult(kvs_b.get(0))) + .thenReturn(Deferred.fromResult(kvs_b.get(1))) + .thenReturn(Deferred.<ArrayList<ArrayList<KeyValue>>>fromResult(null)); + } else { + when(scanner_a.nextRows()) + .thenReturn(Deferred.fromResult(kvs_a.get(0))) + .thenReturn(Deferred.fromResult(kvs_a.get(1))) + .thenReturn(Deferred.fromResult(kvs_a.get(2))) + .thenReturn(Deferred.fromResult(kvs_b.get(0))) + .thenReturn(Deferred.fromResult(kvs_b.get(1))) + .thenReturn(Deferred.<ArrayList<ArrayList<KeyValue>>>fromResult(null)); + } + } +} diff --git a/test/core/TestSaltScannerHistogramSalted.java b/test/core/TestSaltScannerHistogramSalted.java new file mode 100644 index 0000000000..9e4d56fac3 --- /dev/null +++ b/test/core/TestSaltScannerHistogramSalted.java @@ -0,0 +1,34 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import org.junit.Before; +import org.powermock.api.mockito.PowerMockito; + +public class TestSaltScannerHistogramSalted extends TestSaltScannerHistogram { + + @Before + public void beforeLocal() throws Exception { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(2); + + key_a = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_STRING, + TAGK_B_STRING, TAGV_STRING); + key_b = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_B_STRING, + TAGK_B_STRING, TAGV_STRING); + key_c = getRowKey(METRIC_STRING, 1359680400, TAGK_STRING, TAGV_STRING, + TAGK_B_STRING, TAGV_STRING); + } + +} diff --git a/test/core/TestSaltScannerSalted.java b/test/core/TestSaltScannerSalted.java new file mode 100644 index 0000000000..bd859fb09f --- /dev/null +++ b/test/core/TestSaltScannerSalted.java @@ -0,0 +1,53 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import java.util.ArrayList; +import java.util.TreeMap; + +import net.opentsdb.query.filter.TagVFilter; +import net.opentsdb.uid.UniqueId; + +import org.hbase.async.Scanner; +import org.junit.Before; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ TSDB.class, Scanner.class, SaltScanner.class, Span.class, + Const.class, UniqueId.class }) +public class TestSaltScannerSalted extends TestSaltScanner { + + @Before + public void beforeLocal() throws Exception { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(2); + + KEY_A = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_STRING); + KEY_B = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_B_STRING); + KEY_C = getRowKey(METRIC_STRING, 1359680400, TAGK_STRING, TAGV_STRING); + + filters = new ArrayList<TagVFilter>(); + + spans = new TreeMap<byte[], Span>(new RowKey.SaltCmp()); + setupMockScanners(true); + } + +} diff --git a/test/core/TestSeekableViewChain.java b/test/core/TestSeekableViewChain.java new file mode 100644 index 0000000000..5350abd29f --- /dev/null +++ b/test/core/TestSeekableViewChain.java @@ -0,0 +1,99 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +@RunWith(PowerMockRunner.class) +public class TestSeekableViewChain { + + private static final long BASE_TIME = 1356998400000L; + private static final DataPoint[] DATA_POINTS_1 = new DataPoint[]{ + MutableDataPoint.ofLongValue(BASE_TIME, 40), + MutableDataPoint.ofLongValue(BASE_TIME + 10000, 50), + MutableDataPoint.ofLongValue(BASE_TIME + 30000, 70) + }; + + @Before + public void before() throws Exception { + + } + + @Test + public void testIteratorChain() { + List<SeekableView> iterators = new ArrayList<SeekableView>(); + for (int i = 0; i < 3; i++) { + iterators.add(SeekableViewsForTest.fromArray(DATA_POINTS_1)); + } + SeekableViewChain chain = new SeekableViewChain(iterators); + + int items = 0; + while (chain.hasNext()) { + chain.next(); + items += 1; + } + + assertEquals(9, items); + } + + @Test + public void testSeek() { + List<SeekableView> iterators = new ArrayList<SeekableView>(); + iterators.add(SeekableViewsForTest.generator( + BASE_TIME, 10000, 5, true + )); + iterators.add(SeekableViewsForTest.generator( + BASE_TIME + 50000, 10000, 5, true + )); + + SeekableViewChain chain = new SeekableViewChain(iterators); + + chain.seek(BASE_TIME + 75000); + + int items = 0; + while (chain.hasNext()) { + chain.next(); + items += 1; + } + + assertEquals(2, items); + } + + @Test + public void testEmptyChain() { + SeekableViewChain chain = new SeekableViewChain(new ArrayList<SeekableView>()); + assertFalse(chain.hasNext()); + } + + @Test(expected = UnsupportedOperationException.class) + public void testRemoveUnsupported() { + makeChain(1).remove(); + } + + private SeekableViewChain makeChain(int numIterators) { + List<SeekableView> iterators = new ArrayList<SeekableView>(); + for (int i = 0; i < numIterators; i++) { + iterators.add(SeekableViewsForTest.fromArray(DATA_POINTS_1)); + } + return new SeekableViewChain(iterators); + } +} diff --git a/test/core/TestSimpleHistogram.java b/test/core/TestSimpleHistogram.java new file mode 100644 index 0000000000..046232678a --- /dev/null +++ b/test/core/TestSimpleHistogram.java @@ -0,0 +1,623 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Map; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; +import com.fasterxml.jackson.databind.ObjectMapper; + +import net.opentsdb.core.HistogramDataPoint.HistogramBucket; +import net.opentsdb.core.HistogramDataPoint.HistogramBucket.BucketType; +import net.opentsdb.utils.Config; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({ TSDB.class }) +public class TestSimpleHistogram { + + private TSDB tsdb; + private Config config; + private HistogramCodecManager manager; + + @Before + public void before() throws Exception { + tsdb = PowerMockito.mock(TSDB.class); + config = new Config(false); + + config.overrideConfig("tsd.core.histograms.config", + "{\"net.opentsdb.core.SimpleHistogramDecoder\": 0}"); + when(tsdb.getConfig()).thenReturn(config); + + manager = new HistogramCodecManager(tsdb); + when(tsdb.histogramManager()).thenReturn(manager); + } + + @Test + public void verifyE2EKryo() { + Kryo kryo = new Kryo(); + + //Encoding stage + ByteArrayOutputStream outBuffer = new ByteArrayOutputStream(); + Output output = new Output(outBuffer); + // This is the type of histogram (or sketch) written to storage + output.writeByte(0); + output.writeShort(8); + output.writeFloat(1.0f); + output.writeFloat(2.0f); + output.writeLong(5, true); + output.writeFloat(2.0f); + output.writeFloat(3.0f); + output.writeLong(5, true); + output.writeFloat(3.0f); + output.writeFloat(4.0f); + output.writeLong(5, true); + output.writeFloat(4.0f); + output.writeFloat(5.0f); + output.writeLong(0, true); + output.writeFloat(5.0f); + output.writeFloat(6.0f); + output.writeLong(0, true); + output.writeFloat(6.0f); + output.writeFloat(7.0f); + output.writeLong(0, true); + output.writeFloat(7.0f); + output.writeFloat(8.0f); + output.writeLong(0, true); + output.writeFloat(8.0f); + output.writeFloat(9.0f); + output.writeLong(0, true); + output.writeLong(0, true); + output.writeLong(2, true); + output.close(); + + //Decoding stage + Input input = new Input(new ByteArrayInputStream(outBuffer.toByteArray())); + int metricType = input.readByte(); + + switch (metricType) { + case 0: + SimpleHistogram y1Hist = new SimpleHistogram(0); + y1Hist.read(kryo, input); + Input verifyHist = new Input(new ByteArrayInputStream(y1Hist.histogram(false))); + + int bucketCount = verifyHist.readShort(); + assertEquals(bucketCount, 8); + + Float bucketLowerBound = verifyHist.readFloat(); + Float bucketUpperBound = verifyHist.readFloat(); + long bucketVal = verifyHist.readLong(true); + assertEquals(bucketLowerBound, 1.0f, 0.0001); + assertEquals(bucketVal, 5); + assertEquals(y1Hist.getOverflow(), Long.valueOf(2L)); + break; + default: + System.out.println("Failed to detect histogram type"); + assertTrue(false); + } + input.close(); + } + + @Test + public void testToFromBytes() { + //Encoding stage + ByteArrayOutputStream out = new ByteArrayOutputStream(); + Output output = new Output(out); + // This is the type of histogram (or sketch) written to storage + output.writeByte(42); + output.writeShort(8); + output.writeFloat(1.0f); + output.writeFloat(2.0f); + output.writeLong(5, true); + output.writeFloat(2.0f); + output.writeFloat(3.0f); + output.writeLong(5, true); + output.writeFloat(3.0f); + output.writeFloat(4.0f); + output.writeLong(5, true); + output.writeFloat(4.0f); + output.writeFloat(5.0f); + output.writeLong(0, true); + output.writeFloat(5.0f); + output.writeFloat(6.0f); + output.writeLong(0, true); + output.writeFloat(6.0f); + output.writeFloat(7.0f); + output.writeLong(0, true); + output.writeFloat(7.0f); + output.writeFloat(8.0f); + output.writeLong(0, true); + output.writeFloat(8.0f); + output.writeFloat(9.0f); + output.writeLong(0, true); + output.writeLong(0, true); + output.writeLong(2, true); + output.close(); + + SimpleHistogram hist = new SimpleHistogram(42); + hist.fromHistogram(out.toByteArray(), true); + assertArrayEquals(out.toByteArray(), hist.histogram(true)); + + // skip the id + out = new ByteArrayOutputStream(); + output = new Output(out); + // This is the type of histogram (or sketch) written to storage + //output.writeByte(42); // <-- Skipped! + output.writeShort(8); + output.writeFloat(1.0f); + output.writeFloat(2.0f); + output.writeLong(5, true); + output.writeFloat(2.0f); + output.writeFloat(3.0f); + output.writeLong(5, true); + output.writeFloat(3.0f); + output.writeFloat(4.0f); + output.writeLong(5, true); + output.writeFloat(4.0f); + output.writeFloat(5.0f); + output.writeLong(0, true); + output.writeFloat(5.0f); + output.writeFloat(6.0f); + output.writeLong(0, true); + output.writeFloat(6.0f); + output.writeFloat(7.0f); + output.writeLong(0, true); + output.writeFloat(7.0f); + output.writeFloat(8.0f); + output.writeLong(0, true); + output.writeFloat(8.0f); + output.writeFloat(9.0f); + output.writeLong(0, true); + output.writeLong(0, true); + output.writeLong(2, true); + output.close(); + + hist = new SimpleHistogram(42); + hist.fromHistogram(out.toByteArray(), false); + assertArrayEquals(out.toByteArray(), hist.histogram(false)); + } + + @Test + public void testHistogramSerialization() { + Kryo kryo = new Kryo(); + ByteArrayOutputStream outBuffer = new ByteArrayOutputStream(); + Output output = new Output(outBuffer); + + SimpleHistogram y1Hist = new SimpleHistogram(0); + y1Hist.addBucket(1.0f, 2.0f, 5L); + y1Hist.addBucket(2.0f, 3.0f, 5L); + y1Hist.addBucket(3.0f, 10.0f, 0L); + y1Hist.write(kryo, output); + output.close(); + + SimpleHistogram y1HistVerify = new SimpleHistogram(0); + y1HistVerify.fromHistogram(outBuffer.toByteArray(), false); + + Input input = new Input(new ByteArrayInputStream(y1HistVerify.histogram(false))); + int bucketCount = input.readShort(); + assertEquals(bucketCount, 3); + + Float bucketLB = input.readFloat(); + Float bucketUB = input.readFloat(); + long bucketVal = input.readLong(true); + assertEquals(bucketLB, 1.0f, 0.001); + assertEquals(bucketVal, 5); + input.close(); + } + + @Test + public void testIncompletByteArray() { + ByteArrayOutputStream outBuffer = new ByteArrayOutputStream(); + Output output = new Output(outBuffer); + output.writeShort(4); + output.close(); + + SimpleHistogram y1Hist = new SimpleHistogram(0); + boolean exceptionCaught = false; + try { + y1Hist.fromHistogram(outBuffer.toByteArray(), false); + } + catch(Exception e) { + exceptionCaught = true; + } + + assertTrue(exceptionCaught); + } + + @Test + public void testInvalidHistogramLength() { + Kryo kryo = new Kryo(); + ByteArrayOutputStream outBuffer = new ByteArrayOutputStream(); + Output output = new Output(outBuffer); + output.writeShort(1); + output.writeInt(-1); + output.close(); + Input input = new Input(new ByteArrayInputStream(outBuffer.toByteArray())); + Integer metricType = input.readInt(); + + SimpleHistogram y1Hist = new SimpleHistogram(0); + boolean exceptionCaught = false; + try { + y1Hist.read(kryo, input); + } + catch(Exception e) { + exceptionCaught = true; + } + + assertFalse(exceptionCaught); + } + + @Test + public void testSinglePercentile() { + ByteArrayOutputStream outBuffer = new ByteArrayOutputStream(); + Output output = new Output(outBuffer); + output.writeShort(3); + output.writeFloat(1.0f); + output.writeFloat(6.0f); + output.writeLong(5, true); + output.writeFloat(6.0f); + output.writeFloat(10.0f); + output.writeLong(10, true); + output.writeFloat(10.0f); + output.writeFloat(20.0f); + output.writeLong(1, true); + output.writeLong(0, true); + output.writeLong(5, true); + output.close(); + + SimpleHistogram y1Hist = new SimpleHistogram(0); + y1Hist.fromHistogram(outBuffer.toByteArray(), false); + double perc50 = y1Hist.percentile(50.0f); + + assertEquals(perc50, 8.0f, 0.0001); + assertEquals(y1Hist.percentile(1000.0f), -1.0f, 0.0001); + } + + @Test + public void testPercentileList() { + ByteArrayOutputStream outBuffer = new ByteArrayOutputStream(); + Output output = new Output(outBuffer); + output.writeShort(4); + output.writeFloat(1.0f); + output.writeFloat(6.0f); + output.writeLong(5, true); + output.writeFloat(6.0f); + output.writeFloat(10.0f); + output.writeLong(10, true); + output.writeFloat(10.0f); + output.writeFloat(20.0f); + output.writeLong(1, true); + output.writeFloat(20.0f); + output.writeFloat(40.0f); + output.writeLong(0, true); + output.writeLong(0, true); + output.writeLong(5, true); + output.close(); + + SimpleHistogram y1Hist = new SimpleHistogram(0); + y1Hist.fromHistogram(outBuffer.toByteArray(), false); + ArrayList<Double> percs = new ArrayList<Double>(); + percs.add(50.0); + percs.add(99.0); + ArrayList<Double> percValues = (ArrayList<Double>) y1Hist.percentiles(percs); + double perc50 = percValues.get(0); + double perc99 = percValues.get(1); + + assertEquals(perc50, 8.0, 0.001); + assertEquals(perc99, 15.0, 0.001); + } + + @Test + public void testSingleHistogramMerge() { + ByteArrayOutputStream outBuffer = new ByteArrayOutputStream(); + Output output = new Output(outBuffer); + output.writeShort(4); + output.writeFloat(1.0f); + output.writeFloat(6.0f); + output.writeLong(5, true); + output.writeFloat(6.0f); + output.writeFloat(10.0f); + output.writeLong(10, true); + output.writeFloat(10.0f); + output.writeFloat(20.0f); + output.writeLong(1, true); + output.writeFloat(20.0f); + output.writeFloat(40.0f); + output.writeLong(0, true); + output.writeLong(0, true); + output.writeLong(5, true); + output.close(); + + SimpleHistogram y1Hist = new SimpleHistogram(0); + y1Hist.fromHistogram(outBuffer.toByteArray(), false); + + SimpleHistogram y1Hist1 = new SimpleHistogram(0); + y1Hist1.fromHistogram(outBuffer.toByteArray(), false); + y1Hist1.setUnderflow(2L); + + y1Hist.aggregate(y1Hist1, HistogramAggregation.SUM); + assertEquals(y1Hist.getBucketCount(1.0f, 6.0f), Long.valueOf(10L)); + assertEquals(y1Hist.getBucketCount(6.0f, 10.0f), Long.valueOf(20L)); + assertEquals(y1Hist.getBucketCount(10.0f, 20.0f), Long.valueOf(2L)); + assertEquals(y1Hist.getOverflow(), Long.valueOf(10L)); + assertEquals(y1Hist.getUnderflow(), Long.valueOf(2L)); + } + + @Test + public void testMultipleHistogramMerge() { + ByteArrayOutputStream outBuffer = new ByteArrayOutputStream(); + Output output = new Output(outBuffer); + output.writeShort(4); + output.writeFloat(1.0f); + output.writeFloat(6.0f); + output.writeLong(5, true); + output.writeFloat(6.0f); + output.writeFloat(10.0f); + output.writeLong(10, true); + output.writeFloat(10.0f); + output.writeFloat(20.0f); + output.writeLong(1, true); + output.writeFloat(20.0f); + output.writeFloat(40.0f); + output.writeLong(0, true); + output.writeLong(0, true); + output.writeLong(5, true); + output.close(); + + SimpleHistogram y1Hist = new SimpleHistogram(0); + y1Hist.fromHistogram(outBuffer.toByteArray(), false); + + ArrayList<Histogram> histos = new ArrayList<Histogram>(); + SimpleHistogram y1Hist1 = new SimpleHistogram(0); + y1Hist1.fromHistogram(outBuffer.toByteArray(), false); + histos.add(y1Hist1); + SimpleHistogram y1Hist2 = new SimpleHistogram(0); + y1Hist2.fromHistogram(outBuffer.toByteArray(), false); + histos.add(y1Hist2); + + y1Hist.aggregate(histos, HistogramAggregation.SUM); + assertEquals(y1Hist.getBucketCount(1.0f, 6.0f), Long.valueOf(15L)); + assertEquals(y1Hist.getBucketCount(6.0f, 10.0f), Long.valueOf(30L)); + assertEquals(y1Hist.getBucketCount(10.0f, 20.0f), Long.valueOf(3L)); + assertEquals(y1Hist.getOverflow(), Long.valueOf(15L)); + } + + @Test + public void testArbitraryBuckets() { + ByteArrayOutputStream outBuffer = new ByteArrayOutputStream(); + Output output = new Output(outBuffer); + output.writeShort(4); + output.writeFloat(1.0f); + output.writeFloat(6.0f); + output.writeLong(5, true); + output.writeFloat(6.0f); + output.writeFloat(10.0f); + output.writeLong(10, true); + output.writeFloat(10.0f); + output.writeFloat(20.0f); + output.writeLong(1, true); + output.writeFloat(20.0f); + output.writeFloat(40.0f); + output.writeLong(0, true); + output.writeLong(0, true); + output.writeLong(5, true); + output.close(); + + SimpleHistogram y1Hist = new SimpleHistogram(0); + y1Hist.fromHistogram(outBuffer.toByteArray(), false); + + assertEquals(y1Hist.getBucketCount(6.0f, 10.0f), Long.valueOf(10L)); + assertEquals(y1Hist.getBucketCount(5.0f, 6.0f), Long.valueOf(0L)); + assertEquals(y1Hist.getBucketCount(5.0f, 12.0f), Long.valueOf(0L)); + assertEquals(y1Hist.getBucketCount(1.0f, 10.0f), Long.valueOf(0L)); + } + + @Test + public void testAddBuckets() { + SimpleHistogram y1Hist = new SimpleHistogram(0); + + y1Hist.addBucket(5.0f, 7.0f, 3L); + assertEquals(y1Hist.getBucketCount(5.0f, 7.0f), Long.valueOf(3L));; + + y1Hist.addBucket(5.0f, 7.0f, 5L); + assertEquals(y1Hist.getBucketCount(5.0f, 7.0f), Long.valueOf(5L)); + } + + @Test + public void testNullBucketVal() { + SimpleHistogram y1Hist = new SimpleHistogram(0); + + y1Hist.addBucket(5.0f, 7.0f, null); + assertEquals(y1Hist.getBucketCount(5.0f, 7.0f), Long.valueOf(0L)); + + y1Hist.addBucket(5.0f, 7.0f, 5L); + } + + @Test + public void testJsonSerialization() { + SimpleHistogram y1Hist = new SimpleHistogram(0); + + y1Hist.addBucket(5.0f, 7.0f, 3L); + y1Hist.addBucket(7.0f, 10.0f, 5L); + y1Hist.setOverflow(1L); + + String jsonOut = new String(); + ObjectMapper mapper = new ObjectMapper(); + try { + StringWriter output = new StringWriter(); + mapper.writeValue(output, y1Hist); + jsonOut = output.toString(); + } catch (IOException e) { + e.printStackTrace(); + } + assertTrue(jsonOut.contains("\"buckets\":{")); + assertTrue(jsonOut.contains("\"5.0-7.0\":3")); + assertTrue(jsonOut.contains("\"7.0-10.0\":5")); + assertTrue(jsonOut.contains("\"underflow\":0")); + assertTrue(jsonOut.contains("\"overflow\":1")); + + SimpleHistogram y1Hist1 = new SimpleHistogram(0); + + y1Hist1.addBucket(Float.NEGATIVE_INFINITY, 5.0f, 3L); + y1Hist1.addBucket(7.0f, 10.0f, 5L); + y1Hist1.addBucket(10.0f, null, 1L); + try { + StringWriter output = new StringWriter(); + mapper.writeValue(output, y1Hist1); + jsonOut = output.toString(); + } catch (IOException e) { + e.printStackTrace(); + } + assertTrue(jsonOut.contains("\"buckets\":{")); + assertTrue(jsonOut.contains("\"-Infinity-5.0\":3")); + assertTrue(jsonOut.contains("\"7.0-10.0\":5")); + assertTrue(jsonOut.contains("\"underflow\":0")); + assertTrue(jsonOut.contains("\"overflow\":0")); + } + + @Test + public void testImmutableGetHistogram() { + SimpleHistogram y1Hist = new SimpleHistogram(0); + + y1Hist.addBucket(5.0f, 7.0f, 3L); + y1Hist.addBucket(7.0f, 10.0f, 5L); + y1Hist.addBucket(10.0f, null, 1L); + + Map<HistogramBucket, Long> histMap = y1Hist.getHistogram(); + boolean histModBlocked = false; + try { + histMap.put(new HistogramBucket(BucketType.REGULAR, 1.0f, 5.0f), 3L); + } catch (UnsupportedOperationException e) { + histModBlocked = true; + } + + assertTrue(histModBlocked); + } + + @Test + public void testMissingBucketsHistogramAggregation() { + SimpleHistogram hist1 = new SimpleHistogram(0); + hist1.addBucket(5.0f, 7.0f, 3L); + hist1.addBucket(7.0f, 10.0f, 5L); + hist1.addBucket(15.0f, 20.0f, 2L); + + SimpleHistogram hist2 = new SimpleHistogram(0); + hist2.addBucket(5.0f, 7.0f, 3L); + hist2.addBucket(7.0f, 10.0f, 5L); + hist2.addBucket(10.0f, 15.0f, 2L); + hist2.addBucket(15.0f, 20.0f, 1L); + + hist1.aggregate(hist2, HistogramAggregation.SUM); + assertEquals(hist1.getBucketCount(10.0f, 15.0f).longValue(), 2L); + } + + @Test + public void testInit() { + double[] dynaHist = SimpleHistogram.initializeHistogram(1.0f, 6000.0f, + 100.0f, 2000.0f, 0.05f); + // TODO - proper bucket testing +// System.out.println("No. of buckets: " + dynaHist.length); +// for (int i = 0; i < dynaHist.length; i++) { +// System.out.print(dynaHist[i] + ", "); +// } +// System.out.println(); + assertEquals(33, dynaHist.length); + } + + @Test + public void testWholeRangeFocus() { + double[] dynaHist = SimpleHistogram.initializeHistogram(100.0f, 2000.0f, + 100.0f, 2000.0f, 0.05f); + // TODO - proper bucket testing +// System.out.println("No. of buckets: " + dynaHist.length); +// for (int i = 0; i < dynaHist.length; i++) { +// System.out.print(dynaHist[i] + ", "); +// } +// System.out.println(); + assertEquals(31, dynaHist.length); + } + + @Test + public void testStartEqualtoFocusStart() { + double[] dynaHist = SimpleHistogram.initializeHistogram(100.0f, 6000.0f, + 100.0f, 2000.0f, 0.05f); + // TODO - proper bucket testing +// System.out.println("No. of buckets: " + dynaHist.length); +// for (int i = 0; i < dynaHist.length; i++) { +// System.out.print(dynaHist[i] + ", "); +// } +// System.out.println(); + assertEquals(32, dynaHist.length); + } + + @Test + public void testEndEqualtoFocusEnd() { + double[] dynaHist = SimpleHistogram.initializeHistogram(1.0f, 2000.0f, + 100.0f, 2000.0f, 0.05f); + // TODO - proper bucket testing +// System.out.println("No. of buckets: " + dynaHist.length); +// for (int i = 0; i < dynaHist.length; i++) { +// System.out.print(dynaHist[i] + ", "); +// } + assertEquals(32, dynaHist.length); + } + + @Test (expected = IllegalArgumentException.class) + public void testErrorStartGreaterThanEnd() { + SimpleHistogram.initializeHistogram(10000.0f, 6000.0f, 100.0f, 2000.0f, 0.05f); + } + + @Test (expected = IllegalArgumentException.class) + public void testErrorFocusStartGreaterThanFocusEnd() { + SimpleHistogram.initializeHistogram(1.0f, 6000.0f, 3000.0f, 2000.0f, 0.05f); + } + + @Test (expected = IllegalArgumentException.class) + public void testErrorRateLessThanZero() { + SimpleHistogram.initializeHistogram(1.0f, 6000.0f, 100.0f, 2000.0f, -0.05f); + } + + @Test (expected = IllegalArgumentException.class) + public void testFocusEndGreaterThanEnd() { + SimpleHistogram.initializeHistogram(1.0f, 1000.0f, 100.0f, 2000.0f, 0.05f); + } + + @Test (expected = IllegalArgumentException.class) + public void testFocusStartLessThanStart() { + SimpleHistogram.initializeHistogram(200.0f, 100.0f, 1500.0f, 2000.0f, 0.05f); + } + + @Test (expected = IllegalArgumentException.class) + public void testExcessiveBuckets() { + SimpleHistogram.initializeHistogram(100.0f, 6000.0f, 100.0f, 6000.0f, 0.01f); + } +} \ No newline at end of file diff --git a/test/core/TestSpan.java b/test/core/TestSpan.java index 4b06626498..b9c1b668f3 100644 --- a/test/core/TestSpan.java +++ b/test/core/TestSpan.java @@ -12,8 +12,10 @@ // see <http://www.gnu.org/licenses/>. package net.opentsdb.core; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.when; import static org.powermock.api.mockito.PowerMockito.mock; @@ -25,9 +27,11 @@ import org.hbase.async.Bytes; import org.hbase.async.KeyValue; +import org.hbase.async.Bytes.ByteMap; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; @@ -43,7 +47,7 @@ "ch.qos.*", "org.slf4j.*", "com.sum.*", "org.xml.*"}) @PrepareForTest({ RowSeq.class, TSDB.class, UniqueId.class, KeyValue.class, -Config.class, RowKey.class }) + Config.class, RowKey.class, Const.class }) public final class TestSpan { private TSDB tsdb = mock(TSDB.class); private Config config = mock(Config.class); @@ -85,6 +89,31 @@ public void addRow() { assertEquals(2, span.size()); } + @Test + public void addRowSalted() { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(2); + + final byte[] qual1 = { 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); + + final byte[] hour1 = { 0, 0, 0, 1, 0x50, (byte)0xE2, 0x27, + 0, 0, 0, 1, 0, 0, 2 }; + final byte[] hour2 = { 1, 0, 0, 1, 0x50, (byte)0xE2, 0x35, + 0x10, 0, 0, 1, 0, 0, 2 }; + final Span span = new Span(tsdb); + span.addRow(new KeyValue(hour1, FAMILY, qual12, + MockBase.concatByteArrays(val1, val2, ZERO))); + span.addRow(new KeyValue(hour2, FAMILY, qual12, + MockBase.concatByteArrays(val1, val2, ZERO))); + + assertEquals(4, span.size()); + } + @Test (expected = NullPointerException.class) public void addRowNull() { final Span span = new Span(tsdb); @@ -331,7 +360,8 @@ public void downsampler() throws Exception { assertEquals(6, span.size()); long interval_ms = 1000000; Aggregator downsampler = Aggregators.get("avg"); - final SeekableView it = span.downsampler(interval_ms, downsampler); + final SeekableView it = span.downsampler(1356998000L, 1357007000L, + interval_ms, downsampler, FillPolicy.NONE); List<Double> values = Lists.newArrayList(); List<Long> timestamps_in_millis = Lists.newArrayList(); while (it.hasNext()) { @@ -381,4 +411,120 @@ public void lastTimestampInRowMs() throws Exception { assertEquals(1356998400008L, Span.lastTimestampInRow((short) 3, kv)); } + + @Test + public void metricUID() throws Exception { + final byte[] qual1 = { 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); + + final Span span = new Span(tsdb); + span.addRow(new KeyValue(HOUR1, FAMILY, qual12, + MockBase.concatByteArrays(val1, val2, ZERO))); + + assertEquals(2, span.size()); + + assertArrayEquals(new byte[] { 0, 0, 1 }, span.metricUID()); + } + + @Test + public void metricUIDSalted() throws Exception { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(2); + + final byte[] qual1 = { 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); + + final Span span = new Span(tsdb); + final byte[] key = new byte[HOUR1.length + 1]; + System.arraycopy(HOUR1, 0, key, 1, HOUR1.length); + span.addRow(new KeyValue(key, FAMILY, qual12, + MockBase.concatByteArrays(val1, val2, ZERO))); + + assertEquals(2, span.size()); + + assertArrayEquals(new byte[] { 0, 0, 1 }, span.metricUID()); + } + + @Test + public void getTagUids() { + final byte[] qual1 = { 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); + + final Span span = new Span(tsdb); + span.addRow(new KeyValue(HOUR1, FAMILY, qual12, + MockBase.concatByteArrays(val1, val2, ZERO))); + + assertEquals(2, span.size()); + final ByteMap<byte[]> uids = span.getTagUids(); + assertEquals(1, uids.size()); + assertArrayEquals(new byte[] { 0, 0, 1 }, uids.firstKey()); + assertArrayEquals(new byte[] { 0, 0, 2 }, + uids.firstEntry().getValue()); + } + + @Test + public void getTagUidsSalted() { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(2); + + final byte[] qual1 = { 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); + + final Span span = new Span(tsdb); + final byte[] key = new byte[HOUR1.length + 1]; + System.arraycopy(HOUR1, 0, key, 1, HOUR1.length); + span.addRow(new KeyValue(key, FAMILY, qual12, + MockBase.concatByteArrays(val1, val2, ZERO))); + + assertEquals(2, span.size()); + final ByteMap<byte[]> uids = span.getTagUids(); + assertEquals(1, uids.size()); + assertArrayEquals(new byte[] { 0, 0, 1 }, uids.firstKey()); + assertArrayEquals(new byte[] { 0, 0, 2 }, + uids.firstEntry().getValue()); + } + + @Test (expected = IllegalStateException.class) + public void getTagUidsNotSet() { + final Span span = new Span(tsdb); + span.getTagUids(); + } + + @Test + public void getAggregatedTagUids() { + final byte[] qual1 = { 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual12 = MockBase.concatByteArrays(qual1, qual2); + + final Span span = new Span(tsdb); + span.addRow(new KeyValue(HOUR1, FAMILY, qual12, + MockBase.concatByteArrays(val1, val2, ZERO))); + + assertEquals(2, span.size()); + final List<byte[]> uids = span.getAggregatedTagUids(); + assertEquals(0, uids.size()); + } + + @Test + public void getAggregatedTagUidsNotSet() { + final Span span = new Span(tsdb); + assertTrue(span.getAggregatedTagUids().isEmpty()); + } + } diff --git a/test/core/TestSpanGroup.java b/test/core/TestSpanGroup.java new file mode 100644 index 0000000000..f24458c4c9 --- /dev/null +++ b/test/core/TestSpanGroup.java @@ -0,0 +1,160 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.List; + +import net.opentsdb.utils.Config; + +import org.hbase.async.Bytes.ByteMap; +import org.hbase.async.HBaseClient; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({ TSDB.class, HBaseClient.class, Config.class, SpanGroup.class, + Span.class }) +public final class TestSpanGroup { + private static long start_ts = 1356998400L; + private static long end_ts = 1356998600L; + + private TSDB tsdb; + + @Before + public void before() { + tsdb = PowerMockito.mock(TSDB.class); + } + + @Test + public void metricUID() throws Exception { + final Span span = mock(Span.class); + when(span.metricUID()).thenReturn(new byte[] { 0, 0, 1 }); + + final SpanGroup group = PowerMockito.spy(new SpanGroup(tsdb, start_ts, + end_ts, null, false, Aggregators.SUM, 0, null)); + final ArrayList<Span> spans = Whitebox.getInternalState(group, "spans"); + spans.add(span); + + assertArrayEquals(new byte[] { 0, 0, 1 }, group.metricUID()); + } + + @Test + public void getTagUids() throws Exception { + final ByteMap<byte[]> uids = new ByteMap<byte[]>(); + uids.put(new byte[] { 0, 0, 1 }, new byte[] { 0, 0, 2 }); + final Span span = mock(Span.class); + when(span.getTagUids()).thenReturn(uids); + + final SpanGroup group = PowerMockito.spy(new SpanGroup(tsdb, start_ts, + end_ts, null, false, Aggregators.SUM, 0, null)); + final ArrayList<Span> spans = Whitebox.getInternalState(group, "spans"); + spans.add(span); + + final ByteMap<byte[]> uids_read = group.getTagUids(); + assertEquals(1, uids_read.size()); + assertArrayEquals(new byte[] { 0, 0, 1 }, uids_read.firstKey()); + assertArrayEquals(new byte[] { 0, 0, 2 }, + uids_read.firstEntry().getValue()); + } + + @Test + public void getTagUidsAggedOut() throws Exception { + final ByteMap<byte[]> uids = new ByteMap<byte[]>(); + uids.put(new byte[] { 0, 0, 1 }, new byte[] { 0, 0, 2 }); + final Span span = mock(Span.class); + when(span.getTagUids()).thenReturn(uids); + + final ByteMap<byte[]> uids2 = new ByteMap<byte[]>(); + uids2.put(new byte[] { 0, 0, 1 }, new byte[] { 0, 0, 3 }); + final Span span2 = mock(Span.class); + when(span2.getTagUids()).thenReturn(uids2); + + final SpanGroup group = PowerMockito.spy(new SpanGroup(tsdb, start_ts, + end_ts, null, false, Aggregators.SUM, 0, null)); + final ArrayList<Span> spans = Whitebox.getInternalState(group, "spans"); + spans.add(span); + spans.add(span2); + + final ByteMap<byte[]> uids_read = group.getTagUids(); + assertEquals(0, uids_read.size()); + } + + @Test + public void getTagUidsNoSpans() throws Exception { + final SpanGroup group = new SpanGroup(tsdb, start_ts, end_ts, null, + false, Aggregators.SUM, 0, null); + + final ByteMap<byte[]> uids_read = group.getTagUids(); + assertEquals(0, uids_read.size()); + } + + @Test + public void getAggregatedTagUidsNotAgged() throws Exception { + final ByteMap<byte[]> uids = new ByteMap<byte[]>(); + uids.put(new byte[] { 0, 0, 1 }, new byte[] { 0, 0, 2 }); + final Span span = mock(Span.class); + when(span.getTagUids()).thenReturn(uids); + + final SpanGroup group = PowerMockito.spy(new SpanGroup(tsdb, start_ts, + end_ts, null, false, Aggregators.SUM, 0, null)); + final ArrayList<Span> spans = Whitebox.getInternalState(group, "spans"); + spans.add(span); + + final List<byte[]> uids_read = group.getAggregatedTagUids(); + assertEquals(0, uids_read.size()); + } + + @Test + public void getAggregatedTagUids() throws Exception { + final ByteMap<byte[]> uids = new ByteMap<byte[]>(); + uids.put(new byte[] { 0, 0, 1 }, new byte[] { 0, 0, 2 }); + final Span span = mock(Span.class); + when(span.getTagUids()).thenReturn(uids); + + final ByteMap<byte[]> uids2 = new ByteMap<byte[]>(); + uids2.put(new byte[] { 0, 0, 1 }, new byte[] { 0, 0, 0, 3 }); + final Span span2 = mock(Span.class); + when(span2.getTagUids()).thenReturn(uids2); + + final SpanGroup group = PowerMockito.spy(new SpanGroup(tsdb, start_ts, + end_ts, null, false, Aggregators.SUM, 0, null)); + final ArrayList<Span> spans = Whitebox.getInternalState(group, "spans"); + spans.add(span); + spans.add(span2); + + final List<byte[]> uids_read = group.getAggregatedTagUids(); + assertEquals(1, uids_read.size()); + assertArrayEquals(new byte[] { 0, 0, 1 }, uids_read.get(0)); + } + + @Test + public void getAggregatedTagUidsNoSpans() throws Exception { + final SpanGroup group = new SpanGroup(tsdb, start_ts, end_ts, null, + false, Aggregators.SUM, 0, null); + + final List<byte[]> uids_read = group.getAggregatedTagUids(); + assertEquals(0, uids_read.size()); + } + +} diff --git a/test/core/TestSplitRollupQuery.java b/test/core/TestSplitRollupQuery.java new file mode 100644 index 0000000000..559aaa1717 --- /dev/null +++ b/test/core/TestSplitRollupQuery.java @@ -0,0 +1,329 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2012-2021 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import com.stumbleupon.async.Deferred; +import net.opentsdb.utils.DateTime; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +import java.util.*; + +import static org.junit.Assert.*; +import static org.mockito.Matchers.*; +import static org.mockito.Mockito.verify; +import static org.powermock.api.mockito.PowerMockito.*; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({DateTime.class, TsdbQuery.class}) +public class TestSplitRollupQuery extends BaseTsdbTest { + private SplitRollupQuery queryUnderTest; + private TsdbQuery rollupQuery; + + @Before + public void beforeLocal() { + rollupQuery = spy(new TsdbQuery(tsdb)); + queryUnderTest = new SplitRollupQuery(tsdb, rollupQuery, Deferred.fromResult(null)); + } + + @Test + public void setStartTime() { + queryUnderTest.setStartTime(42L); + assertEquals(42000L, queryUnderTest.getStartTime()); + assertEquals(42000L, rollupQuery.getStartTime()); + } + + @Test + public void setStartTimeBeyondOriginalEnd() { + TsdbQuery rawQuery = new TsdbQuery(tsdb); + Whitebox.setInternalState(queryUnderTest, "rawQuery", rawQuery); + + rollupQuery.setEndTime(41L); + queryUnderTest.setStartTime(42L); + + assertEquals(42000L, queryUnderTest.getStartTime()); + } + + @Test + public void setStartTimeWithoutRollupQuery() { + TsdbQuery rawQuery = new TsdbQuery(tsdb); + Whitebox.setInternalState(queryUnderTest, "rollupQuery", (TsdbQuery)null); + Whitebox.setInternalState(queryUnderTest, "rawQuery", rawQuery); + + queryUnderTest.setStartTime(42L); + + assertEquals(42000L, queryUnderTest.getStartTime()); + } + + @Test + public void setEndTime() { + TsdbQuery rawQuery = new TsdbQuery(tsdb); + Whitebox.setInternalState(queryUnderTest, "rawQuery", rawQuery); + + rollupQuery.setStartTime(0); + rollupQuery.setEndTime(21000L); + rawQuery.setStartTime(21000L); + + queryUnderTest.setEndTime(42L); + + assertEquals(42000L, queryUnderTest.getEndTime()); + assertEquals(21000L, rollupQuery.getEndTime()); + assertEquals(42000L, rawQuery.getEndTime()); + } + + @Test(expected = IllegalArgumentException.class) + public void setEndTimeBeforeRawStartTime() { + TsdbQuery rawQuery = new TsdbQuery(tsdb); + rawQuery.setStartTime(DateTime.currentTimeMillis()); + + Whitebox.setInternalState(queryUnderTest, "rawQuery", rawQuery); + + queryUnderTest.setEndTime(42L); + } + + @Test + public void setDeletePassesThroughToBoth() { + TsdbQuery rawQuery = spy(new TsdbQuery(tsdb)); + Whitebox.setInternalState(queryUnderTest, "rawQuery", rawQuery); + + queryUnderTest.setDelete(true); + + verify(rollupQuery).setDelete(true); + verify(rawQuery).setDelete(true); + } + + @Test + public void setTimeSeriesPassesThroughToBoth() { + TsdbQuery rawQuery = spy(new TsdbQuery(tsdb)); + Whitebox.setInternalState(queryUnderTest, "rawQuery", rawQuery); + + RateOptions options = new RateOptions(); + + queryUnderTest.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false, options); + + verify(rollupQuery).setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false, options); + verify(rawQuery).setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false, options); + } + + @Test + public void setTimeSeriesWithoutRateOptionsPassesThroughToBoth() { + TsdbQuery rawQuery = spy(new TsdbQuery(tsdb)); + Whitebox.setInternalState(queryUnderTest, "rawQuery", rawQuery); + + queryUnderTest.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + + verify(rollupQuery).setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + verify(rawQuery).setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + } + + @Test + public void setTimeSeriesWithTSUIDsPassesThroughToBoth() { + TsdbQuery rawQuery = spy(new TsdbQuery(tsdb)); + Whitebox.setInternalState(queryUnderTest, "rawQuery", rawQuery); + + List<String> tsuids = Arrays.asList("000001000001000001", "000001000001000002"); + RateOptions options = new RateOptions(); + + queryUnderTest.setTimeSeries(tsuids, Aggregators.SUM, false, options); + + verify(rollupQuery).setTimeSeries(tsuids, Aggregators.SUM, false, options); + verify(rawQuery).setTimeSeries(tsuids, Aggregators.SUM, false, options); + } + + @Test + public void setTimeSeriesWithTSUIDsWithoutRateOptionsPassesThroughToBoth() { + TsdbQuery rawQuery = spy(new TsdbQuery(tsdb)); + Whitebox.setInternalState(queryUnderTest, "rawQuery", rawQuery); + + List<String> tsuids = Arrays.asList("000001000001000001", "000001000001000002"); + + queryUnderTest.setTimeSeries(tsuids, Aggregators.SUM, false); + + verify(rollupQuery).setTimeSeries(tsuids, Aggregators.SUM, false); + verify(rawQuery).setTimeSeries(tsuids, Aggregators.SUM, false); + } + + @Test + public void downsamplePassesThroughToBoth() { + TsdbQuery rawQuery = spy(new TsdbQuery(tsdb)); + Whitebox.setInternalState(queryUnderTest, "rawQuery", rawQuery); + + queryUnderTest.downsample(42L, Aggregators.SUM, FillPolicy.ZERO); + + verify(rollupQuery).downsample(42L, Aggregators.SUM, FillPolicy.ZERO); + verify(rawQuery).downsample(42L, Aggregators.SUM, FillPolicy.ZERO); + } + + @Test + public void downsampleWithoutFillPolicyPassesThroughToBoth() { + TsdbQuery rawQuery = spy(new TsdbQuery(tsdb)); + Whitebox.setInternalState(queryUnderTest, "rawQuery", rawQuery); + + queryUnderTest.downsample(42L, Aggregators.SUM); + + verify(rollupQuery).downsample(42L, Aggregators.SUM); + verify(rawQuery).downsample(42L, Aggregators.SUM); + } + + @Test(expected = UnsupportedOperationException.class) + public void configureFromQueryThrowsIfForcedRaw() { + queryUnderTest.configureFromQuery(null, 0, true); + } + + @Test + public void configureFromQuerySplitsRollupQuery() { + mockEnableRollupQuerySplitting(); + doReturn(Deferred.fromResult(null)).when(rollupQuery).split(any(TSQuery.class), anyInt(), any(TsdbQuery.class)); + + assertNull(Whitebox.getInternalState(queryUnderTest, "rawQuery")); + + rollupQuery.setStartTime(0); + queryUnderTest.configureFromQuery(null, 0, false); + + verify(rollupQuery).split(eq((TSQuery) null), eq(0), any(TsdbQuery.class)); + assertNotNull(Whitebox.getInternalState(queryUnderTest, "rawQuery")); + } + + @Test + public void configureFromQuerySplitsRollupQueryWithRawOnlyQuery() { + mockEnableRollupQuerySplitting(); + doReturn(true).when(rollupQuery).needsSplitting(); + doReturn(Deferred.fromResult(null)).when(rollupQuery).split(any(TSQuery.class), anyInt(), any(TsdbQuery.class)); + + rollupQuery.setStartTime(DateTime.currentTimeMillis()); + + assertNull(Whitebox.getInternalState(queryUnderTest, "rawQuery")); + + queryUnderTest.configureFromQuery(null, 0, false); + + verify(rollupQuery).split(eq((TSQuery) null), eq(0), any(TsdbQuery.class)); + assertNotNull(Whitebox.getInternalState(queryUnderTest, "rawQuery")); + assertNull(Whitebox.getInternalState(queryUnderTest, "rollupQuery")); + } + + @Test + public void setPercentiles() { + TsdbQuery rawQuery = spy(new TsdbQuery(tsdb)); + Whitebox.setInternalState(queryUnderTest, "rawQuery", rawQuery); + + List<Float> percentiles = Arrays.asList(50f, 75f, 99f, 99.9f); + + queryUnderTest.setPercentiles(percentiles); + + verify(rollupQuery).setPercentiles(percentiles); + verify(rawQuery).setPercentiles(percentiles); + } + + @Test + public void run() { + TsdbQuery rawQuery = spy(new TsdbQuery(tsdb)); + Whitebox.setInternalState(queryUnderTest, "rawQuery", rawQuery); + + doReturn(Deferred.fromResult(new DataPoints[0])).when(rollupQuery).runAsync(); + doReturn(Deferred.fromResult(new DataPoints[0])).when(rawQuery).runAsync(); + + DataPoints[] actualPoints = queryUnderTest.run(); + + verify(rollupQuery).runAsync(); + verify(rawQuery).runAsync(); + + assertEquals(0, actualPoints.length); + } + + @Test + public void runHistogram() { + TsdbQuery rawQuery = spy(new TsdbQuery(tsdb)); + Whitebox.setInternalState(queryUnderTest, "rawQuery", rawQuery); + + doReturn(true).when(rollupQuery).isHistogramQuery(); + doReturn(Deferred.fromResult(new DataPoints[0])).when(rollupQuery).runHistogramAsync(); + doReturn(true).when(rawQuery).isHistogramQuery(); + doReturn(Deferred.fromResult(new DataPoints[0])).when(rawQuery).runHistogramAsync(); + + DataPoints[] actualPoints = queryUnderTest.runHistogram(); + + verify(rollupQuery).runHistogramAsync(); + verify(rawQuery).runHistogramAsync(); + + assertEquals(0, actualPoints.length); + } + + @Test + public void runAsyncMergesResults() { + TsdbQuery rawQuery = spy(new TsdbQuery(tsdb)); + Whitebox.setInternalState(queryUnderTest, "rawQuery", rawQuery); + + rollupQuery.setStartTime(0); + rollupQuery.setEndTime(21); + rawQuery.setStartTime(21); + rawQuery.setEndTime(42); + + DataPoints[] rollupDataPoints = new DataPoints[] { + makeSpanGroup("group1"), + makeSpanGroup("group2"), + makeSpanGroup("group3"), + }; + + DataPoints[] rawDataPoints = new DataPoints[] { + makeSpanGroup("group2"), + makeSpanGroup("group3"), + makeSpanGroup("group4"), + makeSpanGroup("group5"), + }; + + doReturn(Deferred.fromResult(rollupDataPoints)).when(rollupQuery).runAsync(); + doReturn(Deferred.fromResult(rawDataPoints)).when(rawQuery).runAsync(); + + DataPoints[] actualPoints = queryUnderTest.run(); + + verify(rollupQuery).runAsync(); + verify(rawQuery).runAsync(); + + List<String> actualGroups = new ArrayList<String>(actualPoints.length); + for (DataPoints dataPoints : actualPoints) { + actualGroups.add(new String(((SplitRollupSpanGroup) dataPoints).group())); + } + Collections.sort(actualGroups); + + assertEquals(Arrays.asList("group1", "group2", "group3", "group4", "group5"), actualGroups); + } + + private SpanGroup makeSpanGroup(String group) { + + return new SpanGroup( + tsdb, + 0, + 42, + new ArrayList<Span>(), + false, + new RateOptions(), + Aggregators.SUM, + DownsamplingSpecification.NO_DOWNSAMPLER, + 0, + 42, + 0, + null, + group.getBytes() + ); + } + + private void mockEnableRollupQuerySplitting() { + Whitebox.setInternalState(tsdb, "rollups_split_queries", true); + Whitebox.setInternalState(rollupQuery, "rollup_query", makeRollupQuery()); + } +} diff --git a/test/core/TestSplitRollupSpanGroup.java b/test/core/TestSplitRollupSpanGroup.java new file mode 100644 index 0000000000..48dbf41911 --- /dev/null +++ b/test/core/TestSplitRollupSpanGroup.java @@ -0,0 +1,194 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import net.opentsdb.rollup.RollupSpan; +import net.opentsdb.utils.Config; +import org.hbase.async.Bytes; +import org.hbase.async.HBaseClient; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({TSDB.class, HBaseClient.class, Config.class, SpanGroup.class, + Span.class, RollupSpan.class}) +public class TestSplitRollupSpanGroup { + private final static long START_TS = 1356998400L; + private final static long SPLIT_TS = 1356998500L; + private final static long END_TS = 1356998600L; + + private TSDB tsdb; + private SpanGroup rollupSpanGroup; + private SpanGroup rawSpanGroup; + + @Before + public void before() { + rawSpanGroup = PowerMockito.spy(new SpanGroup(tsdb, START_TS, SPLIT_TS, null, false, Aggregators.SUM, 0, null)); + rollupSpanGroup = PowerMockito.spy(new SpanGroup(tsdb, SPLIT_TS, END_TS, null, false, Aggregators.SUM, 0, null)); + + doAnswer(new SeekableViewAnswer(START_TS, 100, 1, true)).when(rollupSpanGroup).iterator(); + doAnswer(new SeekableViewAnswer(SPLIT_TS, 100, 2, true)).when(rawSpanGroup).iterator(); + + tsdb = PowerMockito.mock(TSDB.class); + } + + @Test + public void testConstructorFiltersNullSpanGroups() { + SplitRollupSpanGroup group = new SplitRollupSpanGroup(null, rawSpanGroup); + final ArrayList<SpanGroup> actual = Whitebox.getInternalState(group, "spanGroups"); + assertEquals(1, actual.size()); + } + + @Test(expected = IllegalArgumentException.class) + public void testConstructorThrowsWhenAllNull() { + new SplitRollupSpanGroup(null, null); + } + + @Test + public void testMetricName() { + when(rollupSpanGroup.metricName()).thenReturn("metric name"); + assertEquals("metric name", (new SplitRollupSpanGroup(rollupSpanGroup, rawSpanGroup)).metricName()); + verifyZeroInteractions(rawSpanGroup); + } + + @Test + public void testMetricUID() { + when(rollupSpanGroup.metricUID()).thenReturn(new byte[]{0, 0, 1}); + assertArrayEquals(new byte[]{0, 0, 1}, (new SplitRollupSpanGroup(rollupSpanGroup, rawSpanGroup)).metricUID()); + verifyZeroInteractions(rawSpanGroup); + } + + @Test + public void testSize() { + when(rollupSpanGroup.size()).thenReturn(5); + when(rawSpanGroup.size()).thenReturn(7); + assertEquals(12, (new SplitRollupSpanGroup(rollupSpanGroup, rawSpanGroup)).size()); + } + + @Test + public void testAggregatedSize() { + when(rollupSpanGroup.aggregatedSize()).thenReturn(5); + when(rawSpanGroup.aggregatedSize()).thenReturn(7); + assertEquals(12, (new SplitRollupSpanGroup(rollupSpanGroup, rawSpanGroup)).aggregatedSize()); + } + + @Test + public void testGetTagUids() { + final Bytes.ByteMap<byte[]> uids1 = new Bytes.ByteMap<byte[]>(); + uids1.put(new byte[]{0, 0, 1}, new byte[]{0, 0, 2}); + final Bytes.ByteMap<byte[]> uids2 = new Bytes.ByteMap<byte[]>(); + uids2.put(new byte[]{0, 0, 3}, new byte[]{0, 0, 4}); + + when(rollupSpanGroup.getTagUids()).thenReturn(uids1); + when(rawSpanGroup.getTagUids()).thenReturn(uids2); + + Bytes.ByteMap<byte[]> actual = (new SplitRollupSpanGroup(rollupSpanGroup, rawSpanGroup)).getTagUids(); + + assertEquals(2, actual.size()); + assertArrayEquals(new byte[]{0, 0, 1}, actual.firstKey()); + assertArrayEquals(new byte[]{0, 0, 2}, actual.firstEntry().getValue()); + assertArrayEquals(new byte[]{0, 0, 3}, actual.lastKey()); + assertArrayEquals(new byte[]{0, 0, 4}, actual.lastEntry().getValue()); + } + + @Test + public void testGetAggregatedTagUids() { + final List<byte[]> uids1 = new ArrayList<byte[]>(); + uids1.add(new byte[]{0, 0, 1}); + final List<byte[]> uids2 = new ArrayList<byte[]>(); + uids2.add(new byte[]{0, 0, 2}); + + when(rollupSpanGroup.getAggregatedTagUids()).thenReturn(uids1); + when(rawSpanGroup.getAggregatedTagUids()).thenReturn(uids2); + + List<byte[]> actual = (new SplitRollupSpanGroup(rollupSpanGroup, rawSpanGroup)).getAggregatedTagUids(); + + assertEquals(2, actual.size()); + assertArrayEquals(new byte[]{0, 0, 1}, actual.get(0)); + assertArrayEquals(new byte[]{0, 0, 2}, actual.get(1)); + } + + @Test + public void testIterator() { + SeekableView iterator = (new SplitRollupSpanGroup(rollupSpanGroup, rawSpanGroup)).iterator(); + int size = 0; + while (iterator.hasNext()) { + size++; + iterator.next(); + } + assertEquals(3, size); + } + + @Test + public void testTimestamp() { + SplitRollupSpanGroup group = new SplitRollupSpanGroup(rollupSpanGroup, rawSpanGroup); + assertEquals(START_TS, group.timestamp(0)); + assertEquals(SPLIT_TS, group.timestamp(1)); + assertEquals(END_TS, group.timestamp(2)); + } + + @Test + public void testIsInteger() { + assertTrue(new SplitRollupSpanGroup(rollupSpanGroup).isInteger(0)); + } + + @Test + public void testLongValue() { + assertEquals(0L, new SplitRollupSpanGroup(rollupSpanGroup).longValue(0)); + } + + @Test + public void testDoubleValue() { + doAnswer(new SeekableViewAnswer(START_TS, 100, 1, false)).when(rollupSpanGroup).iterator(); + assertEquals(0, new SplitRollupSpanGroup(rollupSpanGroup).doubleValue(0), 0.0); + } + + @Test + public void testGroup() { + when(rollupSpanGroup.metricName()).thenReturn("group name"); + assertEquals("group name", (new SplitRollupSpanGroup(rollupSpanGroup, rawSpanGroup)).metricName()); + verifyZeroInteractions(rawSpanGroup); + } + + static class SeekableViewAnswer implements Answer<SeekableView> { + private final long timestamp; + private final int samplePeriod; + private final int numPoints; + private final boolean isInteger; + + SeekableViewAnswer(long startTimestamp, int samplePeriod, int numPoints, boolean isInteger) { + this.timestamp = startTimestamp; + this.samplePeriod = samplePeriod; + this.numPoints = numPoints; + this.isInteger = isInteger; + } + + @Override + public SeekableView answer(InvocationOnMock ignored) { + return SeekableViewsForTest.generator(timestamp, samplePeriod, numPoints, isInteger); + } + } +} diff --git a/test/core/TestTSDB.java b/test/core/TestTSDB.java index 0d9ede48b2..3bf1a01705 100644 --- a/test/core/TestTSDB.java +++ b/test/core/TestTSDB.java @@ -15,14 +15,13 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.mockito.Mockito.when; -import static org.powermock.api.mockito.PowerMockito.mock; import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.when; +import java.io.File; import java.lang.reflect.Field; import java.util.HashMap; -import java.util.Map; import net.opentsdb.storage.MockBase; import net.opentsdb.uid.NoSuchUniqueId; @@ -32,7 +31,6 @@ import net.opentsdb.utils.Config; import org.hbase.async.AtomicIncrementRequest; -import org.hbase.async.Bytes; import org.hbase.async.GetRequest; import org.hbase.async.HBaseClient; import org.hbase.async.KeyValue; @@ -41,57 +39,93 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import com.google.common.io.Files; import com.stumbleupon.async.Deferred; @RunWith(PowerMockRunner.class) @PowerMockIgnore({"javax.management.*", "javax.xml.*", "ch.qos.*", "org.slf4j.*", "com.sum.*", "org.xml.*"}) -@PrepareForTest({TSDB.class, Config.class, UniqueId.class, HBaseClient.class, +@PrepareForTest({ TSDB.class, Config.class, UniqueId.class, HBaseClient.class, CompactionQueue.class, GetRequest.class, PutRequest.class, KeyValue.class, - Scanner.class, AtomicIncrementRequest.class}) -public final class TestTSDB { - private Config config; - private TSDB tsdb; - private HBaseClient client = mock(HBaseClient.class); - private UniqueId metrics = mock(UniqueId.class); - private UniqueId tag_names = mock(UniqueId.class); - private UniqueId tag_values = mock(UniqueId.class); - private CompactionQueue compactionq = mock(CompactionQueue.class); - private MockBase storage; + Scanner.class, AtomicIncrementRequest.class, Const.class, Files.class }) +public final class TestTSDB extends BaseTsdbTest { @Before - public void before() throws Exception { - PowerMockito.whenNew(HBaseClient.class) - .withArguments(anyString(), anyString()).thenReturn(client); - config = new Config(false); + public void beforeLocal() throws Exception { config.setFixDuplicates(true); // TODO(jat): test both ways - tsdb = new TSDB(config); - - Field met = tsdb.getClass().getDeclaredField("metrics"); - met.setAccessible(true); - met.set(tsdb, metrics); + } + + @Test + public void ctorNullClient() throws Exception { + assertNotNull(new TSDB(null, config)); + } + + @Test (expected = NullPointerException.class) + public void ctorNullConfig() throws Exception { + new TSDB(client, null); + } + + @Test + public void ctorOverrideUIDWidths() throws Exception { + // assert defaults + assertEquals(3, TSDB.metrics_width()); + assertEquals(3, TSDB.tagk_width()); + assertEquals(3, TSDB.tagv_width()); + + config.overrideConfig("tsd.storage.uid.width.metric", "1"); + config.overrideConfig("tsd.storage.uid.width.tagk", "4"); + config.overrideConfig("tsd.storage.uid.width.tagv", "5"); + final TSDB tsdb = new TSDB(client, config); + assertEquals(1, TSDB.metrics_width()); + assertEquals(4, TSDB.tagk_width()); + assertEquals(5, TSDB.tagv_width()); + assertEquals(1, tsdb.metrics.width()); + assertEquals(4, tsdb.tag_names.width()); + assertEquals(5, tsdb.tag_values.width()); - Field tagk = tsdb.getClass().getDeclaredField("tag_names"); - tagk.setAccessible(true); - tagk.set(tsdb, tag_names); + // IMPORTANT Restore + config.overrideConfig("tsd.storage.uid.width.metric", "3"); + config.overrideConfig("tsd.storage.uid.width.tagk", "3"); + config.overrideConfig("tsd.storage.uid.width.tagv", "3"); + new TSDB(client, config); + } + + @Test + public void ctorOverrideMaxNumTags() throws Exception { + assertEquals(8, Const.MAX_NUM_TAGS()); + + config.overrideConfig("tsd.storage.max_tags", "12"); + new TSDB(client, config); + assertEquals(12, Const.MAX_NUM_TAGS()); + + // IMPORTANT Restore + config.overrideConfig("tsd.storage.max_tags", "8"); + new TSDB(client, config); + } + + @Test + public void ctorOverrideSalt() throws Exception { + assertEquals(20, Const.SALT_BUCKETS()); + assertEquals(0, Const.SALT_WIDTH()); - Field tagv = tsdb.getClass().getDeclaredField("tag_values"); - tagv.setAccessible(true); - tagv.set(tsdb, tag_values); + config.overrideConfig("tsd.storage.salt.buckets", "15"); + config.overrideConfig("tsd.storage.salt.width", "2"); + new TSDB(client, config); + assertEquals(15, Const.SALT_BUCKETS()); + assertEquals(2, Const.SALT_WIDTH()); - Field cq = tsdb.getClass().getDeclaredField("compactionq"); - cq.setAccessible(true); - cq.set(tsdb, compactionq); + // IMPORTANT Restore + config.overrideConfig("tsd.storage.salt.buckets", "20"); + config.overrideConfig("tsd.storage.salt.width", "0"); + new TSDB(client, config); } - + @Test public void initializePluginsDefaults() { // no configured plugin path, plugins disabled, no exceptions @@ -151,6 +185,103 @@ public void initializePluginsSearchNotFound() throws Exception { tsdb.initializePlugins(true); } + @Test + public void initializePluginsSEH() throws Exception { + config.overrideConfig("tsd.core.plugin_path", "./"); + config.overrideConfig("tsd.core.storage_exception_handler.enable", "true"); + config.overrideConfig("tsd.core.storage_exception_handler.plugin", + "net.opentsdb.tsd.DummySEHPlugin"); + config.overrideConfig( + "tsd.core.storage_exception_handler.DummySEHPlugin.hosts", "localhost"); + tsdb.initializePlugins(true); + assertNotNull(tsdb.getStorageExceptionHandler()); + } + + @Test (expected = RuntimeException.class) + public void initializePluginsSEHBadConfig() throws Exception { + config.overrideConfig("tsd.core.plugin_path", "./"); + config.overrideConfig("tsd.core.storage_exception_handler.enable", "true"); + config.overrideConfig("tsd.core.storage_exception_handler.plugin", + "net.opentsdb.tsd.DummySEHPlugin"); + tsdb.initializePlugins(true); + assertNotNull(tsdb.getStorageExceptionHandler()); + } + + @Test (expected = NullPointerException.class) + public void initializePluginsSEHEnabledButNoName() throws Exception { + config.overrideConfig("tsd.core.plugin_path", "./"); + config.overrideConfig("tsd.core.storage_exception_handler.enable", "true"); + tsdb.initializePlugins(true); + } + + @Test (expected = IllegalArgumentException.class) + public void initializePluginsSEHNotFound() throws Exception { + config.overrideConfig("tsd.core.plugin_path", "./"); + config.overrideConfig("tsd.core.storage_exception_handler.enable", "true"); + config.overrideConfig("tsd.core.storage_exception_handler.plugin", + "net.opentsdb.tsd.DoesNotExistSEHPlugin"); + config.overrideConfig( + "tsd.core.storage_exception_handler.DummySEHPlugin.hosts", "localhost"); + tsdb.initializePlugins(true); + } + + @Test + public void loadRollupConfig() throws Exception { + config.overrideConfig("tsd.rollups.enable", "true"); + config.overrideConfig("tsd.rollups.config", + "{\"intervals\":[{\"interval\":\"1m\",\"table\":\"tsdb\"," + + "\"preAggregationTable\":\"tsdb\",\"defaultInterval\":true," + + "\"rowSpan\":\"1h\"},{\"interval\":\"10m\",\"table\":" + + "\"tsdb-rollup-10m\",\"preAggregationTable\":\"tsdb-rollup-agg-10m\"," + + "\"defaultInterval\":false,\"rowSpan\":\"1d\"}],\"aggregationIds\":" + + "{\"sum\":0,\"max\":1}}"); + tsdb = new TSDB(config); + assertEquals(2, tsdb.getRollupConfig().getRollups().size()); + assertEquals("sum", tsdb.getRollupConfig().getAggregatorForId(0)); + assertEquals("max", tsdb.getRollupConfig().getAggregatorForId(1)); + } + + @Test + public void loadRollupConfigFile() throws Exception { + config.overrideConfig("tsd.rollups.enable", "true"); + config.overrideConfig("tsd.rollups.config", "nosuchfile.json"); + PowerMockito.mockStatic(Files.class); + when(Files.toString(any(File.class), eq(Const.UTF8_CHARSET))).thenReturn( + "{\"intervals\":[{\"interval\":\"1m\",\"table\":\"tsdb\"," + + "\"preAggregationTable\":\"tsdb\",\"defaultInterval\":true," + + "\"rowSpan\":\"1h\"},{\"interval\":\"10m\",\"table\":" + + "\"tsdb-rollup-10m\",\"preAggregationTable\":\"tsdb-rollup-agg-10m\"," + + "\"defaultInterval\":false,\"rowSpan\":\"1d\"}],\"aggregationIds\":" + + "{\"sum\":0,\"max\":1}}"); + tsdb = new TSDB(config); + assertEquals(2, tsdb.getRollupConfig().getRollups().size()); + assertEquals("sum", tsdb.getRollupConfig().getAggregatorForId(0)); + assertEquals("max", tsdb.getRollupConfig().getAggregatorForId(1)); + } + + @Test (expected = IllegalArgumentException.class) + public void loadRollupConfigFileCorruptJson() throws Exception { + config.overrideConfig("tsd.rollups.enable", "true"); + config.overrideConfig("tsd.rollups.config", "nosuchfile.json"); + PowerMockito.mockStatic(Files.class); + when(Files.toString(any(File.class), eq(Const.UTF8_CHARSET))).thenReturn( + "{\"intervals\":[{\"interval\":\"1m\",\"ta"); + new TSDB(config); + } + + @Test (expected = IllegalArgumentException.class) + public void loadRollupConfigNoDefault() throws Exception { + config.overrideConfig("tsd.rollups.enable", "true"); + config.overrideConfig("tsd.rollups.config", + "{\"intervals\":[{\"interval\":\"1m\",\"table\":\"tsdb\"," + + "\"preAggregationTable\":\"tsdb\",\"defaultInterval\":false," + + "\"rowSpan\":\"1h\"},{\"interval\":\"10m\",\"table\":" + + "\"tsdb-rollup-10m\",\"preAggregationTable\":\"tsdb-rollup-agg-10m\"," + + "\"defaultInterval\":false,\"rowSpan\":\"1d\"}],\"aggregationIds\":" + + "{\"sum\":0,\"max\":1}}"); + tsdb = new TSDB(config); + } + @Test public void getClient() { assertNotNull(tsdb.getClient()); @@ -217,621 +348,169 @@ public void getUidNameNullUID() throws Exception { @Test public void getUIDMetric() { - setupAssignUid(); assertArrayEquals(new byte[] { 0, 0, 1 }, - tsdb.getUID(UniqueIdType.METRIC, "sys.cpu.0")); + tsdb.getUID(UniqueIdType.METRIC, METRIC_STRING)); } @Test public void getUIDTagk() { - setupAssignUid(); assertArrayEquals(new byte[] { 0, 0, 1 }, - tsdb.getUID(UniqueIdType.TAGK, "host")); + tsdb.getUID(UniqueIdType.TAGK, TAGK_STRING)); } @Test public void getUIDTagv() { - setupAssignUid(); assertArrayEquals(new byte[] { 0, 0, 1 }, - tsdb.getUID(UniqueIdType.TAGV, "localhost")); + tsdb.getUID(UniqueIdType.TAGV, TAGV_STRING)); } @Test (expected = NoSuchUniqueName.class) public void getUIDMetricNSU() { - setupAssignUid(); - tsdb.getUID(UniqueIdType.METRIC, "sys.cpu.1"); + tsdb.getUID(UniqueIdType.METRIC, NSUN_METRIC); } @Test (expected = NoSuchUniqueName.class) public void getUIDTagkNSU() { - setupAssignUid(); - tsdb.getUID(UniqueIdType.TAGK, "datacenter"); + tsdb.getUID(UniqueIdType.TAGK, NSUN_TAGK); } @Test (expected = NoSuchUniqueName.class) public void getUIDTagvNSU() { - setupAssignUid(); - tsdb.getUID(UniqueIdType.TAGV, "myserver"); + tsdb.getUID(UniqueIdType.TAGV, NSUN_TAGV); } - @Test (expected = NullPointerException.class) + @Test (expected = RuntimeException.class) public void getUIDNullType() { - setupAssignUid(); - tsdb.getUID(null, "sys.cpu.1"); + tsdb.getUID(null, METRIC_STRING); } @Test (expected = IllegalArgumentException.class) public void getUIDNullName() { - setupAssignUid(); tsdb.getUID(UniqueIdType.TAGV, null); } @Test (expected = IllegalArgumentException.class) public void getUIDEmptyName() { - setupAssignUid(); tsdb.getUID(UniqueIdType.TAGV, ""); } @Test public void assignUidMetric() { - setupAssignUid(); + when(metrics.getId("sys.cpu.1")).thenThrow( + new NoSuchUniqueName("metric", "sys.cpu.1")); + when(metrics.getOrCreateId("sys.cpu.1")) + .thenReturn(new byte[] { 0, 0, 2 }); assertArrayEquals(new byte[] { 0, 0, 2 }, tsdb.assignUid("metric", "sys.cpu.1")); } @Test (expected = IllegalArgumentException.class) public void assignUidMetricExists() { - setupAssignUid(); - tsdb.assignUid("metric", "sys.cpu.0"); + tsdb.assignUid("metric", METRIC_STRING); } @Test public void assignUidTagk() { - setupAssignUid(); + when(tag_names.getId("datacenter")).thenThrow( + new NoSuchUniqueName("tagk", "datacenter")); + when(tag_names.getOrCreateId("datacenter")) + .thenReturn(new byte[] { 0, 0, 2 }); assertArrayEquals(new byte[] { 0, 0, 2 }, tsdb.assignUid("tagk", "datacenter")); } @Test (expected = IllegalArgumentException.class) public void assignUidTagkExists() { - setupAssignUid(); - tsdb.assignUid("tagk", "host"); + tsdb.assignUid("tagk", TAGK_STRING); } @Test public void assignUidTagv() { - setupAssignUid(); + when(tag_values.getId("localhost")).thenThrow( + new NoSuchUniqueName("tagv", "localhost")); + when(tag_values.getOrCreateId("localhost")) + .thenReturn(new byte[] { 0, 0, 2 }); assertArrayEquals(new byte[] { 0, 0, 2 }, - tsdb.assignUid("tagv", "myserver")); + tsdb.assignUid("tagv", "localhost")); } @Test (expected = IllegalArgumentException.class) public void assignUidTagvExists() { - setupAssignUid(); - tsdb.assignUid("tagv", "localhost"); + tsdb.assignUid("tagv", TAGV_STRING); } @Test (expected = IllegalArgumentException.class) public void assignUidBadType() { - setupAssignUid(); - tsdb.assignUid("nothere", "localhost"); + tsdb.assignUid("nothere", METRIC_STRING); } @Test (expected = NullPointerException.class) public void assignUidNullType() { - setupAssignUid(); - tsdb.assignUid(null, "localhost"); + tsdb.assignUid(null, METRIC_STRING); } @Test (expected = IllegalArgumentException.class) public void assignUidNullName() { - setupAssignUid(); tsdb.assignUid("metric", null); } @Test (expected = IllegalArgumentException.class) public void assignUidInvalidCharacter() { - setupAssignUid(); tsdb.assignUid("metric", "Not!A:Valid@Name"); } - @Test - public void uidTable() { - assertNotNull(tsdb.uidTable()); - assertArrayEquals("tsdb-uid".getBytes(), tsdb.uidTable()); + @Test (expected = IllegalArgumentException.class) + public void renameUidInvalidNewname() { + tsdb.renameUid("metric", "existing", null); } - @Test - public void addPointLong1Byte() throws Exception { - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 1356998400, 42, tags).joinUninterruptibly(); - final byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, - 0, 0, 1, 0, 0, 1}; - final byte[] value = storage.getColumn(row, new byte[] { 0, 0 }); - assertNotNull(value); - assertEquals(42, value[0]); - } - - @Test - public void addPointLong1ByteNegative() throws Exception { - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 1356998400, -42, tags).joinUninterruptibly(); - final byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, - 0, 0, 1, 0, 0, 1}; - final byte[] value = storage.getColumn(row, new byte[] { 0, 0 }); - assertNotNull(value); - assertEquals(-42, value[0]); - } - - @Test - public void addPointLong2Bytes() throws Exception { - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 1356998400, 257, tags).joinUninterruptibly(); - final byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, - 0, 0, 1, 0, 0, 1}; - final byte[] value = storage.getColumn(row, new byte[] { 0, 1 }); - assertNotNull(value); - assertEquals(257, Bytes.getShort(value)); - } - - @Test - public void addPointLong2BytesNegative() throws Exception { - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 1356998400, -257, tags).joinUninterruptibly(); - final byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, - 0, 0, 1, 0, 0, 1}; - final byte[] value = storage.getColumn(row, new byte[] { 0, 1 }); - assertNotNull(value); - assertEquals(-257, Bytes.getShort(value)); - } - - @Test - public void addPointLong4Bytes() throws Exception { - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 1356998400, 65537, tags).joinUninterruptibly(); - final byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, - 0, 0, 1, 0, 0, 1}; - final byte[] value = storage.getColumn(row, new byte[] { 0, 3 }); - assertNotNull(value); - assertEquals(65537, Bytes.getInt(value)); - } - - @Test - public void addPointLong4BytesNegative() throws Exception { - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 1356998400, -65537, tags).joinUninterruptibly(); - final byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, - 0, 0, 1, 0, 0, 1}; - final byte[] value = storage.getColumn(row, new byte[] { 0, 3 }); - assertNotNull(value); - assertEquals(-65537, Bytes.getInt(value)); - } - - @Test - public void addPointLong8Bytes() throws Exception { - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 1356998400, 4294967296L, tags).joinUninterruptibly(); - final byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, - 0, 0, 1, 0, 0, 1}; - final byte[] value = storage.getColumn(row, new byte[] { 0, 7 }); - assertNotNull(value); - assertEquals(4294967296L, Bytes.getLong(value)); - } - - @Test - public void addPointLong8BytesNegative() throws Exception { - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 1356998400, -4294967296L, tags).joinUninterruptibly(); - final byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, - 0, 0, 1, 0, 0, 1}; - final byte[] value = storage.getColumn(row, new byte[] { 0, 7 }); - assertNotNull(value); - assertEquals(-4294967296L, Bytes.getLong(value)); - } - - @Test - public void addPointLongMs() throws Exception { - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 1356998400500L, 42, tags).joinUninterruptibly(); - final byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, - 0, 0, 1, 0, 0, 1}; - final byte[] value = storage.getColumn(row, - new byte[] { (byte) 0xF0, 0, 0x7D, 0 }); - assertNotNull(value); - assertEquals(42, value[0]); - } - - @Test - public void addPointLongMany() throws Exception { - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - long timestamp = 1356998400; - for (int i = 1; i <= 50; i++) { - tsdb.addPoint("sys.cpu.user", timestamp++, i, tags).joinUninterruptibly(); - } - final byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, - 0, 0, 1, 0, 0, 1}; - final byte[] value = storage.getColumn(row, new byte[] { 0, 0 }); - assertNotNull(value); - assertEquals(1, value[0]); - assertEquals(50, storage.numColumns(row)); - } - - @Test - public void addPointLongManyMs() throws Exception { - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - long timestamp = 1356998400500L; - for (int i = 1; i <= 50; i++) { - tsdb.addPoint("sys.cpu.user", timestamp++, i, tags).joinUninterruptibly(); - } - final byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, - 0, 0, 1, 0, 0, 1}; - final byte[] value = storage.getColumn(row, - new byte[] { (byte) 0xF0, 0, 0x7D, 0 }); - assertNotNull(value); - assertEquals(1, value[0]); - assertEquals(50, storage.numColumns(row)); + @Test (expected = IllegalArgumentException.class) + public void renameUidNonexistentMetric() { + when(metrics.getId("sys.cpu.1")).thenThrow( + new NoSuchUniqueName("metric", "sys.cpu.1")); + tsdb.renameUid("metric", "sys.cpu.1", "sys.cpu.2"); } - + @Test - public void addPointLongEndOfRow() throws Exception { - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 1357001999, 42, tags).joinUninterruptibly(); - final byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, - 0, 0, 1, 0, 0, 1}; - final byte[] value = storage.getColumn(row, new byte[] { (byte) 0xE0, - (byte) 0xF0 }); - assertNotNull(value); - assertEquals(42, value[0]); + public void renameUidMetric() { + tsdb.renameUid("metric", "sys.cpu.1", "sys.cpu.2"); } - - @Test - public void addPointLongOverwrite() throws Exception { - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 1356998400, 42, tags).joinUninterruptibly(); - tsdb.addPoint("sys.cpu.user", 1356998400, 24, tags).joinUninterruptibly(); - final byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, - 0, 0, 1, 0, 0, 1}; - final byte[] value = storage.getColumn(row, new byte[] { 0, 0 }); - assertNotNull(value); - assertEquals(24, value[0]); - } - - @SuppressWarnings("unchecked") - @Test (expected = NoSuchUniqueName.class) - public void addPointNoAutoMetric() throws Exception { - setupAddPointStorage(); - when(metrics.getId(anyString())).thenThrow(new NoSuchUniqueName("sys.cpu.user", "metric")); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 1356998400, 42, tags).joinUninterruptibly(); + @Test (expected = IllegalArgumentException.class) + public void renameUidNonexistentTagk() { + when(tag_names.getId("datacenter")).thenThrow( + new NoSuchUniqueName("tagk", "datacenter")); + tsdb.renameUid("tagk", "datacenter", "datacluster"); } @Test - public void addPointSecondZero() throws Exception { - // Thu, 01 Jan 1970 00:00:00 GMT - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 0, 42, tags).joinUninterruptibly(); - final byte[] row = new byte[] { 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1}; - final byte[] value = storage.getColumn(row, new byte[] { 0, 0 }); - assertNotNull(value); - assertEquals(42, value[0]); - } - - @Test - public void addPointSecondOne() throws Exception { - // hey, it's valid *shrug* Thu, 01 Jan 1970 00:00:01 GMT - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 1, 42, tags).joinUninterruptibly(); - final byte[] row = new byte[] { 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1}; - final byte[] value = storage.getColumn(row, new byte[] { 0, 16 }); - assertNotNull(value); - assertEquals(42, value[0]); - } - - @Test - public void addPointSecond2106() throws Exception { - // Sun, 07 Feb 2106 06:28:15 GMT - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 4294967295L, 42, tags).joinUninterruptibly(); - final byte[] row = new byte[] { 0, 0, 1, (byte) 0xFF, (byte) 0xFF, (byte) 0xF9, - 0x60, 0, 0, 1, 0, 0, 1}; - final byte[] value = storage.getColumn(row, new byte[] { 0x69, (byte) 0xF0 }); - assertNotNull(value); - assertEquals(42, value[0]); - } - - @Test (expected = IllegalArgumentException.class) - public void addPointSecondNegative() throws Exception { - // Fri, 13 Dec 1901 20:45:52 GMT - // may support in the future, but 1.0 didn't - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", -2147483648, 42, tags).joinUninterruptibly(); + public void renameUidTagk() { + tsdb.renameUid("tagk", "datacenter", "datacluster"); } - + @Test (expected = IllegalArgumentException.class) - public void emptyTagValue() throws Exception { - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>() {{ - put("host", ""); - }}; - tsdb.addPoint("sys.cpu.user", 1234567890, 42, tags).joinUninterruptibly(); + public void renameUidNonexistentTagv() { + when(tag_values.getId("localhost")).thenThrow( + new NoSuchUniqueName("tagv", "localhost")); + tsdb.renameUid("tagv", "localhost", "127.0.0.1"); } @Test - public void addPointMS1970() throws Exception { - // Since it's just over Integer.MAX_VALUE, OpenTSDB will treat this as - // a millisecond timestamp since it doesn't fit in 4 bytes. - // Base time is 4294800 which is Thu, 19 Feb 1970 17:00:00 GMT - // offset = F0A36000 or 167296 ms - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 4294967296L, 42, tags).joinUninterruptibly(); - final byte[] row = new byte[] { 0, 0, 1, 0, (byte) 0x41, (byte) 0x88, - (byte) 0x90, 0, 0, 1, 0, 0, 1}; - final byte[] value = storage.getColumn(row, new byte[] { (byte) 0xF0, - (byte) 0xA3, 0x60, 0}); - assertNotNull(value); - assertEquals(42, value[0]); - } - - @Test - public void addPointMS2106() throws Exception { - // Sun, 07 Feb 2106 06:28:15.000 GMT - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 4294967295000L, 42, tags).joinUninterruptibly(); - final byte[] row = new byte[] { 0, 0, 1, (byte) 0xFF, (byte) 0xFF, (byte) 0xF9, - 0x60, 0, 0, 1, 0, 0, 1}; - final byte[] value = storage.getColumn(row, new byte[] { (byte) 0xF6, - (byte) 0x77, 0x46, 0 }); - assertNotNull(value); - assertEquals(42, value[0]); + public void renameUidTagv() { + tsdb.renameUid("tagv", "localhost", "127.0.0.1"); } - - @Test - public void addPointMS2286() throws Exception { - // It's an artificial limit and more thought needs to be put into it - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 9999999999999L, 42, tags).joinUninterruptibly(); - final byte[] row = new byte[] { 0, 0, 1, (byte) 0x54, (byte) 0x0B, (byte) 0xD9, - 0x10, 0, 0, 1, 0, 0, 1}; - final byte[] value = storage.getColumn(row, new byte[] { (byte) 0xFA, - (byte) 0xAE, 0x5F, (byte) 0xC0 }); - assertNotNull(value); - assertEquals(42, value[0]); - } - - @Test (expected = IllegalArgumentException.class) - public void addPointMSTooLarge() throws Exception { - // It's an artificial limit and more thought needs to be put into it - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 10000000000000L, 42, tags).joinUninterruptibly(); - } - + @Test (expected = IllegalArgumentException.class) - public void addPointMSNegative() throws Exception { - // Fri, 13 Dec 1901 20:45:52 GMT - // may support in the future, but 1.0 didn't - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", -2147483648000L, 42, tags).joinUninterruptibly(); + public void renameUidBadType() { + tsdb.renameUid("wrongtype", METRIC_STRING, METRIC_STRING); } @Test - public void addPointFloat() throws Exception { - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 1356998400, 42.5F, tags).joinUninterruptibly(); - final byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, - 0, 0, 1, 0, 0, 1}; - final byte[] value = storage.getColumn(row, new byte[] { 0, 11 }); - assertNotNull(value); - // should have 7 digits of precision - assertEquals(42.5F, Float.intBitsToFloat(Bytes.getInt(value)), 0.0000001); - } - - @Test - public void addPointFloatNegative() throws Exception { - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 1356998400, -42.5F, tags).joinUninterruptibly(); - final byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, - 0, 0, 1, 0, 0, 1}; - final byte[] value = storage.getColumn(row, new byte[] { 0, 11 }); - assertNotNull(value); - // should have 7 digits of precision - assertEquals(-42.5F, Float.intBitsToFloat(Bytes.getInt(value)), 0.0000001); - } - - @Test - public void addPointFloatMs() throws Exception { - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 1356998400500L, 42.5F, tags).joinUninterruptibly(); - final byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, - 0, 0, 1, 0, 0, 1}; - final byte[] value = storage.getColumn(row, - new byte[] { (byte) 0xF0, 0, 0x7D, 11 }); - assertNotNull(value); - // should have 7 digits of precision - assertEquals(42.5F, Float.intBitsToFloat(Bytes.getInt(value)), 0.0000001); - } - - @Test - public void addPointFloatEndOfRow() throws Exception { - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 1357001999, 42.5F, tags).joinUninterruptibly(); - final byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, - 0, 0, 1, 0, 0, 1}; - final byte[] value = storage.getColumn(row, new byte[] { (byte) 0xE0, - (byte) 0xFB }); - assertNotNull(value); - // should have 7 digits of precision - assertEquals(42.5F, Float.intBitsToFloat(Bytes.getInt(value)), 0.0000001); - } - - @Test - public void addPointFloatPrecision() throws Exception { - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 1356998400, 42.5123459999F, tags) - .joinUninterruptibly(); - final byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, - 0, 0, 1, 0, 0, 1}; - final byte[] value = storage.getColumn(row, new byte[] { 0, 11 }); - assertNotNull(value); - // should have 7 digits of precision - assertEquals(42.512345F, Float.intBitsToFloat(Bytes.getInt(value)), 0.0000001); - } - - @Test - public void addPointFloatOverwrite() throws Exception { - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 1356998400, 42.5F, tags).joinUninterruptibly(); - tsdb.addPoint("sys.cpu.user", 1356998400, 25.4F, tags).joinUninterruptibly(); - final byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, - 0, 0, 1, 0, 0, 1}; - final byte[] value = storage.getColumn(row, new byte[] { 0, 11 }); - assertNotNull(value); - // should have 7 digits of precision - assertEquals(25.4F, Float.intBitsToFloat(Bytes.getInt(value)), 0.0000001); - } - - @Test - public void addPointBothSameTimeIntAndFloat() throws Exception { - // this is an odd situation that can occur if the user puts an int and then - // a float (or vice-versa) with the same timestamp. What happens in the - // aggregators when this occurs? - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 1356998400, 42, tags).joinUninterruptibly(); - tsdb.addPoint("sys.cpu.user", 1356998400, 42.5F, tags).joinUninterruptibly(); - final byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, - 0, 0, 1, 0, 0, 1}; - byte[] value = storage.getColumn(row, new byte[] { 0, 0 }); - assertEquals(2, storage.numColumns(row)); - assertNotNull(value); - assertEquals(42, value[0]); - value = storage.getColumn(row, new byte[] { 0, 11 }); - assertNotNull(value); - // should have 7 digits of precision - assertEquals(42.5F, Float.intBitsToFloat(Bytes.getInt(value)), 0.0000001); - } - - @Test - public void addPointBothSameTimeIntAndFloatMs() throws Exception { - // this is an odd situation that can occur if the user puts an int and then - // a float (or vice-versa) with the same timestamp. What happens in the - // aggregators when this occurs? - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 1356998400500L, 42, tags).joinUninterruptibly(); - tsdb.addPoint("sys.cpu.user", 1356998400500L, 42.5F, tags).joinUninterruptibly(); - final byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, - 0, 0, 1, 0, 0, 1}; - byte[] value = storage.getColumn(row, new byte[] { (byte) 0xF0, 0, 0x7D, 0 }); - assertEquals(2, storage.numColumns(row)); - assertNotNull(value); - assertEquals(42, value[0]); - value = storage.getColumn(row, new byte[] { (byte) 0xF0, 0, 0x7D, 11 }); - assertNotNull(value); - // should have 7 digits of precision - assertEquals(42.5F, Float.intBitsToFloat(Bytes.getInt(value)), 0.0000001); - } - - @Test - public void addPointBothSameTimeSecondAndMs() throws Exception { - // this can happen if a second and an ms data point are stored for the same - // timestamp. - setupAddPointStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 1356998400L, 42, tags).joinUninterruptibly(); - tsdb.addPoint("sys.cpu.user", 1356998400000L, 42, tags).joinUninterruptibly(); - final byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, - 0, 0, 1, 0, 0, 1}; - byte[] value = storage.getColumn(row, new byte[] { 0, 0 }); - assertEquals(2, storage.numColumns(row)); - assertNotNull(value); - assertEquals(42, value[0]); - value = storage.getColumn(row, new byte[] { (byte) 0xF0, 0, 0, 0 }); - assertNotNull(value); - // should have 7 digits of precision - assertEquals(42, value[0]); - } - - /** - * Helper to mock the UID caches with valid responses - */ - private void setupAssignUid() { - when(metrics.getId("sys.cpu.0")).thenReturn(new byte[] { 0, 0, 1 }); - when(metrics.getId("sys.cpu.1")).thenThrow( - new NoSuchUniqueName("metric", "sys.cpu.1")); - when(metrics.getOrCreateId("sys.cpu.1")).thenReturn(new byte[] { 0, 0, 2 }); - - when(tag_names.getId("host")).thenReturn(new byte[] { 0, 0, 1 }); - when(tag_names.getId("datacenter")).thenThrow( - new NoSuchUniqueName("tagk", "datacenter")); - when(tag_names.getOrCreateId("datacenter")).thenReturn(new byte[] { 0, 0, 2 }); - - when(tag_values.getId("localhost")).thenReturn(new byte[] { 0, 0, 1 }); - when(tag_values.getId("myserver")).thenThrow( - new NoSuchUniqueName("tagv", "myserver")); - when(tag_values.getOrCreateId("myserver")).thenReturn(new byte[] { 0, 0, 2 }); + public void uidTable() { + assertNotNull(tsdb.uidTable()); + assertArrayEquals("tsdb-uid".getBytes(), tsdb.uidTable()); } - + /** * Helper to mock the UID caches with valid responses */ @@ -858,17 +537,5 @@ private void setGetUidName() { @Test public void setupAddPointStorage() throws Exception { storage = new MockBase(tsdb, client, true, true, true, true); - when(metrics.width()).thenReturn((short)3); - when(metrics.getId(anyString())).thenReturn(new byte[] { 0, 0, 1 }); - when(tag_names.width()).thenReturn((short)3); - when(tag_values.width()).thenReturn((short)3); - when(tag_values.getId(anyString())).thenReturn(new byte[] { 0, 0, 1 }); - when(tag_names.getId(anyString())).thenReturn(new byte[] { 0, 0, 1 }); - when(tag_values.getOrCreateId(anyString())).thenReturn(new byte[] { 0, 0, 1 }); - when(tag_names.getOrCreateId(anyString())).thenReturn(new byte[] { 0, 0, 1 }); - - HashMap<String, String> tags = new HashMap<String, String>() {{ - put("host", "web01"); - }}; } } diff --git a/test/core/TestTSDBAddAggregatePoint.java b/test/core/TestTSDBAddAggregatePoint.java new file mode 100644 index 0000000000..60478b88f6 --- /dev/null +++ b/test/core/TestTSDBAddAggregatePoint.java @@ -0,0 +1,933 @@ +package net.opentsdb.core; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.anyShort; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import net.opentsdb.rollup.NoSuchRollupForIntervalException; +import net.opentsdb.rollup.RollupConfig; +import net.opentsdb.rollup.RollupInterval; +import net.opentsdb.rollup.RollupUtils; +import net.opentsdb.storage.MockBase; +import net.opentsdb.uid.NoSuchUniqueName; + +import org.hbase.async.Bytes; +import org.junit.Before; +import org.junit.Test; +import org.powermock.reflect.Whitebox; + +import com.stumbleupon.async.Deferred; + +public class TestTSDBAddAggregatePoint extends BaseTsdbTest { + protected final static byte[] TSDB_TABLE = "tsdb".getBytes(MockBase.ASCII()); + protected final static byte[] AGG_TABLE = "tsdb-agg".getBytes(MockBase.ASCII()); + protected final static byte[] FAMILY = "t".getBytes(MockBase.ASCII()); + protected RollupConfig rollup_config; + protected String agg_tag_key; + protected byte[] row; + + @Before + public void beforeLocal() throws Exception { + agg_tag_key = config.getString("tsd.rollups.agg_tag_key"); + + storage = new MockBase(tsdb, client, true, true, true, true); + final List<byte[]> families = new ArrayList<byte[]>(); + families.add(FAMILY); + + storage.addTable("tsdb-rollup-10m".getBytes(), families); + storage.addTable("tsdb-rollup-agg-10m".getBytes(), families); + storage.addTable("tsdb-rollup-1h".getBytes(), families); + storage.addTable("tsdb-rollup-agg-1h".getBytes(), families); + storage.addTable("tsdb-rollup-1d".getBytes(), families); + storage.addTable("tsdb-rollup-agg-1d".getBytes(), families); + storage.addTable(AGG_TABLE, families); + + rollup_config = RollupConfig.builder() + .addAggregationId("sum", 0) + .addAggregationId("count", 1) + .addAggregationId("max", 2) + .addAggregationId("min", 3) + .addInterval(RollupInterval.builder() + .setTable("tsdb") + .setPreAggregationTable("tsdb-agg") + .setInterval("1m") + .setRowSpan("1h") + .setDefaultInterval(true)) + .addInterval(RollupInterval.builder() + .setTable("tsdb-rollup-10m") + .setPreAggregationTable("tsdb-rollup-agg-10m") + .setInterval("10m") + .setRowSpan("1d")) + .addInterval(RollupInterval.builder() + .setTable("tsdb-rollup-1h") + .setPreAggregationTable("tsdb-rollup-agg-1h") + .setInterval("1h") + .setRowSpan("1n")) + .addInterval(RollupInterval.builder() + .setTable("tsdb-rollup-1d") + .setPreAggregationTable("tsdb-rollup-agg-1d") + .setInterval("1d") + .setRowSpan("1y")) + .build(); + Whitebox.setInternalState(tsdb, "rollup_config", rollup_config); + Whitebox.setInternalState(tsdb, "default_interval", + rollup_config.getRollupInterval("1m")); + Whitebox.setInternalState(tsdb, "rollups_block_derived", true); + Whitebox.setInternalState(tsdb, "rollups_split_queries", false); + Whitebox.setInternalState(tsdb, "agg_tag_key", + config.getString("tsd.rollups.agg_tag_key")); + Whitebox.setInternalState(tsdb, "raw_agg_tag_value", + config.getString("tsd.rollups.raw_agg_tag_value")); + setupGroupByTagValues(); + + row = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_STRING); + } + + @Test + public void addAggregatePointLong1Byte() throws Exception { + final byte[] qualifier = new byte[] {0, 0, 0}; + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400L, 42, tags, false, + "10m", "sum", null).joinUninterruptibly(); + + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("10m").getTemporalTable(), + row, FAMILY, qualifier); + final byte[] expected = {0x2A}; + assertArrayEquals(expected, value); + } + + @Test + public void addAggregatePointLong1ByteNegative() throws Exception { + final byte[] qualifier = new byte[] {0, 0, 0}; + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400L, -42, tags, false, + "10m", "sum", null).joinUninterruptibly(); + + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("10m").getTemporalTable(), + row, FAMILY, qualifier); + final byte[] expected = {(byte) 0xD6}; + assertArrayEquals(expected, value); + } + + @Test + public void addAggregatePointLong2Bytes() throws Exception { + final byte[] qualifier = new byte[] {0, 0, 1}; + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400L, 257, tags, false, + "10m", "sum", null).joinUninterruptibly(); + + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("10m").getTemporalTable(), + row, FAMILY, qualifier); + final byte[] expected = {1, 1}; + assertArrayEquals(expected, value); + } + + @Test + public void addAggregatePointLong2BytesNegative() throws Exception { + final byte[] qualifier = new byte[] {0, 0, 1}; + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400L, -257, tags, false, + "10m", "sum", null).joinUninterruptibly(); + + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("10m").getTemporalTable(), + row, FAMILY, qualifier); + final byte[] expected = {(byte) 0xFE, (byte) 0xFF}; + assertArrayEquals(expected, value); + } + + @Test + public void addAggregatePointLong4Bytes() throws Exception { + final byte[] qualifier = new byte[] {0, 0, 3}; + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400L, 65537, tags, false, + "10m", "sum", null).joinUninterruptibly(); + + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("10m").getTemporalTable(), + row, FAMILY, qualifier); + final byte[] expected = {0, 1, 0, 1}; + assertArrayEquals(expected, value); + } + + @Test + public void addAggregatePointLong4BytesNegative() throws Exception { + final byte[] qualifier = new byte[] {0, 0, 3}; + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400L, -65537, tags, false, + "10m", "sum", null).joinUninterruptibly(); + + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("10m").getTemporalTable(), + row, FAMILY, qualifier); + final byte[] expected = + {(byte) 0xFF, (byte) 0xFE, (byte) 0xFF, (byte) 0xFF}; + assertArrayEquals(expected, value); + } + + @Test + public void addAggregatePointLong8Bytes() throws Exception { + final byte[] qualifier = new byte[] {0, 0, 7}; + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400L, 4294967296L, tags, false, + "10m", "sum", null).joinUninterruptibly(); + + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("10m").getTemporalTable(), + row, FAMILY, qualifier); + final byte[] expected = {0, 0, 0, 1, 0, 0, 0, 0}; + assertArrayEquals(expected, value); + } + + @Test + public void addAggregatePointLong8BytesNegative() throws Exception { + final byte[] qualifier = new byte[] {0, 0, 7}; + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400L, -4294967296L, tags, false, + "10m", "sum", null).joinUninterruptibly(); + + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("10m").getTemporalTable(), + row, FAMILY, qualifier); + final byte[] expected = + {(byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, 0, 0, 0, 0}; + assertArrayEquals(expected, value); + } + + @Test + public void addAggregatePointFloat4Bytes() throws Exception { + final byte[] qualifier = new byte[] {0, 0, 0x0B}; + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400L, 42.5F, tags, false, + "10m", "sum", null).joinUninterruptibly(); + + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("10m").getTemporalTable(), + row, FAMILY, qualifier); + final float read = + Float.intBitsToFloat(Bytes.getInt(Arrays.copyOf(value, 4))); + assertEquals(42.5F, read, 0.0000001); + } + + @Test + public void addAggregatePointFloat4BytesNegative() throws Exception { + final byte[] qualifier = new byte[] {0, 0, 0x0B}; + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400L, -42.5F, tags, false, + "10m", "sum", null).joinUninterruptibly(); + + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("10m").getTemporalTable(), + row, FAMILY, qualifier); + final float read = + Float.intBitsToFloat(Bytes.getInt(Arrays.copyOf(value, 4))); + assertEquals(-42.5F, read, 0.0000001); + } + + @Test + public void addAggregatePointFloat4BytesPrecision() throws Exception { + final byte[] qualifier = new byte[] {0, 0, 0x0B}; + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400L, 42.5123459999F, tags, false, + "10m", "sum", null).joinUninterruptibly(); + + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("10m").getTemporalTable(), + row, FAMILY, qualifier); + final float read = + Float.intBitsToFloat(Bytes.getInt(Arrays.copyOf(value, 4))); + assertEquals(42.5123459999F, read, 0.0000001); + } + + @Test + public void addAggregatePointFloat4BytesPrecisionNegative() throws Exception { + final byte[] qualifier = new byte[] {0, 0, 0x0B}; + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400L, -42.5123459999F, tags, false, + "10m", "sum", null).joinUninterruptibly(); + + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("10m").getTemporalTable(), + row, FAMILY, qualifier); + final float read = + Float.intBitsToFloat(Bytes.getInt(Arrays.copyOf(value, 4))); + assertEquals(-42.5123459999F, read, 0.0000001); + } + + // not allowing rollups with millisecond precision + @Test (expected = IllegalArgumentException.class) + public void addAggregatePointMilliseconds() throws Exception { + tsdb.addAggregatePoint(METRIC_STRING, 1419992400000L, 42, tags, false, + "10m", "sum", null).joinUninterruptibly(); + } + + @Test (expected = NoSuchRollupForIntervalException.class) + public void addAggregatePointNoSuchRollup() throws Exception { + tsdb.addAggregatePoint(METRIC_STRING, 1356998400L, 42, tags, false, + "11m", "sum", null).joinUninterruptibly(); + } + + @Test + public void addAggregatePoint10mInDayTop() throws Exception { + row = getRowKey(METRIC_STRING, 1370476800, TAGK_STRING, TAGV_STRING); + final byte[] qualifier = new byte[] {0, 0, 0}; + + tsdb.addAggregatePoint(METRIC_STRING, 1370476800, 42, tags, false, + "10m", "sum", null).joinUninterruptibly(); + + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("10m").getTemporalTable(), + row, FAMILY, qualifier); + final byte[] expected = {0x2A}; + assertArrayEquals(expected, value); + } + + @Test + public void addAggregatePoint10mInDayMid() throws Exception { + row = getRowKey(METRIC_STRING, 1370476800, TAGK_STRING, TAGV_STRING); + final byte[] qualifier = new byte[] {0, 5, (byte) 0xD0}; + + tsdb.addAggregatePoint(METRIC_STRING, 1370532925L, 42, tags, false, + "10m", "sum", null).joinUninterruptibly(); + + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("10m").getTemporalTable(), + row, FAMILY, qualifier); + final byte[] expected = {0x2A}; + assertArrayEquals(expected, value); + } + + @Test + public void addAggregatePoint10mInDayEnd() throws Exception { + row = getRowKey(METRIC_STRING, 1370476800, TAGK_STRING, TAGV_STRING); + final byte[] qualifier = new byte[] {0, 5, (byte) 0xF0}; + + tsdb.addAggregatePoint(METRIC_STRING, 1370534399L, 42, tags, false, + "10m", "sum", null).joinUninterruptibly(); + + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("10m").getTemporalTable(), + row, FAMILY, qualifier); + final byte[] expected = {0x2A}; + assertArrayEquals(expected, value); + } + + @Test + public void addAggregatePoint10mInDayOver() throws Exception { + row = getRowKey(METRIC_STRING, 1370563200, TAGK_STRING, TAGV_STRING); + final byte[] qualifier = new byte[] {0, 0, 0}; + + tsdb.addAggregatePoint(METRIC_STRING, 1370563200L, 42, tags, false, + "10m", "sum", null).joinUninterruptibly(); + + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("10m").getTemporalTable(), + row, FAMILY, qualifier); + final byte[] expected = {0x2A}; + assertArrayEquals(expected, value); + } + + @Test + public void addAggregatePoint1hInMonthTop() throws Exception { + row = getRowKey(METRIC_STRING, 1370044800, TAGK_STRING, TAGV_STRING); + final byte[] qualifier = new byte[] {0, 0, 0}; + + tsdb.addAggregatePoint(METRIC_STRING, 1370044800L, 42, tags, false, + "1h", "sum", null).joinUninterruptibly(); + + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("1h").getTemporalTable(), + row, FAMILY, qualifier); + final byte[] expected = {0x2A}; + assertArrayEquals(expected, value); + } + + @Test + public void addAggregatePoint1hInMonthMid() throws Exception { + row = getRowKey(METRIC_STRING, 1370044800, TAGK_STRING, TAGV_STRING); + final byte[] qualifier = new byte[] {0, 0x2C, (byte) 0xF0}; + + tsdb.addAggregatePoint(METRIC_STRING, 1372636799L, 42, tags, false, + "1h", "sum", null).joinUninterruptibly(); + + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("1h").getTemporalTable(), + row, FAMILY, qualifier); + final byte[] expected = {0x2A}; + assertArrayEquals(expected, value); + } + + @Test + public void addAggregatePoint1hInMonthOver() throws Exception { + row = getRowKey(METRIC_STRING, 1372636800, TAGK_STRING, TAGV_STRING); + final byte[] qualifier = new byte[] {0, 0, 0}; + + tsdb.addAggregatePoint(METRIC_STRING, 1372636800L, 42, tags, false, + "1h", "sum", null).joinUninterruptibly(); + + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("1h").getTemporalTable(), + row, FAMILY, qualifier); + final byte[] expected = {0x2A}; + assertArrayEquals(expected, value); + } + + @Test + public void addAggregatePoint1dInYearTop() throws Exception { + row = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_STRING); + final byte[] qualifier = new byte[] {0, 0, 0}; + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400L, 42, tags, false, + "1d", "sum", null).joinUninterruptibly(); + + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("1d").getTemporalTable(), + row, FAMILY, qualifier); + final byte[] expected = {0x2A}; + assertArrayEquals(expected, value); + } + + @Test + public void addAggregatePoint1dInYearMid() throws Exception { + row = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_STRING); + final byte[] qualifier = new byte[] {0, 9, (byte) 0xC0}; + + tsdb.addAggregatePoint(METRIC_STRING, 1370532925L, 42, tags, false, + "1d", "sum", null).joinUninterruptibly(); + + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("1d").getTemporalTable(), + row, FAMILY, qualifier); + final byte[] expected = {0x2A}; + assertArrayEquals(expected, value); + } + + @Test + public void addAggregatePoint1dInYearEnd() throws Exception { + row = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_STRING); + final byte[] qualifier = new byte[] {0, 0x16, (byte) 0xC0}; + + tsdb.addAggregatePoint(METRIC_STRING, 1388534399L, 42, tags, false, + "1d", "sum", null).joinUninterruptibly(); + + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("1d").getTemporalTable(), + row, FAMILY, qualifier); + final byte[] expected = {0x2A}; + assertArrayEquals(expected, value); + } + + @Test + public void addAggregatePoint1dInYearOver() throws Exception { + row = getRowKey(METRIC_STRING, 1388534400, TAGK_STRING, TAGV_STRING); + final byte[] qualifier = new byte[] {0, 0, 0}; + + tsdb.addAggregatePoint(METRIC_STRING, 1388534400L, 42, tags, false, + "1d", "sum", null).joinUninterruptibly(); + + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("1d").getTemporalTable(), + row, FAMILY, qualifier); + final byte[] expected = {0x2A}; + assertArrayEquals(expected, value); + } + + @Test (expected = NoSuchUniqueName.class) + public void addAggregatePointNSUNMetric() throws Exception { + tsdb.addAggregatePoint(NSUN_METRIC, 1388534400L, 42, tags, false, + "1d", "sum", null).joinUninterruptibly(); + } + + @Test (expected = NoSuchUniqueName.class) + public void addAggregatePointNSUNTagK() throws Exception { + tags.clear(); + tags.put(NSUN_TAGK, TAGV_STRING); + tsdb.addAggregatePoint(METRIC_STRING, 1388534400L, 42, tags, false, + "1d", "sum", null).joinUninterruptibly(); + } + + @Test (expected = NoSuchUniqueName.class) + public void addAggregatePointNSUNTagV() throws Exception { + tags.put(TAGK_STRING, NSUN_TAGV); + tsdb.addAggregatePoint(METRIC_STRING, 1388534400L, 42, tags, false, + "1d", "sum", null).joinUninterruptibly(); + } + + @Test (expected = IllegalArgumentException.class) + public void addAggregatePointRollupNoSuchAgg() throws Exception { + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 42, tags, false, "10m", + "nosuchagg", null).joinUninterruptibly(); + } + + @Test (expected = IllegalArgumentException.class) + public void addAggregatePointRollupsNotConfigured() throws Exception { + Whitebox.setInternalState(tsdb, "rollup_config", (RollupConfig)null); + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 42, tags, false, "10m", + "nosuchagg", null).joinUninterruptibly(); + } + + @Test (expected = IllegalArgumentException.class) + public void addAggregatePointNegativeTimestamp() throws Exception { + tsdb.addAggregatePoint(METRIC_STRING, -1356998400, 42, tags, false, "10m", + "sum", null).joinUninterruptibly(); + } + + @Test (expected = IllegalArgumentException.class) + public void addAggregatePointEmptyTags() throws Exception { + tags.put(TAGK_STRING, ""); + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 42, tags, false, "10m", + "sum", null).joinUninterruptibly(); + } + + // not allowed at this time + @Test (expected = IllegalArgumentException.class) + public void addAggregatePointMS() throws Exception { + tsdb.addAggregatePoint(METRIC_STRING, 1356998400000L, 42, tags, false, "10m", + "nosuchagg", null).joinUninterruptibly(); + } + + @Test (expected = IllegalArgumentException.class) + public void addAggregatePointNullInterval() throws Exception { + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 42, tags, false, null, + "sum", null).joinUninterruptibly(); + } + + @Test (expected = IllegalArgumentException.class) + public void addAggregatePointEmptyInterval() throws Exception { + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 42, tags, false, "", + "sum", null).joinUninterruptibly(); + } + + @Test (expected = NoSuchRollupForIntervalException.class) + public void addAggregatePointIntervalNotConfigured() throws Exception { + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 42, tags, false, "6h", + "sum", null).joinUninterruptibly(); + } + + @Test (expected = IllegalArgumentException.class) + public void addAggregatePointNullAggregator() throws Exception { + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 42, tags, false, "10m", + null, null).joinUninterruptibly(); + } + + @Test (expected = IllegalArgumentException.class) + public void addAggregatePointEmptyAggregator() throws Exception { + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 42, tags, false, "10m", + "", null).joinUninterruptibly(); + } + + @Test + public void addAggregatePointRollupRouting() throws Exception { + RollupInterval interval = rollup_config.getRollupInterval("10m"); + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 42, tags, false, "10m", + "sum", null).joinUninterruptibly(); + + assertEquals(42, storage.getColumn(interval.getTemporalTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)0, 0, + interval))[0]); + // make sure it didn't get into the tsdb table + assertNull(storage.getColumn(TSDB_TABLE, + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)0, 0, + interval))); + + storage.flushStorage(); + + interval = rollup_config.getRollupInterval("1h"); + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 42, tags, false, "1h", + "sum", null).joinUninterruptibly(); + + assertEquals(42, storage.getColumn(interval.getTemporalTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)0, 0, + interval))[0]); + assertNull(storage.getColumn(TSDB_TABLE, + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)0, 0, + interval))); + assertNull(storage.getColumn( + rollup_config.getRollupInterval("10m").getTemporalTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)0, 0, + rollup_config.getRollupInterval("10m")))); + + storage.flushStorage(); + + interval = rollup_config.getRollupInterval("1d"); + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 42, tags, false, "1d", + "sum", null).joinUninterruptibly(); + + assertEquals(42, storage.getColumn(interval.getTemporalTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)0, 0, + interval))[0]); + assertNull(storage.getColumn(TSDB_TABLE, + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)0, 0, + interval))); + assertNull(storage.getColumn( + rollup_config.getRollupInterval("1h").getTemporalTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)0, 0, + rollup_config.getRollupInterval("1h")))); + + storage.flushStorage(); + // other aggs + interval = rollup_config.getRollupInterval("1h"); + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 42, tags, false, "1h", + "max", null).joinUninterruptibly(); + assertEquals(42, storage.getColumn(interval.getTemporalTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)0, 2, + interval))[0]); + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 42, tags, false, "1h", + "min", null).joinUninterruptibly(); + assertEquals(42, storage.getColumn(interval.getTemporalTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)0, 3, + interval))[0]); + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 42, tags, false, "1h", + "count", null).joinUninterruptibly(); + assertEquals(42, storage.getColumn(interval.getTemporalTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)0, 1, + interval))[0]); + } + + @Test + public void addAggregatePointLongs() throws Exception { + final RollupInterval interval = rollup_config.getRollupInterval("10m"); + + // 1 byte + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 0, tags, false, "10m", + "sum", null).joinUninterruptibly(); + assertEquals(0, storage.getColumn(interval.getTemporalTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)0, 0, + interval))[0]); + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 42, tags, false, "10m", + "sum", null).joinUninterruptibly(); + assertEquals(42, storage.getColumn(interval.getTemporalTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)0, 0, + interval))[0]); + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, -42, tags, false, "10m", + "sum", null).joinUninterruptibly(); + assertEquals(-42, storage.getColumn(interval.getTemporalTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)0, 0, + interval))[0]); + + // 2 bytes + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 257, tags, false, "10m", + "sum", null).joinUninterruptibly(); + assertEquals(257, Bytes.getShort(storage.getColumn(interval.getTemporalTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)1, 0, + interval)))); + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, -257, tags, false, "10m", + "sum", null).joinUninterruptibly(); + assertEquals(-257, Bytes.getShort(storage.getColumn(interval.getTemporalTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)1, 0, + interval)))); + + // 4 bytes + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 65537, tags, false, "10m", + "sum", null).joinUninterruptibly(); + assertEquals(65537, Bytes.getInt(storage.getColumn(interval.getTemporalTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)3, 0, + interval)))); + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, -65537, tags, false, "10m", + "sum", null).joinUninterruptibly(); + assertEquals(-65537, Bytes.getInt(storage.getColumn(interval.getTemporalTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)3, 0, + interval)))); + + // 8 bytes + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 4294967296L, tags, false, "10m", + "sum", null).joinUninterruptibly(); + assertEquals(4294967296L, Bytes.getLong(storage.getColumn(interval.getTemporalTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)7, 0, + interval)))); + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, -4294967296L, tags, false, "10m", + "sum", null).joinUninterruptibly(); + assertEquals(-4294967296L, Bytes.getLong(storage.getColumn(interval.getTemporalTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)7, 0, + interval)))); + } + + @Test + public void addAggregatePointFloats() throws Exception { + final RollupInterval interval = rollup_config.getRollupInterval("10m"); + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 0.0F, tags, false, "10m", + "sum", null).joinUninterruptibly(); + assertEquals(0.0, Float.intBitsToFloat(Bytes.getInt( + storage.getColumn(interval.getTemporalTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)11, 0, + interval)))), 0.0001); + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 42.5F, tags, false, "10m", + "sum", null).joinUninterruptibly(); + assertEquals(42.5, Float.intBitsToFloat(Bytes.getInt( + storage.getColumn(interval.getTemporalTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)11, 0, + interval)))), 0.0001); + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, -42.5F, tags, false, "10m", + "sum", null).joinUninterruptibly(); + assertEquals(-42.5, Float.intBitsToFloat(Bytes.getInt( + storage.getColumn(interval.getTemporalTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)11, 0, + interval)))), 0.0001); + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 42.5123459999F, tags, false, "10m", + "sum", null).joinUninterruptibly(); + assertEquals(42.5123459999F, Float.intBitsToFloat(Bytes.getInt( + storage.getColumn(interval.getTemporalTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)11, 0, + interval)))), 0.0000001); + } + + @Test + public void addAggregatePointGroupByRollupRouting() throws Exception { + row = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_STRING, + agg_tag_key, "SUM"); + + assertNull(tags.get(agg_tag_key)); + RollupInterval interval = rollup_config.getRollupInterval("10m"); + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 42, tags, true, "10m", + "sum", "sum").joinUninterruptibly(); + + assertEquals(42, storage.getColumn(interval.getGroupbyTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)0, 0, + interval))[0]); + // make sure it didn't get into the tsdb table OR rollup table + assertNull(storage.getColumn(TSDB_TABLE, + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)0, 0, + interval))); + assertNull(storage.getColumn(interval.getTemporalTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)0, 0, + interval))); + + storage.flushStorage(); + + interval = rollup_config.getRollupInterval("1h"); + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 42, tags, true, "1h", + "sum", "sum").joinUninterruptibly(); + + assertEquals(42, storage.getColumn(interval.getGroupbyTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)0, 0, + interval))[0]); + assertNull(storage.getColumn(TSDB_TABLE, + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)0, 0, + interval))); + assertNull(storage.getColumn(interval.getTemporalTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)0, 0, + interval))); + + storage.flushStorage(); + + interval = rollup_config.getRollupInterval("1d"); + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 42, tags, true, "1d", + "sum", "sum").joinUninterruptibly(); + + assertEquals(42, storage.getColumn(interval.getGroupbyTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)0, 0, + interval))[0]); + assertNull(storage.getColumn(TSDB_TABLE, + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)0, 0, + interval))); + assertNull(storage.getColumn(interval.getTemporalTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)0, 0, + interval))); + + storage.flushStorage(); + tags.remove(agg_tag_key); + + // other aggs + row = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_STRING, + agg_tag_key, "MAX"); + RowKey.prefixKeyWithSalt(row); + interval = rollup_config.getRollupInterval("1h"); + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 42, tags, true, "1h", + "max", "max").joinUninterruptibly(); + assertEquals(42, storage.getColumn(interval.getGroupbyTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)0, 2, + interval))[0]); + + storage.flushStorage(); + tags.remove(agg_tag_key); + + row = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_STRING, + agg_tag_key, "MIN"); + RowKey.prefixKeyWithSalt(row); + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 42, tags, true, "1h", + "min", "min").joinUninterruptibly(); + assertEquals(42, storage.getColumn(interval.getGroupbyTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)0, 3, + interval))[0]); + + storage.flushStorage(); + tags.remove(agg_tag_key); + + row = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_STRING, + agg_tag_key, "COUNT"); + RowKey.prefixKeyWithSalt(row); + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 42, tags, true, "1h", + "count", "count").joinUninterruptibly(); + assertEquals(42, storage.getColumn(interval.getGroupbyTable(), + row, FAMILY, RollupUtils.buildRollupQualifier(1356998400, (short)0, 1, + interval))[0]); + } + + @Test (expected = IllegalArgumentException.class) + public void addAggregatePointGroupByRollupNoSuchAgg() throws Exception { + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 42, tags, true, "10m", + "nosuchagg", "sum").joinUninterruptibly(); + } + + @Test (expected = IllegalArgumentException.class) + public void addAggregatePointGroupByNoSuchAgg() throws Exception { + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 42, tags, true, null, + null, "nosuchagg").joinUninterruptibly(); + } + + @Test + public void addAggregatePointGroupBy() throws Exception { + storage.flushStorage(); + tags.remove(agg_tag_key); + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 42, tags, true, null, + null, "sum").joinUninterruptibly(); + row = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_STRING, + agg_tag_key, "SUM"); + storage.dumpToSystemOut(); + assertEquals(42, storage.getColumn(AGG_TABLE, row, FAMILY, + new byte[] { 0, 0 })[0]); + assertNull(storage.getColumn(TSDB_TABLE, row, FAMILY, + new byte[] { 0, 0 })); + assertNull(storage.getColumn( + rollup_config.getRollupInterval("10m").getGroupbyTable(), row, FAMILY, + new byte[] { 0, 0 })); + + storage.flushStorage(); + tags.remove(agg_tag_key); + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400, 24, tags, true, null, + null, "count").joinUninterruptibly(); + row = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_STRING, + agg_tag_key, "COUNT"); + RowKey.prefixKeyWithSalt(row); + assertEquals(24, storage.getColumn(AGG_TABLE, row, FAMILY, + new byte[] { 0, 0 })[0]); + assertNull(storage.getColumn(TSDB_TABLE, row, FAMILY, + new byte[] { 0, 0 })); + assertNull(storage.getColumn( + rollup_config.getRollupInterval("10m").getGroupbyTable(), row, FAMILY, + new byte[] { 0, 0 })); + } + + @Test + public void dpFilterOK() throws Exception { + final WriteableDataPointFilterPlugin filter = + mock(WriteableDataPointFilterPlugin.class); + when(filter.filterDataPoints()).thenReturn(true); + when(filter.allowDataPoint(eq(METRIC_STRING), anyLong(), + any(byte[].class), eq(tags), anyShort())) + .thenReturn(Deferred.<Boolean>fromResult(true)); + Whitebox.setInternalState(tsdb, "ts_filter", filter); + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400L, 42, tags, false, + "10m", "sum", null).joinUninterruptibly(); + + final byte[] qualifier = new byte[] {0, 0, 0}; + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("10m").getTemporalTable(), + row, FAMILY, qualifier); + final byte[] expected = {0x2A}; + assertArrayEquals(expected, value); + } + + @Test + public void uidFilterBlocked() throws Exception { + final WriteableDataPointFilterPlugin filter = + mock(WriteableDataPointFilterPlugin.class); + when(filter.filterDataPoints()).thenReturn(true); + when(filter.allowDataPoint(eq(METRIC_STRING), anyLong(), + any(byte[].class), eq(tags), anyShort())) + .thenReturn(Deferred.<Boolean>fromResult(false)); + Whitebox.setInternalState(tsdb, "ts_filter", filter); + + tsdb.addAggregatePoint(METRIC_STRING, 1356998400L, 42, tags, false, + "10m", "sum", null).joinUninterruptibly(); + + final byte[] row = new byte[] { 0, 0, 0, 0, 1, + 0x50, (byte) 0xE2, 0x27, 0, 0, 0, 0, 1, 0, 0, 0, 1}; + RowKey.prefixKeyWithSalt(row); + final byte[] qualifier = new byte[] {0, 0, 0}; + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("10m").getTemporalTable(), + row, FAMILY, qualifier); + assertNull(value); + } + + @Test + public void dpFilterReturnsException() throws Exception { + final WriteableDataPointFilterPlugin filter = + mock(WriteableDataPointFilterPlugin.class); + when(filter.filterDataPoints()).thenReturn(true); + when(filter.allowDataPoint(eq(METRIC_STRING), anyLong(), + any(byte[].class), eq(tags), anyShort())) + .thenReturn(Deferred.<Boolean>fromError(new UnitTestException("Boo!"))); + Whitebox.setInternalState(tsdb, "ts_filter", filter); + + final Deferred<Object> deferred = tsdb.addAggregatePoint(METRIC_STRING, + 1356998400L, 42, tags, false, "10m", "sum", null); + + try { + deferred.join(); + fail("Expected an UnitTestException"); + } catch (UnitTestException e) { }; + final byte[] row = new byte[] { 0, 0, 0, 0, 1, + 0x50, (byte) 0xE2, 0x27, 0, 0, 0, 0, 1, 0, 0, 0, 1}; + RowKey.prefixKeyWithSalt(row); + final byte[] qualifier = new byte[] {0, 0, 0}; + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("10m").getTemporalTable(), + row, FAMILY, qualifier); + assertNull(value); + } + + @Test + public void dpFilterThrowsException() throws Exception { + final WriteableDataPointFilterPlugin filter = + mock(WriteableDataPointFilterPlugin.class); + when(filter.filterDataPoints()).thenReturn(true); + when(filter.allowDataPoint(eq(METRIC_STRING), anyLong(), + any(byte[].class), eq(tags), anyShort())) + .thenThrow(new UnitTestException("Boo!")); + Whitebox.setInternalState(tsdb, "ts_filter", filter); + + try { + tsdb.addAggregatePoint(METRIC_STRING, + 1356998400L, 42, tags, false, "10m", "sum", null); + fail("Expected an UnitTestException"); + } catch (UnitTestException e) { }; + final byte[] row = new byte[] { 0, 0, 0, 0, 1, + 0x50, (byte) 0xE2, 0x27, 0, 0, 0, 0, 1, 0, 0, 0, 1}; + RowKey.prefixKeyWithSalt(row); + final byte[] qualifier = new byte[] {0, 0, 0}; + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("10m").getTemporalTable(), + row, FAMILY, qualifier); + assertNull(value); + } +} diff --git a/test/core/TestTSDBAddAggregatePointSalted.java b/test/core/TestTSDBAddAggregatePointSalted.java new file mode 100644 index 0000000000..6b3642c8cc --- /dev/null +++ b/test/core/TestTSDBAddAggregatePointSalted.java @@ -0,0 +1,97 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Before; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +import net.opentsdb.rollup.RollupConfig; +import net.opentsdb.rollup.RollupInterval; +import net.opentsdb.storage.MockBase; + +/** + * Integration test that runs all of the tests in {@see TestTSDBAddAggregatePoint} + * but with salting enabled just to verify nothing goes wrong with the extra + * salt bytes. + */ +@RunWith(PowerMockRunner.class) +public class TestTSDBAddAggregatePointSalted extends TestTSDBAddAggregatePoint { + + @Before + public void beforeLocal() throws Exception { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(2); + PowerMockito.when(Const.MAX_NUM_TAGS()).thenReturn((short) 8); + + agg_tag_key = config.getString("tsd.rollups.agg_tag_key"); + + storage = new MockBase(tsdb, client, true, true, true, true); + final List<byte[]> families = new ArrayList<byte[]>(); + families.add(FAMILY); + + storage.addTable("tsdb-rollup-10m".getBytes(), families); + storage.addTable("tsdb-rollup-agg-10m".getBytes(), families); + storage.addTable("tsdb-rollup-1h".getBytes(), families); + storage.addTable("tsdb-rollup-agg-1h".getBytes(), families); + storage.addTable("tsdb-rollup-1d".getBytes(), families); + storage.addTable("tsdb-rollup-agg-1d".getBytes(), families); + storage.addTable(AGG_TABLE, families); + + rollup_config = RollupConfig.builder() + .addAggregationId("sum", 0) + .addAggregationId("count", 1) + .addAggregationId("max", 2) + .addAggregationId("min", 3) + .addInterval(RollupInterval.builder() + .setTable("tsdb") + .setPreAggregationTable("tsdb-agg") + .setInterval("1m") + .setRowSpan("1h") + .setDefaultInterval(true)) + .addInterval(RollupInterval.builder() + .setTable("tsdb-rollup-10m") + .setPreAggregationTable("tsdb-rollup-agg-10m") + .setInterval("10m") + .setRowSpan("1d")) + .addInterval(RollupInterval.builder() + .setTable("tsdb-rollup-1h") + .setPreAggregationTable("tsdb-rollup-agg-1h") + .setInterval("1h") + .setRowSpan("1n")) + .addInterval(RollupInterval.builder() + .setTable("tsdb-rollup-1d") + .setPreAggregationTable("tsdb-rollup-agg-1d") + .setInterval("1d") + .setRowSpan("1y")) + .build(); + Whitebox.setInternalState(tsdb, "rollup_config", rollup_config); + Whitebox.setInternalState(tsdb, "default_interval", + rollup_config.getRollupInterval("1m")); + Whitebox.setInternalState(tsdb, "rollups_block_derived", true); + Whitebox.setInternalState(tsdb, "rollups_split_queries", false); + Whitebox.setInternalState(tsdb, "agg_tag_key", + config.getString("tsd.rollups.agg_tag_key")); + Whitebox.setInternalState(tsdb, "raw_agg_tag_value", + config.getString("tsd.rollups.raw_agg_tag_value")); + setupGroupByTagValues(); + + row = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_STRING); + } +} diff --git a/test/core/TestTSDBAddHistogramPoint.java b/test/core/TestTSDBAddHistogramPoint.java new file mode 100644 index 0000000000..db3b3f5f3c --- /dev/null +++ b/test/core/TestTSDBAddHistogramPoint.java @@ -0,0 +1,97 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import com.stumbleupon.async.Deferred; +import net.opentsdb.storage.MockBase; +import net.opentsdb.tsd.RTPublisher; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Mockito.*; + +public class TestTSDBAddHistogramPoint extends BaseTsdbTest { + + private static final byte HISTOGRAM_PREFIX = 0x6; + + @Before + public void beforeLocal() throws Exception { + storage = new MockBase(tsdb, client, true, true, true, true); + } + + @Test + public void addHistogramPoint() throws Exception { + byte[] testRawValue = "Test Raw Value".getBytes(); + tsdb.addHistogramPoint(METRIC_STRING, 1356998400, testRawValue, tags) + .joinUninterruptibly(); + final byte[] row = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_STRING); + byte[] qualifier = Internal.getQualifier(1356998400, HISTOGRAM_PREFIX); + final byte[] value = storage.getColumn(row, qualifier); + assertNotNull(value); + assertArrayEquals(testRawValue, value); + } + + @Test (expected = IllegalArgumentException.class) + public void addHistogramPointShortRawData() throws Exception { + byte[] testRawValue = new byte[0]; + tsdb.addHistogramPoint(METRIC_STRING, 1356998400, testRawValue, tags) + .joinUninterruptibly(); + } + + @Test (expected = IllegalArgumentException.class) + public void addHistogramPointNullRawData() throws Exception { + tsdb.addHistogramPoint(METRIC_STRING, 1356998400, null, tags) + .joinUninterruptibly(); + } + + @Test + public void addHistogramPointCallRTPublisher() throws Exception { + byte[] raw_data = new byte[5]; + RTPublisher rt_publisher = mock(RTPublisher.class); + setField(tsdb, "rt_publisher", rt_publisher); + tsdb.addHistogramPoint(METRIC_STRING, 1356998400, raw_data, tags) + .joinUninterruptibly(); + + byte[] tsuid = new byte[]{ 0, 0, 1, 0, 0, 1, 0, 0, 1}; + verify(rt_publisher, times(1)).publishHistogramPoint(METRIC_STRING, + 1356998400, raw_data, tags, tsuid); + } + + @Test + public void addHistogramPointTSFilterPlugin() throws Exception { + byte[] raw_data = new byte[5]; + WriteableDataPointFilterPlugin ts_filter = + mock(WriteableDataPointFilterPlugin.class); + when(ts_filter.filterDataPoints()).thenReturn(true); + when(ts_filter.allowHistogramPoint(METRIC_STRING, 1356998400, raw_data, tags)). + thenReturn(Deferred.fromResult(true)); + setField(tsdb, "ts_filter", ts_filter); + tsdb.addHistogramPoint(METRIC_STRING, 1356998400, raw_data, tags) + .joinUninterruptibly(); + + verify(ts_filter, times(1)).allowHistogramPoint(METRIC_STRING, + 1356998400, raw_data, tags); + } + + private static void setField(TSDB tsdb, String field_name, Object field_value) + throws NoSuchFieldException, IllegalAccessException { + Field field = TSDB.class.getDeclaredField(field_name); + field.setAccessible(true); + field.set(tsdb, field_value); + } + +} diff --git a/test/core/TestTSDBAddPoint.java b/test/core/TestTSDBAddPoint.java new file mode 100644 index 0000000000..fd8b5cbfb2 --- /dev/null +++ b/test/core/TestTSDBAddPoint.java @@ -0,0 +1,590 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.anyShort; +import static org.mockito.Matchers.eq; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.HashMap; +import java.util.Map.Entry; +import java.util.TreeMap; + +import org.hbase.async.Bytes; +import org.junit.Before; +import org.junit.Test; +import org.powermock.reflect.Whitebox; + +import com.stumbleupon.async.Deferred; + +import net.opentsdb.uid.NoSuchUniqueName; + +public class TestTSDBAddPoint extends BaseTsdbTest { + protected byte[] row; + + @Before + public void beforeLocal() throws Exception { + row = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_STRING); + setDataPointStorage(); + } + + @Test + public void addPointLong1Byte() throws Exception { + tsdb.addPoint(METRIC_STRING, 1356998400, 42, tags).joinUninterruptibly(); + final byte[] value = storage.getColumn(row, new byte[] { 0, 0 }); + assertNotNull(value); + assertEquals(42, value[0]); + } + + @Test + public void addPointLong1ByteNegative() throws Exception { + tsdb.addPoint(METRIC_STRING, 1356998400, -42, tags).joinUninterruptibly(); + final byte[] value = storage.getColumn(row, new byte[] { 0, 0 }); + assertNotNull(value); + assertEquals(-42, value[0]); + } + + @Test + public void addPointLong2Bytes() throws Exception { + tsdb.addPoint(METRIC_STRING, 1356998400, 257, tags).joinUninterruptibly(); + final byte[] value = storage.getColumn(row, new byte[] { 0, 1 }); + assertNotNull(value); + assertEquals(257, Bytes.getShort(value)); + } + + @Test + public void addPointLong2BytesNegative() throws Exception { + tsdb.addPoint(METRIC_STRING, 1356998400, -257, tags).joinUninterruptibly(); + + final byte[] value = storage.getColumn(row, new byte[] { 0, 1 }); + assertNotNull(value); + assertEquals(-257, Bytes.getShort(value)); + } + + @Test + public void addPointLong4Bytes() throws Exception { + tsdb.addPoint(METRIC_STRING, 1356998400, 65537, tags).joinUninterruptibly(); + + final byte[] value = storage.getColumn(row, new byte[] { 0, 3 }); + assertNotNull(value); + assertEquals(65537, Bytes.getInt(value)); + } + + @Test + public void addPointLong4BytesNegative() throws Exception { + tsdb.addPoint(METRIC_STRING, 1356998400, -65537, tags).joinUninterruptibly(); + + final byte[] value = storage.getColumn(row, new byte[] { 0, 3 }); + assertNotNull(value); + assertEquals(-65537, Bytes.getInt(value)); + } + + @Test + public void addPointLong8Bytes() throws Exception { + tsdb.addPoint(METRIC_STRING, 1356998400, 4294967296L, tags).joinUninterruptibly(); + + final byte[] value = storage.getColumn(row, new byte[] { 0, 7 }); + assertNotNull(value); + assertEquals(4294967296L, Bytes.getLong(value)); + } + + @Test + public void addPointLong8BytesNegative() throws Exception { + tsdb.addPoint(METRIC_STRING, 1356998400, -4294967296L, tags).joinUninterruptibly(); + + final byte[] value = storage.getColumn(row, new byte[] { 0, 7 }); + assertNotNull(value); + assertEquals(-4294967296L, Bytes.getLong(value)); + } + + @Test + public void addPointLongMs() throws Exception { + tsdb.addPoint(METRIC_STRING, 1356998400500L, 42, tags).joinUninterruptibly(); + + final byte[] value = storage.getColumn(row, + new byte[] { (byte) 0xF0, 0, 0x7D, 0 }); + assertNotNull(value); + assertEquals(42, value[0]); + } + + @Test + public void addPointLongMany() throws Exception { + long timestamp = 1356998400; + for (int i = 1; i <= 50; i++) { + tsdb.addPoint(METRIC_STRING, timestamp++, i, tags).joinUninterruptibly(); + } + final byte[] value = storage.getColumn(row, new byte[] { 0, 0 }); + assertNotNull(value); + assertEquals(1, value[0]); + assertEquals(50, storage.numColumns(row)); + } + + @Test + public void addPointLongManyMs() throws Exception { + long timestamp = 1356998400500L; + for (int i = 1; i <= 50; i++) { + tsdb.addPoint(METRIC_STRING, timestamp++, i, tags).joinUninterruptibly(); + } + final byte[] value = storage.getColumn(row, + new byte[] { (byte) 0xF0, 0, 0x7D, 0 }); + assertNotNull(value); + assertEquals(1, value[0]); + assertEquals(50, storage.numColumns(row)); + } + + @Test + public void addPointLongEndOfRow() throws Exception { + tsdb.addPoint(METRIC_STRING, 1357001999, 42, tags).joinUninterruptibly(); + final byte[] value = storage.getColumn(row, new byte[] { (byte) 0xE0, + (byte) 0xF0 }); + assertNotNull(value); + assertEquals(42, value[0]); + } + + @Test + public void addPointLongOverwrite() throws Exception { + tsdb.addPoint(METRIC_STRING, 1356998400, 42, tags).joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, 1356998400, 24, tags).joinUninterruptibly(); + final byte[] value = storage.getColumn(row, new byte[] { 0, 0 }); + assertNotNull(value); + assertEquals(24, value[0]); + } + + @Test (expected = NoSuchUniqueName.class) + public void addPointNoAutoMetric() throws Exception { + tsdb.addPoint(NSUN_METRIC, 1356998400, 42, tags).joinUninterruptibly(); + } + + @Test + public void addPointSecondZero() throws Exception { + // Thu, 01 Jan 1970 00:00:00 GMT + row = getRowKey(METRIC_STRING, 0, TAGK_STRING, TAGV_STRING); + tsdb.addPoint(METRIC_STRING, 0, 42, tags).joinUninterruptibly(); + final byte[] value = storage.getColumn(row, new byte[] { 0, 0 }); + assertNotNull(value); + assertEquals(42, value[0]); + } + + @Test + public void addPointSecondOne() throws Exception { + // hey, it's valid *shrug* Thu, 01 Jan 1970 00:00:01 GMT + row = getRowKey(METRIC_STRING, 0, TAGK_STRING, TAGV_STRING); + tsdb.addPoint(METRIC_STRING, 1, 42, tags).joinUninterruptibly(); + final byte[] value = storage.getColumn(row, new byte[] { 0, 16 }); + assertNotNull(value); + assertEquals(42, value[0]); + } + + @Test + public void addPointSecond2106() throws Exception { + // Sun, 07 Feb 2106 06:28:15 GMT + row = getRowKey(METRIC_STRING, (int) 4294965600L, TAGK_STRING, TAGV_STRING); + tsdb.addPoint(METRIC_STRING, 4294967295L, 42, tags).joinUninterruptibly(); + final byte[] value = storage.getColumn(row, new byte[] { 0x69, (byte) 0xF0 }); + assertNotNull(value); + assertEquals(42, value[0]); + } + + @Test (expected = IllegalArgumentException.class) + public void addPointSecondNegative() throws Exception { + // Fri, 13 Dec 1901 20:45:52 GMT + // may support in the future, but 1.0 didn't + tsdb.addPoint(METRIC_STRING, -2147483648, 42, tags).joinUninterruptibly(); + } + + @Test (expected = IllegalArgumentException.class) + public void emptyTagValue() throws Exception { + tags.put(TAGK_STRING, ""); + tsdb.addPoint(METRIC_STRING, 1234567890, 42, tags).joinUninterruptibly(); + } + + @Test + public void addPointMS1970() throws Exception { + // Since it's just over Integer.MAX_VALUE, OpenTSDB will treat this as + // a millisecond timestamp since it doesn't fit in 4 bytes. + // Base time is 4294800 which is Thu, 19 Feb 1970 17:00:00 GMT + // offset = F0A36000 or 167296 ms + row = getRowKey(METRIC_STRING, 4294800, TAGK_STRING, TAGV_STRING); + tsdb.addPoint(METRIC_STRING, 4294967296L, 42, tags).joinUninterruptibly(); + final byte[] value = storage.getColumn(row, new byte[] { (byte) 0xF0, + (byte) 0xA3, 0x60, 0}); + assertNotNull(value); + assertEquals(42, value[0]); + } + + @Test + public void addPointMS2106() throws Exception { + // Sun, 07 Feb 2106 06:28:15.000 GMT + row = getRowKey(METRIC_STRING, (int) 4294965600L, TAGK_STRING, TAGV_STRING); + tsdb.addPoint(METRIC_STRING, 4294967295000L, 42, tags).joinUninterruptibly(); + final byte[] value = storage.getColumn(row, new byte[] { (byte) 0xF6, + (byte) 0x77, 0x46, 0 }); + assertNotNull(value); + assertEquals(42, value[0]); + } + + @Test + public void addPointMS2286() throws Exception { + // It's an artificial limit and more thought needs to be put into it + // TODO - and doesn't conform anyways, bad timestamp! + row = getRowKey(METRIC_STRING, 1410062608, TAGK_STRING, TAGV_STRING); + tsdb.addPoint(METRIC_STRING, 9999999999999L, 42, tags).joinUninterruptibly(); + final byte[] value = storage.getColumn(row, new byte[] { (byte) 0xFA, + (byte) 0xAE, 0x5F, (byte) 0xC0 }); + assertNotNull(value); + assertEquals(42, value[0]); + } + + @Test (expected = IllegalArgumentException.class) + public void addPointMSTooLarge() throws Exception { + // It's an artificial limit and more thought needs to be put into it + tsdb.addPoint(METRIC_STRING, 10000000000000L, 42, tags).joinUninterruptibly(); + } + + @Test (expected = IllegalArgumentException.class) + public void addPointMSNegative() throws Exception { + // Fri, 13 Dec 1901 20:45:52 GMT + // may support in the future, but 1.0 didn't + HashMap<String, String> tags = new HashMap<String, String>(1); + tags.put("host", "web01"); + tsdb.addPoint(METRIC_STRING, -2147483648000L, 42, tags).joinUninterruptibly(); + } + + @Test + public void addPointFloat() throws Exception { + tsdb.addPoint(METRIC_STRING, 1356998400, 42.5F, tags).joinUninterruptibly(); + + final byte[] value = storage.getColumn(row, new byte[] { 0, 11 }); + assertNotNull(value); + // should have 7 digits of precision + assertEquals(42.5F, Float.intBitsToFloat(Bytes.getInt(value)), 0.0000001); + } + + @Test + public void addPointFloatNegative() throws Exception { + HashMap<String, String> tags = new HashMap<String, String>(1); + tags.put("host", "web01"); + tsdb.addPoint(METRIC_STRING, 1356998400, -42.5F, tags).joinUninterruptibly(); + + final byte[] value = storage.getColumn(row, new byte[] { 0, 11 }); + assertNotNull(value); + // should have 7 digits of precision + assertEquals(-42.5F, Float.intBitsToFloat(Bytes.getInt(value)), 0.0000001); + } + + @Test + public void addPointFloatMs() throws Exception { + tsdb.addPoint(METRIC_STRING, 1356998400500L, 42.5F, tags).joinUninterruptibly(); + + final byte[] value = storage.getColumn(row, + new byte[] { (byte) 0xF0, 0, 0x7D, 11 }); + assertNotNull(value); + // should have 7 digits of precision + assertEquals(42.5F, Float.intBitsToFloat(Bytes.getInt(value)), 0.0000001); + } + + @Test + public void addPointFloatEndOfRow() throws Exception { + HashMap<String, String> tags = new HashMap<String, String>(1); + tags.put("host", "web01"); + tsdb.addPoint(METRIC_STRING, 1357001999, 42.5F, tags).joinUninterruptibly(); + + final byte[] value = storage.getColumn(row, new byte[] { (byte) 0xE0, + (byte) 0xFB }); + assertNotNull(value); + // should have 7 digits of precision + assertEquals(42.5F, Float.intBitsToFloat(Bytes.getInt(value)), 0.0000001); + } + + @Test + public void addPointFloatPrecision() throws Exception { + tsdb.addPoint(METRIC_STRING, 1356998400, 42.5123459999F, tags) + .joinUninterruptibly(); + + final byte[] value = storage.getColumn(row, new byte[] { 0, 11 }); + assertNotNull(value); + // should have 7 digits of precision + assertEquals(42.512345F, Float.intBitsToFloat(Bytes.getInt(value)), 0.0000001); + } + + @Test + public void addPointFloatOverwrite() throws Exception { + tsdb.addPoint(METRIC_STRING, 1356998400, 42.5F, tags).joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, 1356998400, 25.4F, tags).joinUninterruptibly(); + + final byte[] value = storage.getColumn(row, new byte[] { 0, 11 }); + assertNotNull(value); + // should have 7 digits of precision + assertEquals(25.4F, Float.intBitsToFloat(Bytes.getInt(value)), 0.0000001); + } + + @Test + public void addPointBothSameTimeIntAndFloat() throws Exception { + // this is an odd situation that can occur if the user puts an int and then + // a float (or vice-versa) with the same timestamp. What happens in the + // aggregators when this occurs? + tsdb.addPoint(METRIC_STRING, 1356998400, 42, tags).joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, 1356998400, 42.5F, tags).joinUninterruptibly(); + + byte[] value = storage.getColumn(row, new byte[] { 0, 0 }); + assertEquals(2, storage.numColumns(row)); + assertNotNull(value); + assertEquals(42, value[0]); + value = storage.getColumn(row, new byte[] { 0, 11 }); + assertNotNull(value); + // should have 7 digits of precision + assertEquals(42.5F, Float.intBitsToFloat(Bytes.getInt(value)), 0.0000001); + } + + @Test + public void addPointBothSameTimeIntAndFloatMs() throws Exception { + // this is an odd situation that can occur if the user puts an int and then + // a float (or vice-versa) with the same timestamp. What happens in the + // aggregators when this occurs? + tsdb.addPoint(METRIC_STRING, 1356998400500L, 42, tags).joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, 1356998400500L, 42.5F, tags).joinUninterruptibly(); + + byte[] value = storage.getColumn(row, new byte[] { (byte) 0xF0, 0, 0x7D, 0 }); + assertEquals(2, storage.numColumns(row)); + assertNotNull(value); + assertEquals(42, value[0]); + value = storage.getColumn(row, new byte[] { (byte) 0xF0, 0, 0x7D, 11 }); + assertNotNull(value); + // should have 7 digits of precision + assertEquals(42.5F, Float.intBitsToFloat(Bytes.getInt(value)), 0.0000001); + } + + @Test + public void addPointBothSameTimeSecondAndMs() throws Exception { + // this can happen if a second and an ms data point are stored for the same + // timestamp. + tsdb.addPoint(METRIC_STRING, 1356998400L, 42, tags).joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, 1356998400000L, 42, tags).joinUninterruptibly(); + + byte[] value = storage.getColumn(row, new byte[] { 0, 0 }); + assertEquals(2, storage.numColumns(row)); + assertNotNull(value); + assertEquals(42, value[0]); + value = storage.getColumn(row, new byte[] { (byte) 0xF0, 0, 0, 0 }); + assertNotNull(value); + // should have 7 digits of precision + assertEquals(42, value[0]); + } + + @Test + public void addPointAppend() throws Exception { + Whitebox.setInternalState(config, "enable_appends", true); + + tsdb.addPoint(METRIC_STRING, 1356998400, 42, tags).joinUninterruptibly(); + + final byte[] value = storage.getColumn(row, + AppendDataPoints.APPEND_COLUMN_QUALIFIER); + assertArrayEquals(new byte[] { 0, 0, 42 }, value); + } + + @Test + public void addPointAppendWithOffset() throws Exception { + Whitebox.setInternalState(config, "enable_appends", true); + + tsdb.addPoint(METRIC_STRING, 1356998430, 42, tags).joinUninterruptibly(); + + final byte[] value = storage.getColumn(row, + AppendDataPoints.APPEND_COLUMN_QUALIFIER); + assertArrayEquals(new byte[] { 1, -32, 42 }, value); + } + + @Test + public void addPointAppendAppending() throws Exception { + Whitebox.setInternalState(config, "enable_appends", true); + + tsdb.addPoint(METRIC_STRING, 1356998400, 42, tags).joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, 1356998430, 24, tags).joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, 1356998460, 1, tags).joinUninterruptibly(); + + final byte[] value = storage.getColumn(row, + AppendDataPoints.APPEND_COLUMN_QUALIFIER); + assertArrayEquals(new byte[] { 0, 0, 42, 1, -32, 24, 3, -64, 1 }, value); + } + + @Test + public void addPointAppendAppendingOutOfOrder() throws Exception { + Whitebox.setInternalState(config, "enable_appends", true); + + tsdb.addPoint(METRIC_STRING, 1356998400, 42, tags).joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, 1356998460, 1, tags).joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, 1356998430, 24, tags).joinUninterruptibly(); + + final byte[] value = storage.getColumn(row, + AppendDataPoints.APPEND_COLUMN_QUALIFIER); + assertArrayEquals(new byte[] { 0, 0, 42, 3, -64, 1, 1, -32, 24 }, value); + } + + @Test + public void addPointAppendAppendingDuplicates() throws Exception { + Whitebox.setInternalState(config, "enable_appends", true); + + tsdb.addPoint(METRIC_STRING, 1356998400, 42, tags).joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, 1356998430, 24, tags).joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, 1356998430, 1, tags).joinUninterruptibly(); + + final byte[] value = storage.getColumn(row, + AppendDataPoints.APPEND_COLUMN_QUALIFIER); + assertArrayEquals(new byte[] { 0, 0, 42, 1, -32, 24, 1, -32, 1 }, value); + } + + @Test + public void addPointAppendMS() throws Exception { + Whitebox.setInternalState(config, "enable_appends", true); + + tsdb.addPoint(METRIC_STRING, 1356998400050L, 42, tags).joinUninterruptibly(); + + final byte[] value = storage.getColumn(row, + AppendDataPoints.APPEND_COLUMN_QUALIFIER); + assertArrayEquals(new byte[] { (byte) 0xF0, 0, 12, -128, 42 }, value); + } + + @Test + public void addPointAppendAppendingMixMS() throws Exception { + Whitebox.setInternalState(config, "enable_appends", true); + + tsdb.addPoint(METRIC_STRING, 1356998400, 42, tags).joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, 1356998400050L, 1, tags).joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, 1356998430, 24, tags).joinUninterruptibly(); + + final byte[] value = storage.getColumn(row, + AppendDataPoints.APPEND_COLUMN_QUALIFIER); + assertArrayEquals(new byte[] { + 0, 0, 42, (byte) 0xF0, 0, 12, -128, 1, 1, -32, 24 }, value); + } + + @Test + public void dpFilterOK() throws Exception { + final WriteableDataPointFilterPlugin filter = + mock(WriteableDataPointFilterPlugin.class); + when(filter.filterDataPoints()).thenReturn(true); + when(filter.allowDataPoint(eq(METRIC_STRING), anyLong(), + any(byte[].class), eq(tags), anyShort())) + .thenReturn(Deferred.<Boolean>fromResult(true)); + Whitebox.setInternalState(tsdb, "ts_filter", filter); + + tsdb.addPoint(METRIC_STRING, 1356998400, 42, tags).joinUninterruptibly(); + final byte[] value = storage.getColumn(row, new byte[] { 0, 0 }); + assertNotNull(value); + assertEquals(42, value[0]); + + verify(filter, times(1)).filterDataPoints(); + verify(filter, times(1)).allowDataPoint(eq(METRIC_STRING), anyLong(), + any(byte[].class), eq(tags), anyShort()); + } + + @Test + public void dpFilterBlocked() throws Exception { + final WriteableDataPointFilterPlugin filter = + mock(WriteableDataPointFilterPlugin.class); + when(filter.filterDataPoints()).thenReturn(true); + when(filter.allowDataPoint(eq(METRIC_STRING), anyLong(), + any(byte[].class), eq(tags), anyShort())) + .thenReturn(Deferred.<Boolean>fromResult(false)); + Whitebox.setInternalState(tsdb, "ts_filter", filter); + + tsdb.addPoint(METRIC_STRING, 1356998400, 42, tags).joinUninterruptibly(); + final byte[] value = storage.getColumn(row, new byte[] { 0, 0 }); + assertNull(value); + + verify(filter, times(1)).filterDataPoints(); + verify(filter, times(1)).allowDataPoint(eq(METRIC_STRING), anyLong(), + any(byte[].class), eq(tags), anyShort()); + } + + @Test + public void dpFilterReturnsException() throws Exception { + final WriteableDataPointFilterPlugin filter = + mock(WriteableDataPointFilterPlugin.class); + when(filter.filterDataPoints()).thenReturn(true); + when(filter.allowDataPoint(eq(METRIC_STRING), anyLong(), + any(byte[].class), eq(tags), anyShort())) + .thenReturn(Deferred.<Boolean>fromError(new UnitTestException("Boo!"))); + Whitebox.setInternalState(tsdb, "ts_filter", filter); + + final Deferred<Object> deferred = + tsdb.addPoint(METRIC_STRING, 1356998400, 42, tags); + try { + deferred.join(); + fail("Expected an UnitTestException"); + } catch (UnitTestException e) { }; + final byte[] value = storage.getColumn(row, new byte[] { 0, 0 }); + assertNull(value); + + verify(filter, times(1)).filterDataPoints(); + verify(filter, times(1)).allowDataPoint(eq(METRIC_STRING), anyLong(), + any(byte[].class), eq(tags), anyShort()); + } + + @Test + public void uidFilterThrowsException() throws Exception { + final WriteableDataPointFilterPlugin filter = + mock(WriteableDataPointFilterPlugin.class); + when(filter.filterDataPoints()).thenReturn(true); + when(filter.allowDataPoint(eq(METRIC_STRING), anyLong(), + any(byte[].class), eq(tags), anyShort())) + .thenThrow(new UnitTestException("Boo!")); + Whitebox.setInternalState(tsdb, "ts_filter", filter); + + try { + tsdb.addPoint(METRIC_STRING, 1356998400, 42, tags); + fail("Expected an UnitTestException"); + } catch (UnitTestException e) { }; + final byte[] value = storage.getColumn(row, new byte[] { 0, 0 }); + assertNull(value); + + verify(filter, times(1)).filterDataPoints(); + verify(filter, times(1)).allowDataPoint(eq(METRIC_STRING), anyLong(), + any(byte[].class), eq(tags), anyShort()); + } + + @Test + public void addPointWithOTSDBTimeStamp() throws Exception { + long ts = 1356998400; + tsdb.getConfig().overrideConfig("tsd.storage.use_otsdb_timestamp", "true"); + tsdb.addPoint(METRIC_STRING, ts, 42, tags).joinUninterruptibly(); + TreeMap<Long, byte[]> result = storage.getFullColumn(tsdb.dataTable(), row, tsdb.FAMILY(), new byte[] { 0, 0 }); + assert (result != null); + for (Entry<Long, byte[]> e : result.entrySet()) { + long retrievedTs = e.getKey(); + assert ((ts * 1000) == retrievedTs); + } + } + + @Test + public void addPointWithoutOTSDBTimeStamp() throws Exception { + long ts = 1356998400; + tsdb.getConfig().overrideConfig("tsd.storage.use_otsdb_timestamp", "false"); + tsdb.addPoint(METRIC_STRING, ts, 42, tags).joinUninterruptibly(); + TreeMap<Long, byte[]> result = storage.getFullColumn(tsdb.dataTable(), row, tsdb.FAMILY(), new byte[] { 0, 0 }); + assert(result != null); + for (Entry<Long, byte[]> e : result.entrySet()) { + long retrievedTs = e.getKey(); + assert((ts * 1000) != retrievedTs); + } + } +} \ No newline at end of file diff --git a/test/core/TestTSDBAddPointSalted.java b/test/core/TestTSDBAddPointSalted.java new file mode 100644 index 0000000000..2045556ac6 --- /dev/null +++ b/test/core/TestTSDBAddPointSalted.java @@ -0,0 +1,38 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import org.junit.Before; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.modules.junit4.PowerMockRunner; + +/** + * Integration test that runs all of the tests in {@see TestTSDBAddPoint} + * but with salting enabled just to verify nothing goes wrong with the extra + * salt bytes. + */ +@RunWith(PowerMockRunner.class) +public class TestTSDBAddPointSalted extends TestTSDBAddPoint { + + @Before + public void beforeLocal() throws Exception { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(2); + PowerMockito.when(Const.MAX_NUM_TAGS()).thenReturn((short) 8); + + row = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_STRING); + setDataPointStorage(); + } +} diff --git a/test/core/TestTSDBTableAvailability.java b/test/core/TestTSDBTableAvailability.java new file mode 100644 index 0000000000..e3d442db69 --- /dev/null +++ b/test/core/TestTSDBTableAvailability.java @@ -0,0 +1,171 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2013 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import java.util.ArrayList; +import java.util.List; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.Mock; +import org.mockito.stubbing.Answer; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +import org.hbase.async.AtomicIncrementRequest; +import org.hbase.async.GetRequest; +import org.hbase.async.HBaseClient; +import org.hbase.async.KeyValue; +import org.hbase.async.PutRequest; +import org.hbase.async.Scanner; +import org.hbase.async.RegionLocation; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.stumbleupon.async.Deferred; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ TSDB.class, HBaseClient.class, + CompactionQueue.class, GetRequest.class, PutRequest.class, KeyValue.class, + Scanner.class, AtomicIncrementRequest.class, Const.class, }) +public final class TestTSDBTableAvailability extends BaseTsdbTest { + + /** If locateRegions() throws an exception, availability is NONE */ + @Test + public void failedToGetRegions() throws Exception { + Deferred<List<RegionLocation>> d = new Deferred<List<RegionLocation>>(); + d.callback(new Exception()); + Deferred<List<RegionLocation>> d2 = new Deferred<List<RegionLocation>>(); + d2.callback(new Exception()); + TSDB tsdb = new TSDB(mock(HBaseClient.class), config); + when(tsdb.getClient() + .locateRegions(config.getString("tsd.storage.hbase.uid_table")) + ).thenReturn(d); + when(tsdb.getClient() + .locateRegions(config.getString("tsd.storage.hbase.data_table")) + ).thenReturn(d2); + assertEquals(tsdb.checkNecessaryTablesAvailability().join(), + TSDB.TableAvailability.NONE); + } + + /** If locateRegions() returns empty list, availability is NONE */ + @Test + public void noRegions() throws Exception { + TSDB tsdb = new TSDB(mock(HBaseClient.class), config); + String uid_table = config.getString("tsd.storage.hbase.uid_table"); + String data_table = config.getString("tsd.storage.hbase.data_table"); + + Deferred<List<RegionLocation>> d = new Deferred<List<RegionLocation>>(); + d.callback(new ArrayList<RegionLocation>()); + when(tsdb.getClient().locateRegions(uid_table)).thenReturn(d); + + Deferred<List<RegionLocation>> d2 = new Deferred<List<RegionLocation>>(); + d2.callback(new ArrayList<RegionLocation>()); + when(tsdb.getClient().locateRegions(data_table)).thenReturn(d2); + assertEquals(tsdb.checkNecessaryTablesAvailability().join(), + TSDB.TableAvailability.NONE); + } + + private TSDB createTSDB(ArrayList<Boolean> uid_regions, + ArrayList<Boolean> data_regions) { + TSDB original = new TSDB(mock(HBaseClient.class), config); + TSDB tsdb = PowerMockito.spy(original); + + Deferred<ArrayList<Boolean>> get_results = new Deferred<ArrayList<Boolean>>(); + get_results.callback(uid_regions); + Deferred<ArrayList<Boolean>> get_results2 = new Deferred<ArrayList<Boolean>>(); + get_results2.callback(data_regions); + + PowerMockito.doReturn(get_results).when(tsdb) + .getTableRegionAvailability("tsd.storage.hbase.uid_table"); + PowerMockito.doReturn(get_results2).when(tsdb) + .getTableRegionAvailability("tsd.storage.hbase.data_table"); + + return tsdb; + } + + /* If all returned regions return a result, availability is FULL. */ + @Test + public void allRegionsAvailable() throws Exception { + ArrayList<Boolean> region_availability = new ArrayList<Boolean>(); + region_availability.add(true); + + TSDB tsdb = createTSDB(region_availability, region_availability); + assertEquals(tsdb.checkNecessaryTablesAvailability().join(), + TSDB.TableAvailability.FULL); + } + + /* If one out of many regions returned by locateRegions(), one is unavailable, + availability is PARTIAL. */ + @Test + public void partialRegionsAvailable() throws Exception { + ArrayList<Boolean> region_availability = new ArrayList<Boolean>(); + region_availability.add(false); + region_availability.add(true); + + TSDB tsdb = createTSDB(region_availability, region_availability); + assertEquals(tsdb.checkNecessaryTablesAvailability().join(), + TSDB.TableAvailability.PARTIAL); + } + + /* If one table returns PARTIAL and the other returns FULL, final result is + PARTIAL. If one table returns NONE and the other returns FULL, final result + is NONE. If one returns PARTIAL and the other NONE, final is result is + NONE. */ + @Test + public void differentRegionsAvailable() throws Exception { + ArrayList<Boolean> full_availability = new ArrayList<Boolean>(); + full_availability.add(true); + full_availability.add(true); + ArrayList<Boolean> partial_availability = new ArrayList<Boolean>(); + partial_availability.add(false); + partial_availability.add(true); + ArrayList<Boolean> no_availability = new ArrayList<Boolean>(); + no_availability.add(false); + + assertEquals(createTSDB(full_availability, partial_availability) + .checkNecessaryTablesAvailability().join(), + TSDB.TableAvailability.PARTIAL); + assertEquals(createTSDB(partial_availability, full_availability) + .checkNecessaryTablesAvailability().join(), + TSDB.TableAvailability.PARTIAL); + assertEquals(createTSDB(full_availability, no_availability) + .checkNecessaryTablesAvailability().join(), + TSDB.TableAvailability.NONE); + assertEquals(createTSDB(no_availability, full_availability) + .checkNecessaryTablesAvailability().join(), + TSDB.TableAvailability.NONE); + assertEquals(createTSDB(partial_availability, no_availability) + .checkNecessaryTablesAvailability().join(), + TSDB.TableAvailability.NONE); + assertEquals(createTSDB(no_availability, partial_availability) + .checkNecessaryTablesAvailability().join(), + TSDB.TableAvailability.NONE); + } + + +} diff --git a/test/core/TestTSQuery.java b/test/core/TestTSQuery.java index 894ae23586..40f6622e84 100644 --- a/test/core/TestTSQuery.java +++ b/test/core/TestTSQuery.java @@ -13,10 +13,18 @@ package net.opentsdb.core; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.mockito.Mockito.when; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.*; +import static org.mockito.Mockito.*; import java.util.ArrayList; +import java.util.HashMap; + +import com.stumbleupon.async.Deferred; +import net.opentsdb.utils.DateTime; import org.junit.Test; import org.junit.runner.RunWith; @@ -25,7 +33,7 @@ import org.powermock.modules.junit4.PowerMockRunner; @RunWith(PowerMockRunner.class) -@PrepareForTest({ TSQuery.class }) +@PrepareForTest({ TSQuery.class, TsdbQuery.class, TSDB.class, SplitRollupQuery.class, DateTime.class }) public final class TestTSQuery { @Test @@ -40,11 +48,56 @@ public void validate() { assertEquals(1356998400000L, q.startTime()); assertEquals(1356998460000L, q.endTime()); assertEquals("sys.cpu.0", q.getQueries().get(0).getMetric()); - assertEquals("*", q.getQueries().get(0).getTags().get("host")); - assertEquals("lga", q.getQueries().get(0).getTags().get("dc")); + assertEquals("wildcard(*)", q.getQueries().get(0).getTags().get("host")); + assertEquals("literal_or(lga)", q.getQueries().get(0).getTags().get("dc")); assertEquals(Aggregators.SUM, q.getQueries().get(0).aggregator()); assertEquals(Aggregators.AVG, q.getQueries().get(0).downsampler()); assertEquals(300000, q.getQueries().get(0).downsampleInterval()); + assertNull(q.getTimezone()); + assertEquals("UTC", q.getQueries().get(0).downsamplingSpecification() + .getTimezone().getID()); + assertFalse(q.getQueries().get(0).downsamplingSpecification().useCalendar()); + } + + @Test + public void validateWithTimezone() { + TSQuery q = this.getMetricForValidate(); + q.setUseCalendar(true); + q.setTimezone("Pacific/Funafuti"); + q.validateAndSetQuery(); + assertEquals(1356998400000L, q.startTime()); + assertEquals(1356998460000L, q.endTime()); + assertEquals("sys.cpu.0", q.getQueries().get(0).getMetric()); + assertEquals("wildcard(*)", q.getQueries().get(0).getTags().get("host")); + assertEquals("literal_or(lga)", q.getQueries().get(0).getTags().get("dc")); + assertEquals(Aggregators.SUM, q.getQueries().get(0).aggregator()); + assertEquals(Aggregators.AVG, q.getQueries().get(0).downsampler()); + assertEquals(300000, q.getQueries().get(0).downsampleInterval()); + assertEquals("Pacific/Funafuti", q.getTimezone()); + assertEquals("Pacific/Funafuti", q.getQueries().get(0).downsamplingSpecification() + .getTimezone().getID()); + assertTrue(q.getQueries().get(0).downsamplingSpecification().useCalendar()); + } + + @Test + public void validateVerifyNoDSOverrideWithCalendar() { + TSQuery q = this.getMetricForValidate(); + q.setUseCalendar(true); + q.setTimezone("Pacific/Funafuti"); + q.getQueries().get(0).setDownsample(null); + q.validateAndSetQuery(); + assertEquals(1356998400000L, q.startTime()); + assertEquals(1356998460000L, q.endTime()); + assertEquals("sys.cpu.0", q.getQueries().get(0).getMetric()); + assertEquals("wildcard(*)", q.getQueries().get(0).getTags().get("host")); + assertEquals("literal_or(lga)", q.getQueries().get(0).getTags().get("dc")); + assertEquals(Aggregators.SUM, q.getQueries().get(0).aggregator()); + assertNull(q.getQueries().get(0).downsampler()); + assertEquals(0, q.getQueries().get(0).downsampleInterval()); + assertEquals("Pacific/Funafuti", q.getTimezone()); + assertEquals("UTC", q.getQueries().get(0).downsamplingSpecification() + .getTimezone().getID()); + assertFalse(q.getQueries().get(0).downsamplingSpecification().useCalendar()); } @Test (expected = IllegalArgumentException.class) @@ -102,6 +155,530 @@ public void validateEmptyQueries() { q.validateAndSetQuery(); } + // NOTE: Each of the hash and equals tests should make sure that we the code + // doesn't change after validation. + + @Test + public void testHashCodeandEqualsStart() { + TSQuery sub1 = getMetricForValidate(); + final int hash_a = sub1.hashCode(); + sub1.setStart("1356998300"); + final int hash_b = sub1.hashCode(); + assertTrue(hash_a != hash_b); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSQuery sub2 = getMetricForValidate(); + sub2.setStart("1356998300"); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test (expected = IllegalArgumentException.class) + public void testHashCodeandEqualsStartNull() { + TSQuery sub1 = getMetricForValidate(); + + final int hash_a = sub1.hashCode(); + sub1.setStart(null); + assertTrue(hash_a != sub1.hashCode()); + sub1.validateAndSetQuery(); + } + + @Test (expected = IllegalArgumentException.class) + public void testHashCodeandEqualsStartInvalid() { + TSQuery sub1 = getMetricForValidate(); + + final int hash_a = sub1.hashCode(); + sub1.setStart("1h-ago"); + assertTrue(hash_a != sub1.hashCode()); + sub1.validateAndSetQuery(); + } + + @Test + public void testHashCodeandEqualsEnd() { + TSQuery sub1 = getMetricForValidate(); + + final int hash_a = sub1.hashCode(); + sub1.setEnd("1356998490"); + final int hash_b = sub1.hashCode(); + assertTrue(hash_a != hash_b); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSQuery sub2 = getMetricForValidate(); + sub2.setEnd("1356998490"); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test + public void testHashCodeandEqualsEndNull() { + TSQuery sub1 = getMetricForValidate(); + + final int hash_a = sub1.hashCode(); + sub1.setEnd(null); + final int hash_b = sub1.hashCode(); + assertTrue(hash_a != hash_b); + // this is ok since we assume "now" if end is missing + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSQuery sub2 = getMetricForValidate(); + sub2.setEnd(null); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test (expected = IllegalArgumentException.class) + public void testHashCodeandEqualsEndInvalid() { + TSQuery sub1 = getMetricForValidate(); + + final int hash_a = sub1.hashCode(); + sub1.setEnd("1356998300"); + assertTrue(hash_a != sub1.hashCode()); + sub1.validateAndSetQuery(); + } + + @Test + public void testHashCodeandEqualsTimezone() { + TSQuery sub1 = getMetricForValidate(); + + final int hash_a = sub1.hashCode(); + sub1.setTimezone("America/New_York"); + final int hash_b = sub1.hashCode(); + assertTrue(hash_a != hash_b); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSQuery sub2 = getMetricForValidate(); + sub2.setTimezone("America/New_York"); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test (expected = IllegalArgumentException.class) + public void testHashCodeandEqualsTimezoneInvalid() throws Exception { + // silly test isn't calling into the real method, mockit! + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.parseDateTimeString(anyString(), anyString())) + .thenThrow(new IllegalArgumentException("Invalid timezone")); + + TSQuery sub1 = getMetricForValidate(); + + final int hash_a = sub1.hashCode(); + sub1.setTimezone("Not a timezone"); + final int hash_b = sub1.hashCode(); + assertTrue(hash_a != hash_b); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSQuery sub2 = getMetricForValidate(); + sub2.setTimezone("Not a timezone"); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test + public void testHashCodeandEqualsUseCalendar() { + TSQuery sub1 = getMetricForValidate(); + + final int hash_a = sub1.hashCode(); + sub1.setUseCalendar(true); + final int hash_b = sub1.hashCode(); + assertTrue(hash_a != hash_b); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSQuery sub2 = getMetricForValidate(); + sub2.setUseCalendar(true); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test + public void testHashCodeandEqualsOptions() { + TSQuery sub1 = getMetricForValidate(); + + final int hash_a = sub1.hashCode(); + HashMap<String, ArrayList<String>> options = + new HashMap<String, ArrayList<String>>(2); + ArrayList<String> params = new ArrayList<String>(1); + params.add("1419x576"); + options.put("wxh", params); + params = new ArrayList<String>(1); + params.add("latency"); + options.put("label", params); + sub1.setOptions(options); + + final int hash_b = sub1.hashCode(); + assertTrue(hash_a != hash_b); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSQuery sub2 = getMetricForValidate(); + + options = new HashMap<String, ArrayList<String>>(2); + params = new ArrayList<String>(1); + params.add("1419x576"); + options.put("wxh", params); + params = new ArrayList<String>(1); + params.add("latency"); + options.put("label", params); + sub2.setOptions(options); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test + public void testHashCodeandEqualsOptionsNewPut() { + TSQuery sub1 = getMetricForValidate(); + HashMap<String, ArrayList<String>> options = + new HashMap<String, ArrayList<String>>(3); + ArrayList<String> params = new ArrayList<String>(1); + params.add("1419x576"); + options.put("wxh", params); + params = new ArrayList<String>(1); + params.add("latency"); + options.put("label", params); + sub1.setOptions(options); + + final int hash_a = sub1.hashCode(); + + params = new ArrayList<String>(1); + params.add("top"); + options.put("key", params); + final int hash_b = sub1.hashCode(); + assertTrue(hash_a != hash_b); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSQuery sub2 = getMetricForValidate(); + + options = new HashMap<String, ArrayList<String>>(2); + params = new ArrayList<String>(1); + params.add("1419x576"); + options.put("wxh", params); + params = new ArrayList<String>(1); + params.add("latency"); + options.put("label", params); + params = new ArrayList<String>(1); + params.add("top"); + options.put("key", params); + sub2.setOptions(options); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test + public void testHashCodeandEqualsOptionsNewParam() { + TSQuery sub1 = getMetricForValidate(); + HashMap<String, ArrayList<String>> options = + new HashMap<String, ArrayList<String>>(3); + ArrayList<String> params = new ArrayList<String>(1); + params.add("1419x576"); + options.put("wxh", params); + params = new ArrayList<String>(1); + params.add("latency"); + options.put("label", params); + sub1.setOptions(options); + + final int hash_a = sub1.hashCode(); + + params = new ArrayList<String>(1); + params.add("cycles"); + options.put("label", params); + final int hash_b = sub1.hashCode(); + assertTrue(hash_a != hash_b); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSQuery sub2 = getMetricForValidate(); + + options = new HashMap<String, ArrayList<String>>(2); + params = new ArrayList<String>(1); + params.add("1419x576"); + options.put("wxh", params); + params = new ArrayList<String>(1); + params.add("cycles"); + options.put("label", params); + params = new ArrayList<String>(1);; + sub2.setOptions(options); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test + public void testHashCodeandEqualsOptionsExtraParam() { + TSQuery sub1 = getMetricForValidate(); + HashMap<String, ArrayList<String>> options = + new HashMap<String, ArrayList<String>>(3); + ArrayList<String> params = new ArrayList<String>(1); + params.add("1419x576"); + options.put("wxh", params); + params = new ArrayList<String>(1); + params.add("latency"); + options.put("label", params); + sub1.setOptions(options); + + final int hash_a = sub1.hashCode(); + + options.get("label").add("extra"); + final int hash_b = sub1.hashCode(); + assertTrue(hash_a != hash_b); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSQuery sub2 = getMetricForValidate(); + + options = new HashMap<String, ArrayList<String>>(2); + params = new ArrayList<String>(1); + params.add("1419x576"); + options.put("wxh", params); + params = new ArrayList<String>(1); + params.add("latency"); + params.add("extra"); + options.put("label", params); + params = new ArrayList<String>(1);; + sub2.setOptions(options); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test + public void testHashCodeandEqualsPadding() { + TSQuery sub1 = getMetricForValidate(); + + final int hash_a = sub1.hashCode(); + sub1.setPadding(true); + final int hash_b = sub1.hashCode(); + assertTrue(hash_a != hash_b); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSQuery sub2 = getMetricForValidate(); + sub2.setPadding(true); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test + public void testHashCodeandEqualsNoAnnotations() { + TSQuery sub1 = getMetricForValidate(); + + final int hash_a = sub1.hashCode(); + sub1.setNoAnnotations(true); + final int hash_b = sub1.hashCode(); + assertTrue(hash_a != hash_b); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSQuery sub2 = getMetricForValidate(); + sub2.setNoAnnotations(true); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test + public void testHashCodeandEqualsWithGlobalAnnotations() { + TSQuery sub1 = getMetricForValidate(); + + final int hash_a = sub1.hashCode(); + sub1.setGlobalAnnotations(true); + final int hash_b = sub1.hashCode(); + assertTrue(hash_a != hash_b); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSQuery sub2 = getMetricForValidate(); + sub2.setGlobalAnnotations(true); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test + public void testHashCodeandEqualsShowTSUIDs() { + TSQuery sub1 = getMetricForValidate(); + + final int hash_a = sub1.hashCode(); + sub1.setShowTSUIDs(true); + final int hash_b = sub1.hashCode(); + assertTrue(hash_a != hash_b); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSQuery sub2 = getMetricForValidate(); + sub2.setShowTSUIDs(true); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test + public void testHashCodeandEqualsMSResolution() { + TSQuery sub1 = getMetricForValidate(); + + final int hash_a = sub1.hashCode(); + sub1.setMsResolution(true); + final int hash_b = sub1.hashCode(); + assertTrue(hash_a != hash_b); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSQuery sub2 = getMetricForValidate(); + sub2.setMsResolution(true); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test + public void testHashCodeandEqualsNewSubQuery() { + TSQuery sub1 = getMetricForValidate(); + + final int hash_a = sub1.hashCode(); + sub1.getQueries().add(TestTSSubQuery.getBaseQuery()); + final int hash_b = sub1.hashCode(); + assertTrue(hash_a != hash_b); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSQuery sub2 = getMetricForValidate(); + sub2.getQueries().add(TestTSSubQuery.getBaseQuery()); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test + public void testHashCodeandEqualsChangeSubQuery() { + TSQuery sub1 = getMetricForValidate(); + + final int hash_a = sub1.hashCode(); + sub1.getQueries().get(0).setMetric("foo"); + final int hash_b = sub1.hashCode(); + assertTrue(hash_a != hash_b); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSQuery sub2 = getMetricForValidate(); + sub2.getQueries().get(0).setMetric("foo"); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test (expected = IllegalArgumentException.class) + public void testHashCodeandEqualsEmptySubQueries() { + TSQuery sub1 = getMetricForValidate(); + + final int hash_a = sub1.hashCode(); + sub1.setQueries(new ArrayList<TSSubQuery>(0)); + assertTrue(hash_a != sub1.hashCode()); + sub1.validateAndSetQuery(); + } + + @Test (expected = IllegalArgumentException.class) + public void testHashCodeandEqualsNullSubQueries() { + TSQuery sub1 = getMetricForValidate(); + + final int hash_a = sub1.hashCode(); + sub1.setQueries(null); + assertTrue(hash_a != sub1.hashCode()); + sub1.validateAndSetQuery(); + } + + @Test + public void testEqualsNull() { + TSQuery sub1 = getMetricForValidate(); + assertFalse(sub1.equals(null)); + } + + @Test + public void testEqualsWrongType() { + TSQuery sub1 = getMetricForValidate(); + assertFalse(sub1.equals(new String("Foobar"))); + } + + @Test + public void testEqualsSame() { + TSQuery sub1 = getMetricForValidate(); + assertTrue(sub1.equals(sub1)); + } + + @Test + public void testSplitsEligibleRollupQuery() throws Exception { + final TSQuery queryUnderTest = getMetricForValidate(); + + TSDB tsdb = PowerMockito.mock(TSDB.class); + TsdbQuery mockTsdbQuery = PowerMockito.mock(TsdbQuery.class); + when(mockTsdbQuery.configureFromQuery(eq(queryUnderTest), anyInt())).thenReturn(Deferred.fromResult(null)); + when(mockTsdbQuery.needsSplitting()).thenReturn(true); + when(tsdb.newQuery()).thenReturn(mockTsdbQuery); + + SplitRollupQuery mockSplitQuery = PowerMockito.mock(SplitRollupQuery.class); + when(mockSplitQuery.configureFromQuery(eq(queryUnderTest), anyInt())).thenReturn(Deferred.fromResult(null)); + + PowerMockito.whenNew(SplitRollupQuery.class).withAnyArguments().thenReturn(mockSplitQuery); + + queryUnderTest.buildQueriesAsync(tsdb); + + verify(mockSplitQuery).configureFromQuery(queryUnderTest, 0); + } + + @Test + public void testDoesNotSplitIneligibleRollupQuery() { + final TSQuery queryUnderTest = getMetricForValidate(); + + TSDB tsdb = PowerMockito.mock(TSDB.class); + TsdbQuery mockTsdbQuery = PowerMockito.mock(TsdbQuery.class); + when(mockTsdbQuery.configureFromQuery(eq(queryUnderTest), anyInt())).thenReturn(Deferred.fromResult(null)); + when(mockTsdbQuery.needsSplitting()).thenReturn(false); + when(tsdb.newQuery()).thenReturn(mockTsdbQuery); + + SplitRollupQuery mockedSplitQuery = PowerMockito.mock(SplitRollupQuery.class); + + queryUnderTest.buildQueriesAsync(tsdb); + + verifyZeroInteractions(mockedSplitQuery); + } + + /** + * Sets up an object with good, common values for testing the validation + * function with an query string query. Each test can "set" the + * method it wants to fool with and call .validateAndSetQuery() + * <b>Warning:</b> This method calls into {@link TestTSQuery} + * @return A query object + */ private TSQuery getMetricForValidate() { final TSQuery query = new TSQuery(); query.setStart("1356998400"); diff --git a/test/core/TestTSSubQuery.java b/test/core/TestTSSubQuery.java index eac7bcf291..a86ed8bb8f 100644 --- a/test/core/TestTSSubQuery.java +++ b/test/core/TestTSSubQuery.java @@ -13,11 +13,20 @@ package net.opentsdb.core; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import net.opentsdb.query.filter.TagVFilter; +import net.opentsdb.query.filter.TagVLiteralOrFilter; +import net.opentsdb.query.filter.TagVWildcardFilter; import org.junit.Test; @@ -33,8 +42,8 @@ public void validate() { TSSubQuery sub = getMetricForValidate(); sub.validateAndSetQuery(); assertEquals("sys.cpu.0", sub.getMetric()); - assertEquals("*", sub.getTags().get("host")); - assertEquals("lga", sub.getTags().get("dc")); + assertEquals("wildcard(*)", sub.getTags().get("host")); + assertEquals("literal_or(lga)", sub.getTags().get("dc")); assertEquals(Aggregators.SUM, sub.aggregator()); assertEquals(Aggregators.AVG, sub.downsampler()); assertEquals(300000, sub.downsampleInterval()); @@ -49,8 +58,8 @@ public void validateTS() { sub.setTsuids(tsuids); sub.validateAndSetQuery(); assertNotNull(sub.getTsuids()); - assertEquals("*", sub.getTags().get("host")); - assertEquals("lga", sub.getTags().get("dc")); + assertEquals("wildcard(*)", sub.getTags().get("host")); + assertEquals("literal_or(lga)", sub.getTags().get("dc")); assertEquals(Aggregators.SUM, sub.aggregator()); assertEquals(Aggregators.AVG, sub.downsampler()); assertEquals(300000, sub.downsampleInterval()); @@ -62,8 +71,8 @@ public void validateNoDS() { sub.setDownsample(null); sub.validateAndSetQuery(); assertEquals("sys.cpu.0", sub.getMetric()); - assertEquals("*", sub.getTags().get("host")); - assertEquals("lga", sub.getTags().get("dc")); + assertEquals("wildcard(*)", sub.getTags().get("host")); + assertEquals("literal_or(lga)", sub.getTags().get("dc")); assertEquals(Aggregators.SUM, sub.aggregator()); assertNull(sub.downsampler()); assertEquals(0, sub.downsampleInterval()); @@ -113,6 +122,518 @@ public void validateBadDS() { sub.validateAndSetQuery(); } + @Test + public void validateWithFilter() { + TSSubQuery sub = getMetricForValidate(); + sub.setFilters(Arrays.asList(TagVFilter.Builder() + .setFilter("*nari").setType("wildcard").setTagk("host").build())); + sub.validateAndSetQuery(); + assertEquals("sys.cpu.0", sub.getMetric()); + assertEquals(1, sub.getFilters().size()); + assertTrue(sub.getFilters().get(0) instanceof TagVWildcardFilter); + assertEquals(0, sub.getTags().size()); + assertEquals(Aggregators.SUM, sub.aggregator()); + assertEquals(Aggregators.AVG, sub.downsampler()); + assertEquals(300000, sub.downsampleInterval()); + } + + @Test + public void validateWithFilterViaTags() { + TSSubQuery sub = getMetricForValidate(); + + final Map<String, String> tags = new HashMap<String, String>(); + tags.put("host", TagVWildcardFilter.FILTER_NAME + "(*nari)"); + sub.setTags(tags); + sub.validateAndSetQuery(); + assertEquals("sys.cpu.0", sub.getMetric()); + assertEquals(1, sub.getFilters().size()); + assertTrue(sub.getFilters().get(0) instanceof TagVWildcardFilter); + assertEquals("wildcard(*nari)", sub.getTags().get("host")); + assertEquals(Aggregators.SUM, sub.aggregator()); + assertEquals(Aggregators.AVG, sub.downsampler()); + assertEquals(300000, sub.downsampleInterval()); + } + + @Test + public void validateGroupByFilterMissingParensViaTags() { + TSSubQuery sub = getMetricForValidate(); + final Map<String, String> tags = new HashMap<String, String>(); + tags.put("host", TagVWildcardFilter.FILTER_NAME); + sub.setTags(tags); + sub.validateAndSetQuery(); + assertEquals("sys.cpu.0", sub.getMetric()); + assertEquals(1, sub.getFilters().size()); + assertTrue(sub.getFilters().get(0) instanceof TagVLiteralOrFilter); + assertEquals(Aggregators.SUM, sub.aggregator()); + assertEquals(Aggregators.AVG, sub.downsampler()); + assertEquals(300000, sub.downsampleInterval()); + } + + @Test + public void validateWithGroupByFilter() { + TSSubQuery sub = getMetricForValidate(); + sub.setFilters(Arrays.asList(TagVFilter.Builder() + .setFilter("*nari").setType("wildcard").setTagk("host") + .setGroupBy(true).build())); + sub.validateAndSetQuery(); + assertEquals("sys.cpu.0", sub.getMetric()); + assertEquals("wildcard(*nari)", sub.getTags().get("host")); + assertEquals(1, sub.getFilters().size()); + assertTrue(sub.getFilters().get(0) instanceof TagVWildcardFilter); + assertEquals(Aggregators.SUM, sub.aggregator()); + assertEquals(Aggregators.AVG, sub.downsampler()); + assertEquals(300000, sub.downsampleInterval()); + } + + @Test + public void validateWithFilterAndGroupByFilter() { + TSSubQuery sub = getMetricForValidate(); + final List<TagVFilter> filters = new ArrayList<TagVFilter>(1); + filters.add(new TagVWildcardFilter("colo", "lga*")); + sub.setFilters(filters); + Map<String, String> tags = new HashMap<String, String>(); + tags.put("host", TagVWildcardFilter.FILTER_NAME + "(*nari)"); + sub.setTags(tags); + + sub.validateAndSetQuery(); + assertEquals("sys.cpu.0", sub.getMetric()); + assertEquals(TagVWildcardFilter.FILTER_NAME + "(*nari)", + sub.getTags().get("host")); + assertEquals(1, sub.getFilters().size()); + assertEquals(Aggregators.SUM, sub.aggregator()); + assertEquals(Aggregators.AVG, sub.downsampler()); + assertEquals(300000, sub.downsampleInterval()); + } + + @Test + public void validateWithFilterAndGroupByFilterSameTag() { + TSSubQuery sub = getMetricForValidate(); + final List<TagVFilter> filters = new ArrayList<TagVFilter>(1); + filters.add(new TagVWildcardFilter("host", "veti*")); + sub.setFilters(filters); + Map<String, String> tags = new HashMap<String, String>(); + tags.put("host", TagVWildcardFilter.FILTER_NAME + "(*nari)"); + sub.setTags(tags); + + sub.validateAndSetQuery(); + assertEquals("sys.cpu.0", sub.getMetric()); + assertEquals(TagVWildcardFilter.FILTER_NAME + "(*nari)", + sub.getTags().get("host")); + assertEquals(1, sub.getFilters().size()); + assertEquals(Aggregators.SUM, sub.aggregator()); + assertEquals(Aggregators.AVG, sub.downsampler()); + assertEquals(300000, sub.downsampleInterval()); + } + + @Test (expected = IllegalArgumentException.class) + public void validateWithDownsampleNone() { + TSSubQuery sub = getMetricForValidate(); + sub.setDownsample("1m-none"); + sub.validateAndSetQuery(); + } + + // NOTE: Each of the hash and equals tests should make sure that we the code + // doesn't change after validation. + + @Test + public void testHashCodeandEqualsAggregator() { + final TSSubQuery sub1 = getBaseQuery(); + final int hash_a = sub1.hashCode(); + + sub1.setAggregator("max"); + final int has_b = sub1.hashCode(); + assertFalse(hash_a == sub1.hashCode()); + sub1.validateAndSetQuery(); + assertEquals(has_b, sub1.hashCode()); + + final TSSubQuery sub2 = getBaseQuery(); + sub2.setAggregator("max"); + + assertEquals(has_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test (expected = IllegalArgumentException.class) + public void testHashCodeandEqualsAggregatorNull() { + final TSSubQuery sub1 = getBaseQuery(); + final int hash_a = sub1.hashCode(); + + sub1.setAggregator(null); + assertFalse(hash_a == sub1.hashCode()); + sub1.validateAndSetQuery(); + } + + @Test (expected = IllegalArgumentException.class) + public void testHashCodeandEqualsAggregatorNonExistant() { + final TSSubQuery sub1 = getBaseQuery(); + final int hash_a = sub1.hashCode(); + + sub1.setAggregator("nosuchagg"); + assertFalse(hash_a == sub1.hashCode()); + sub1.validateAndSetQuery(); + } + + @Test + public void testHashCodeandEqualsMetric() { + final TSSubQuery sub1 = getBaseQuery(); + final int hash_a = sub1.hashCode(); + + sub1.setMetric("foo"); + assertEquals(hash_a, sub1.hashCode()); + sub1.validateAndSetQuery(); + assertEquals(hash_a, sub1.hashCode()); + + TSSubQuery sub2 = getBaseQuery(); + sub2.setMetric("foo"); + + assertEquals(hash_a, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test (expected = IllegalArgumentException.class) + public void testHashCodeandEqualsMetricNull() { + final TSSubQuery sub1 = getBaseQuery(); + final int hash_a = sub1.hashCode(); + + sub1.setMetric(null); + assertFalse(hash_a == sub1.hashCode()); + sub1.validateAndSetQuery(); + } + + @Test + public void testHashCodeandEqualsTSUIDs() { + final TSSubQuery sub1 = getBaseQuery(); + final int hash_a = sub1.hashCode(); + + List<String> tsuids = new ArrayList<String>(2); + tsuids.add("01010101"); + tsuids.add("01010102"); + sub1.setTsuids(tsuids); + final int hash_b = sub1.hashCode(); + assertFalse(hash_a == sub1.hashCode()); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSSubQuery sub2 = getBaseQuery(); + List<String> tsuids2 = new ArrayList<String>(2); + tsuids2.add("01010101"); + tsuids2.add("01010102"); + sub2.setTsuids(tsuids2); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test + public void testHashCodeandEqualsTSUIDsChange() { + final TSSubQuery sub1 = getBaseQuery(); + final int hash_a = sub1.hashCode(); + List<String> tsuids = new ArrayList<String>(2); + tsuids.add("01010101"); + tsuids.add("01010102"); + sub1.setTsuids(tsuids); + + tsuids.set(1, "01010103"); + final int hash_b = sub1.hashCode(); + assertFalse(hash_a == sub1.hashCode()); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSSubQuery sub2 = getBaseQuery(); + List<String> tsuids2 = new ArrayList<String>(2); + tsuids2.add("01010101"); + tsuids2.add("01010103"); + sub2.setTsuids(tsuids2); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test + public void testHashCodeandEqualsTag() { + final TSSubQuery sub1 = getBaseQuery(); + final int hash_a = sub1.hashCode(); + Map<String, String> tags = new HashMap<String, String>(); + tags.put("host", "web02"); + sub1.setTags(tags); + final int hash_b = sub1.hashCode(); + assertFalse(hash_a == sub1.hashCode()); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSSubQuery sub2 = getBaseQuery(); + tags = new HashMap<String, String>(); + tags.put("host", "web02"); + sub2.setTags(tags); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test + public void testHashCodeandEqualsTags() { + final TSSubQuery sub1 = getBaseQuery(); + final int hash_a = sub1.hashCode(); + Map<String, String> tags = new HashMap<String, String>(); + tags.put("host", "web02"); + tags.put("foo", "bar"); + sub1.setTags(tags); + final int hash_b = sub1.hashCode(); + assertFalse(hash_a == sub1.hashCode()); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSSubQuery sub2 = getBaseQuery(); + tags = new HashMap<String, String>(); + tags.put("host", "web02"); + tags.put("foo", "bar"); + sub2.setTags(tags); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test + public void testHashCodeandEqualsTagsNull() { + final TSSubQuery sub1 = getBaseQuery(); + final int hash_a = sub1.hashCode(); + + sub1.setTags(null); + final int hash_b = sub1.hashCode(); + assertFalse(hash_a == sub1.hashCode()); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSSubQuery sub2 = getBaseQuery(); + sub2.setTags(null); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test + public void testHashCodeandEqualsDownsampler() { + final TSSubQuery sub1 = getBaseQuery(); + final int hash_a = sub1.hashCode(); + + sub1.setDownsample("1h-avg"); + final int hash_b = sub1.hashCode(); + assertFalse(hash_a == sub1.hashCode()); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSSubQuery sub2 = getBaseQuery(); + sub2.setDownsample("1h-avg"); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test (expected = IllegalArgumentException.class) + public void testHashCodeandEqualsDownsamplerInvalid() { + final TSSubQuery sub1 = getBaseQuery(); + final int hash_a = sub1.hashCode(); + + sub1.setDownsample("bad ds"); + assertFalse(hash_a == sub1.hashCode()); + sub1.validateAndSetQuery(); + } + + @Test + public void testHashCodeandEqualsRate() { + final TSSubQuery sub1 = getBaseQuery(); + final int hash_a = sub1.hashCode(); + + sub1.setRate(false); + final int hash_b = sub1.hashCode(); + assertFalse(hash_a == sub1.hashCode()); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSSubQuery sub2 = getBaseQuery(); + sub2.setRate(false); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test + public void testHashCodeandEqualsRateOptionsSameNew() { + final TSSubQuery sub1 = getBaseQuery(); + final int hash_a = sub1.hashCode(); + + sub1.setRateOptions(new RateOptions(true, 1024, 16)); + final int hash_b = sub1.hashCode(); + assertTrue(hash_a == sub1.hashCode()); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSSubQuery sub2 = getBaseQuery(); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test + public void testHashCodeandEqualsRateOptionsNotCounter() { + final TSSubQuery sub1 = getBaseQuery(); + final int hash_a = sub1.hashCode(); + + sub1.setRateOptions(new RateOptions(false, 1024, 16)); + final int hash_b = sub1.hashCode(); + assertFalse(hash_a == sub1.hashCode()); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSSubQuery sub2 = getBaseQuery(); + sub2.setRateOptions(new RateOptions(false, 1024, 16)); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test + public void testHashCodeandEqualsRateOptionsNewMax() { + final TSSubQuery sub1 = getBaseQuery(); + final int hash_a = sub1.hashCode(); + + sub1.setRateOptions(new RateOptions(true, 768, 16)); + final int hash_b = sub1.hashCode(); + assertFalse(hash_a == sub1.hashCode()); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSSubQuery sub2 = getBaseQuery(); + sub2.setRateOptions(new RateOptions(true, 768, 16)); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test + public void testHashCodeandEqualsRateOptionsNewReset() { + final TSSubQuery sub1 = getBaseQuery(); + final int hash_a = sub1.hashCode(); + + sub1.setRateOptions(new RateOptions(true, 1024, 32)); + final int hash_b = sub1.hashCode(); + assertFalse(hash_a == sub1.hashCode()); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSSubQuery sub2 = getBaseQuery(); + sub2.setRateOptions(new RateOptions(true, 1024, 32)); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test + public void testHashCodeandEqualsRateOptionsNull() { + final TSSubQuery sub1 = getBaseQuery(); + final int hash_a = sub1.hashCode(); + + sub1.setRateOptions(null); + final int hash_b = sub1.hashCode(); + assertFalse(hash_a == sub1.hashCode()); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSSubQuery sub2 = getBaseQuery(); + sub2.setRateOptions(null); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test + public void testHashCodeandEqualsExplicitTags() { + final TSSubQuery sub1 = getBaseQuery(); + final int hash_a = sub1.hashCode(); + + sub1.setExplicitTags(true); + final int hash_b = sub1.hashCode(); + assertFalse(hash_a == sub1.hashCode()); + sub1.validateAndSetQuery(); + assertEquals(hash_b, sub1.hashCode()); + + TSSubQuery sub2 = getBaseQuery(); + sub2.setExplicitTags(true); + + assertEquals(hash_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + @Test + public void testEqualsNull() { + final TSSubQuery sub1 = getBaseQuery(); + assertFalse(sub1.equals(null)); + } + + @Test + public void testEqualsWrongType() { + final TSSubQuery sub1 = getBaseQuery(); + assertFalse(sub1.equals(new String("Foobar"))); + } + + @Test + public void testEqualsSame() { + final TSSubQuery sub1 = getBaseQuery(); + assertTrue(sub1.equals(sub1)); + } + + @Test + public void testHashCodeandEqualsFilter() { + TSSubQuery sub1 = getBaseQuery(); + final int hash_a = sub1.hashCode(); + sub1.setFilters(Arrays.asList(TagVFilter.Builder() + .setFilter("*nari").setType("wildcard").setTagk("host") + .setGroupBy(true).build())); + assertFalse(hash_a == sub1.hashCode()); + sub1.validateAndSetQuery(); + final int has_b = sub1.hashCode(); + assertEquals(has_b, sub1.hashCode()); + + TSSubQuery sub2 = getBaseQuery(); + sub2.setFilters(Arrays.asList(TagVFilter.Builder() + .setFilter("*nari").setType("wildcard").setTagk("host") + .setGroupBy(true).build())); + sub2.validateAndSetQuery(); + + assertEquals(has_b, sub2.hashCode()); + assertEquals(sub1, sub2); + assertFalse(sub1 == sub2); + } + + /** @return a sub query object with some defaults set for testing */ + public static TSSubQuery getBaseQuery() { + TSSubQuery query = new TSSubQuery(); + query.setAggregator("sum"); + query.setMetric("foo"); + HashMap<String, String> tags = new HashMap<String, String>(2); + tags.put("host", "web01"); + tags.put("dc", "lax"); + query.setTags(tags); + query.setRate(true); + query.setRateOptions(new RateOptions(true, 1024, 16)); + return query; + } + /** * Sets up an object with good, common values for testing the validation * function with an "m" type query (no tsuids). Each test can "set" the diff --git a/test/core/TestTags.java b/test/core/TestTags.java index f1ae95b992..1583336b43 100644 --- a/test/core/TestTags.java +++ b/test/core/TestTags.java @@ -18,31 +18,45 @@ import java.util.List; import java.util.Map; +import net.opentsdb.query.filter.TagVFilter; +import net.opentsdb.query.filter.TagVLiteralOrFilter; +import net.opentsdb.query.filter.TagVLiteralOrFilter.TagVILiteralOrFilter; +import net.opentsdb.query.filter.TagVRegexFilter; +import net.opentsdb.query.filter.TagVWildcardFilter; import net.opentsdb.storage.MockBase; +import net.opentsdb.uid.FailedToAssignUniqueIdException; import net.opentsdb.uid.NoSuchUniqueId; import net.opentsdb.uid.NoSuchUniqueName; import net.opentsdb.uid.UniqueId; import net.opentsdb.utils.Config; import net.opentsdb.utils.Pair; +import org.hbase.async.Bytes; import org.hbase.async.DeleteRequest; import org.hbase.async.GetRequest; import org.hbase.async.HBaseClient; import org.hbase.async.KeyValue; import org.hbase.async.PutRequest; +import org.hbase.async.Bytes.ByteMap; import org.junit.Test; import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; import com.stumbleupon.async.Deferred; +import com.stumbleupon.async.DeferredGroupException; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.anyMapOf; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.when; import static org.powermock.api.mockito.PowerMockito.mock; @@ -51,7 +65,8 @@ "com.sum.*", "org.xml.*"}) @RunWith(PowerMockRunner.class) @PrepareForTest({TSDB.class, Config.class, UniqueId.class, HBaseClient.class, - GetRequest.class, PutRequest.class, DeleteRequest.class, KeyValue.class}) + GetRequest.class, PutRequest.class, DeleteRequest.class, KeyValue.class, + Const.class}) public final class TestTags { private TSDB tsdb; private Config config; @@ -376,6 +391,121 @@ public void parseWithMetricOnlyEquals() { Tags.parseWithMetric("{=}", tags); } + @Test + public void parseWithMetricAndFilters() { + final List<TagVFilter> filters = new ArrayList<TagVFilter>(); + String metric = Tags.parseWithMetricAndFilters("sys.cpu.user", filters); + assertEquals("sys.cpu.user", metric); + assertEquals(0, filters.size()); + + filters.clear(); + metric = Tags.parseWithMetricAndFilters("sys.cpu.user{host=web01}", filters); + assertEquals("sys.cpu.user", metric); + assertEquals(1, filters.size()); + assertEquals("host", filters.get(0).getTagk()); + assertTrue(filters.get(0).isGroupBy()); + assertTrue(filters.get(0) instanceof TagVLiteralOrFilter); + + filters.clear(); + metric = Tags.parseWithMetricAndFilters("sys.cpu.user{host=*}", filters); + assertEquals("sys.cpu.user", metric); + assertEquals(1, filters.size()); + assertEquals("host", filters.get(0).getTagk()); + assertTrue(filters.get(0).isGroupBy()); + assertTrue(filters.get(0) instanceof TagVWildcardFilter); + + filters.clear(); + metric = Tags.parseWithMetricAndFilters("sys.cpu.user{host=web01}{}", filters); + assertEquals("sys.cpu.user", metric); + assertEquals(1, filters.size()); + assertEquals("host", filters.get(0).getTagk()); + assertTrue(filters.get(0).isGroupBy()); + assertTrue(filters.get(0) instanceof TagVLiteralOrFilter); + + filters.clear(); + metric = Tags.parseWithMetricAndFilters( + "sys.cpu.user{host=*,owner=regexp(.*ob)}", filters); + assertEquals("sys.cpu.user", metric); + assertEquals(2, filters.size()); + for (final TagVFilter filter : filters) { + if (filter instanceof TagVWildcardFilter) { + assertEquals("host", filter.getTagk()); + } else if (filter instanceof TagVRegexFilter) { + assertEquals("owner", filter.getTagk()); + } + assertTrue(filter.isGroupBy()); + } + + filters.clear(); + metric = Tags.parseWithMetricAndFilters("sys.cpu.user{}{host=web01}", filters); + assertEquals("sys.cpu.user", metric); + assertEquals(1, filters.size()); + assertEquals("host", filters.get(0).getTagk()); + assertFalse(filters.get(0).isGroupBy()); + assertTrue(filters.get(0) instanceof TagVLiteralOrFilter); + + filters.clear(); + metric = Tags.parseWithMetricAndFilters( + "sys.cpu.user{}{host=iliteral_or(web01|Web02)}", filters); + assertEquals("sys.cpu.user", metric); + assertEquals(1, filters.size()); + assertEquals("host", filters.get(0).getTagk()); + assertFalse(filters.get(0).isGroupBy()); + assertTrue(filters.get(0) instanceof TagVILiteralOrFilter); + + filters.clear(); + metric = Tags.parseWithMetricAndFilters( + "sys.cpu.user{}{host=iliteral_or(web01|Web02),owner=*}", filters); + assertEquals("sys.cpu.user", metric); + assertEquals(2, filters.size()); + for (final TagVFilter filter : filters) { + if (filter instanceof TagVWildcardFilter) { + assertEquals("owner", filter.getTagk()); + } else if (filter instanceof TagVILiteralOrFilter) { + assertEquals("host", filter.getTagk()); + } + assertFalse(filter.isGroupBy()); + } + + filters.clear(); + metric = Tags.parseWithMetricAndFilters( + "sys.cpu.user{host=iliteral_or(web01|Web02)}{owner=*}", filters); + assertEquals("sys.cpu.user", metric); + System.out.println(filters); + assertEquals(2, filters.size()); + for (final TagVFilter filter : filters) { + if (filter instanceof TagVWildcardFilter) { + assertEquals("owner", filter.getTagk()); + assertFalse(filter.isGroupBy()); + } else if (filter instanceof TagVILiteralOrFilter) { + assertEquals("host", filter.getTagk()); + assertTrue(filter.isGroupBy()); + } + } + } + + @Test (expected = IllegalArgumentException.class) + public void parseWithMetricAndFiltersMissingTrailingCurly() { + final List<TagVFilter> filters = new ArrayList<TagVFilter>(); + Tags.parseWithMetricAndFilters("sys.cpu.user{}{host=web01", filters); + } + + @Test (expected = IllegalArgumentException.class) + public void parseWithMetricAndFiltersNullString() { + final List<TagVFilter> filters = new ArrayList<TagVFilter>(); + Tags.parseWithMetricAndFilters(null, filters); + } + + @Test (expected = IllegalArgumentException.class) + public void parseWithMetricAndFiltersEmptyString() { + final List<TagVFilter> filters = new ArrayList<TagVFilter>(); + Tags.parseWithMetricAndFilters("", filters); + } + + @Test (expected = IllegalArgumentException.class) + public void parseWithMetricAndFiltersNullFilters() { + Tags.parseWithMetricAndFilters("sys.cpu.user{}{host=web01}", null); + } @Test public void parseSuccessful() { final HashMap<String, String> tags = new HashMap<String, String>(2); @@ -631,6 +761,71 @@ public void resolveOrCreateTagvNotAllowedBlocked() throws Exception { Tags.resolveOrCreateAll(tsdb, tags); } + @Test + public void resolveOrCreateAllAsync() throws Exception { + setupStorage(); + setupResolveAll(); + + final Map<String, String> tags = new HashMap<String, String>(1); + tags.put("host", "nohost"); + final List<byte[]> uids = Tags.resolveOrCreateAllAsync(tsdb, "metric", tags).join(); + assertEquals(1, uids.size()); + assertArrayEquals(new byte[] { 0, 0, 1, 0, 0, 3}, uids.get(0)); + } + + @Test (expected = DeferredGroupException.class) + public void resolveOrCreateAllAsyncFilterBlocked() throws Exception { + setupStorage(); + setupResolveAll(); + when(tag_names.getOrCreateIdAsync(eq("host"), anyString(), + anyMapOf(String.class, String.class))) + .thenReturn(Deferred.<byte[]>fromError(new FailedToAssignUniqueIdException( + "tagk", "host", 0, "Blocked by UID filter."))); + + final Map<String, String> tags = new HashMap<String, String>(1); + tags.put("host", "nohost"); + Tags.resolveOrCreateAllAsync(tsdb, "metric", tags).join(); + } + + @Test + public void looksLikeIntegerSimple() { + assertEquals(true, Tags.looksLikeInteger("123")); + } + + @Test + public void looksLikeIntegerFloat() { + assertEquals(false, Tags.looksLikeInteger("12.3")); + } + + @Test + public void looksLikeIntegerExponent() { + assertEquals(false, Tags.looksLikeInteger("1e10")); + } + + @Test + public void fitsInFloatSimple() { + // deceiving eh? + assertEquals(false, Tags.fitsInFloat("12.3")); + } + + @Test + public void fitsInFloatDoublePrecision() { + assertEquals(false, Tags.fitsInFloat("1.234556789123456")); + } + + @Test(expected=NumberFormatException.class) + public void fitsInFloatMalformed() { + assertEquals(false, Tags.fitsInFloat("1.2abc34")); + } + + @Test + public void fitsInFloat() { + assertEquals(true, Tags.fitsInFloat("0.6116398572921753")); + assertEquals(true, Tags.fitsInFloat("1.01417856E9")); + assertEquals(false, Tags.fitsInFloat("4.508277154265837E7")); + assertEquals(false, Tags.fitsInFloat("8.208611994536002E8")); + } + // PRIVATE helpers to setup unit tests private void setupStorage() throws Exception { @@ -674,18 +869,132 @@ private void setupResolveIds() throws Exception { } private void setupResolveAll() throws Exception { - when(tag_names.getOrCreateId("host")).thenReturn(new byte[] { 0, 0, 1 }); - when(tag_names.getOrCreateId("doesnotexist")) + when(tag_names.getOrCreateId(eq("host"))) + .thenReturn(new byte[] { 0, 0, 1 }); + when(tag_names.getOrCreateIdAsync(eq("host"), anyString(), + anyMapOf(String.class, String.class))) + .thenReturn(Deferred.fromResult(new byte[] { 0, 0, 1 })); + when(tag_names.getOrCreateId(eq("doesnotexist"))) .thenReturn(new byte[] { 0, 0, 3 }); + when(tag_names.getOrCreateIdAsync(eq("doesnotexist"), anyString(), + anyMapOf(String.class, String.class))) + .thenReturn(Deferred.fromResult(new byte[] { 0, 0, 3 })); when(tag_names.getId("pop")).thenReturn(new byte[] { 0, 0, 2 }); when(tag_names.getId("nonesuch")) .thenThrow(new NoSuchUniqueName("tagv", "nonesuch")); - when(tag_values.getOrCreateId("web01")).thenReturn(new byte[] { 0, 0, 1 }); - when(tag_values.getOrCreateId("nohost")) - .thenReturn(new byte[] { 0, 0, 3 }); + when(tag_values.getOrCreateId(eq("web01"))) + .thenReturn(new byte[] { 0, 0, 1 }); + when(tag_values.getOrCreateIdAsync(eq("web01"), anyString(), + anyMapOf(String.class, String.class))) + .thenReturn(Deferred.fromResult(new byte[] { 0, 0, 1 })); + when(tag_values.getOrCreateId(eq("nohost"))) + .thenReturn(new byte[] { 0, 0, 3 }); + when(tag_values.getOrCreateIdAsync(eq("nohost"), anyString(), + anyMapOf(String.class, String.class))) + .thenReturn(Deferred.fromResult(new byte[] { 0, 0, 3 })); when(tag_values.getId("web02")).thenReturn(new byte[] { 0, 0, 2 }); when(tag_values.getId("invalidhost")) .thenThrow(new NoSuchUniqueName("tagk", "invalidhost")); } + + @Test + public void getTagUids() throws Exception { + byte[] row = new byte[] { 0, 0, 1, 0x50, (byte)0xE2, 0x27, 0, + 0, 0, 1, 0, 0, 2}; + ByteMap<byte[]> uids = Tags.getTagUids(row); + assertEquals(1, uids.size()); + assertEquals(0, Bytes.memcmp(new byte[] { 0, 0, 1 }, uids.firstKey())); + assertEquals(0, Bytes.memcmp(new byte[] { 0, 0, 2 }, uids.firstEntry() + .getValue())); + + row = new byte[] { 0, 0, 1, 0x50, (byte)0xE2, 0x27, 0, + 0, 0, 1, 0, 0, 2, 0, 0, 3, 0, 0, 4}; + uids = Tags.getTagUids(row); + assertEquals(2, uids.size()); + assertEquals(0, Bytes.memcmp(new byte[] { 0, 0, 1 }, uids.firstKey())); + assertEquals(0, Bytes.memcmp(new byte[] { 0, 0, 2 }, uids.firstEntry() + .getValue())); + assertEquals(0, Bytes.memcmp(new byte[] { 0, 0, 3 }, uids.lastKey())); + assertEquals(0, Bytes.memcmp(new byte[] { 0, 0, 4 }, uids.lastEntry() + .getValue())); + + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + + row = new byte[] { 1, 0, 0, 1, 0x50, (byte)0xE2, 0x27, 0, + 0, 0, 1, 0, 0, 2}; + uids = Tags.getTagUids(row); + assertEquals(1, uids.size()); + assertEquals(0, Bytes.memcmp(new byte[] { 0, 0, 1 }, uids.firstKey())); + assertEquals(0, Bytes.memcmp(new byte[] { 0, 0, 2 }, uids.firstEntry() + .getValue())); + + row = new byte[] { 1, 0, 0, 1, 0x50, (byte)0xE2, 0x27, 0, + 0, 0, 1, 0, 0, 2, 0, 0, 3, 0, 0, 4}; + uids = Tags.getTagUids(row); + assertEquals(2, uids.size()); + assertEquals(0, Bytes.memcmp(new byte[] { 0, 0, 1 }, uids.firstKey())); + assertEquals(0, Bytes.memcmp(new byte[] { 0, 0, 2 }, uids.firstEntry() + .getValue())); + assertEquals(0, Bytes.memcmp(new byte[] { 0, 0, 3 }, uids.lastKey())); + assertEquals(0, Bytes.memcmp(new byte[] { 0, 0, 4 }, uids.lastEntry() + .getValue())); + } + + @Test (expected = ArrayIndexOutOfBoundsException.class) + public void getTagUidsMissingTagV() throws Exception { + byte[] row = new byte[] { 0, 0, 1, 0x50, (byte)0xE2, 0x27, 0, 0, 0, 1 }; + Tags.getTagUids(row); + } + + @Test (expected = ArrayIndexOutOfBoundsException.class) + public void getTagUidsMissingTagVSalted() throws Exception { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + + byte[] row = new byte[] { 1, 0, 0, 1, 0x50, (byte)0xE2, 0x27, 0, 0, 0, 1 }; + Tags.getTagUids(row); + } + + @Test + public void getTagUidsMissingTags() throws Exception { + byte[] row = new byte[] { 0, 0, 1, 0x50, (byte)0xE2, 0x27, 0 }; + ByteMap<byte[]> uids = Tags.getTagUids(row); + assertEquals(0, uids.size()); + + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + + row = new byte[] { 1, 0, 0, 1, 0x50, (byte)0xE2, 0x27, 0 }; + uids = Tags.getTagUids(row); + assertEquals(0, uids.size()); + } + + @Test (expected = NullPointerException.class) + public void getTagUidsNullRow() throws Exception { + Tags.getTagUids(null); + } + + @Test + public void getTagUidsEmptyRow() throws Exception { + final ByteMap<byte[]> uids = Tags.getTagUids(new byte[] {}); + assertEquals(0, uids.size()); + } + + @Test + public void setAllowSpecialChars() throws Exception { + assertFalse(Tags.isAllowSpecialChars('!')); + + Tags.setAllowSpecialChars(null); + assertFalse(Tags.isAllowSpecialChars('!')); + + Tags.setAllowSpecialChars(""); + assertFalse(Tags.isAllowSpecialChars('!')); + + Tags.setAllowSpecialChars("!)(%"); + assertTrue(Tags.isAllowSpecialChars('!')); + assertTrue(Tags.isAllowSpecialChars('(')); + assertTrue(Tags.isAllowSpecialChars('%')); + } } diff --git a/test/core/TestTsdbQuery.java b/test/core/TestTsdbQuery.java index 52e0d6a1a4..d22bdfed0a 100644 --- a/test/core/TestTsdbQuery.java +++ b/test/core/TestTsdbQuery.java @@ -12,250 +12,192 @@ // see <http://www.gnu.org/licenses/>. package net.opentsdb.core; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyString; -import static org.mockito.Mockito.when; -import static org.powermock.api.mockito.PowerMockito.mock; - -import java.lang.reflect.Field; import java.util.ArrayList; -import java.util.HashMap; +import java.util.Collections; import java.util.List; -import java.util.Map; -import net.opentsdb.meta.Annotation; +import com.stumbleupon.async.Deferred; +import net.opentsdb.core.TsdbQuery.ForTesting; +import net.opentsdb.query.QueryLimitOverride; +import net.opentsdb.query.filter.TagVFilter; +import net.opentsdb.query.filter.TagVWildcardFilter; +import net.opentsdb.rollup.RollupInterval; +import net.opentsdb.rollup.RollupQuery; import net.opentsdb.storage.MockBase; -import net.opentsdb.uid.NoSuchUniqueId; import net.opentsdb.uid.NoSuchUniqueName; -import net.opentsdb.uid.UniqueId; -import net.opentsdb.utils.Config; - -import org.apache.zookeeper.proto.DeleteRequest; -import org.hbase.async.Bytes; -import org.hbase.async.GetRequest; -import org.hbase.async.HBaseClient; -import org.hbase.async.KeyValue; -import org.hbase.async.PutRequest; -import org.hbase.async.Scanner; +import net.opentsdb.utils.DateTime; + +import org.jboss.netty.util.internal.ThreadLocalRandom; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; -import com.stumbleupon.async.Deferred; +import com.stumbleupon.async.DeferredGroupException; + +import static org.junit.Assert.*; +import static org.mockito.Matchers.*; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.powermock.api.mockito.PowerMockito.doReturn; +import static org.powermock.api.mockito.PowerMockito.spy; /** - * Massive test class that is used to test all facets of querying for data. - * Since data is fetched using the TsdbQuery class, it makes sense to put all - * of the unit tests here that deal with actual data. This includes: - * - queries - * - aggregations - * - rate conversion - * - downsampling - * - compactions (read and write) + * This class is for unit testing the TsdbQuery class. Pretty much making sure + * the various ctors and methods function as expected. For actually running the + * queries and validating the group by and aggregation logic, see + * {@link TestTsdbQueryQueries} */ @RunWith(PowerMockRunner.class) -@PowerMockIgnore({"javax.management.*", "javax.xml.*", - "ch.qos.*", "org.slf4j.*", - "com.sum.*", "org.xml.*"}) -@PrepareForTest({TSDB.class, Config.class, UniqueId.class, HBaseClient.class, - CompactionQueue.class, GetRequest.class, PutRequest.class, KeyValue.class, - Scanner.class, TsdbQuery.class, DeleteRequest.class, Annotation.class, - RowKey.class, Span.class, SpanGroup.class, IncomingDataPoints.class }) -public final class TestTsdbQuery { - private Config config; - private TSDB tsdb = null; - private HBaseClient client = mock(HBaseClient.class); - private UniqueId metrics = mock(UniqueId.class); - private UniqueId tag_names = mock(UniqueId.class); - private UniqueId tag_values = mock(UniqueId.class); +@PrepareForTest({ DateTime.class, TsdbQuery.class }) +public final class TestTsdbQuery extends BaseTsdbTest { + + private static final long ONE_DAY_MS = 24 * 60 * 60 * 1000; + private TsdbQuery query = null; - private MockBase storage = null; @Before - public void before() throws Exception { - PowerMockito.whenNew(HBaseClient.class) - .withArguments(anyString(), anyString()).thenReturn(client); - config = new Config(false); - config.setFixDuplicates(true); // TODO(jat): test both ways - tsdb = new TSDB(config); + public void beforeLocal() throws Exception { query = new TsdbQuery(tsdb); - - // replace the "real" field objects with mocks - Field met = tsdb.getClass().getDeclaredField("metrics"); - met.setAccessible(true); - met.set(tsdb, metrics); - - Field tagk = tsdb.getClass().getDeclaredField("tag_names"); - tagk.setAccessible(true); - tagk.set(tsdb, tag_names); - - Field tagv = tsdb.getClass().getDeclaredField("tag_values"); - tagv.setAccessible(true); - tagv.set(tsdb, tag_values); - - // mock UniqueId - when(metrics.getId("sys.cpu.user")).thenReturn(new byte[] { 0, 0, 1 }); - when(metrics.getNameAsync(new byte[] { 0, 0, 1 })) - .thenReturn(Deferred.fromResult("sys.cpu.user")); - when(metrics.getId("sys.cpu.system")) - .thenThrow(new NoSuchUniqueName("sys.cpu.system", "metric")); - when(metrics.getId("sys.cpu.nice")).thenReturn(new byte[] { 0, 0, 2 }); - when(metrics.getNameAsync(new byte[] { 0, 0, 2 })) - .thenReturn(Deferred.fromResult("sys.cpu.nice")); - when(tag_names.getId("host")).thenReturn(new byte[] { 0, 0, 1 }); - when(tag_names.getIdAsync("host")).thenReturn( - Deferred.fromResult(new byte[] { 0, 0, 1 })); - when(tag_names.getNameAsync(new byte[] { 0, 0, 1 })) - .thenReturn(Deferred.fromResult("host")); - when(tag_names.getOrCreateIdAsync("host")).thenReturn( - Deferred.fromResult(new byte[] { 0, 0, 1 })); - when(tag_names.getIdAsync("dc")) - .thenThrow(new NoSuchUniqueName("dc", "metric")); - when(tag_values.getId("web01")).thenReturn(new byte[] { 0, 0, 1 }); - when(tag_values.getIdAsync("web01")).thenReturn( - Deferred.fromResult(new byte[] { 0, 0, 1 })); - when(tag_values.getNameAsync(new byte[] { 0, 0, 1 })) - .thenReturn(Deferred.fromResult("web01")); - when(tag_values.getOrCreateIdAsync("web01")).thenReturn( - Deferred.fromResult(new byte[] { 0, 0, 1 })); - when(tag_values.getId("web02")).thenReturn(new byte[] { 0, 0, 2 }); - when(tag_values.getIdAsync("web02")).thenReturn( - Deferred.fromResult(new byte[] { 0, 0, 2 })); - when(tag_values.getNameAsync(new byte[] { 0, 0, 2 })) - .thenReturn(Deferred.fromResult("web02")); - when(tag_values.getOrCreateIdAsync("web02")).thenReturn( - Deferred.fromResult(new byte[] { 0, 0, 2 })); - when(tag_values.getId("web03")) - .thenThrow(new NoSuchUniqueName("web03", "metric")); - - when(metrics.width()).thenReturn((short)3); - when(tag_names.width()).thenReturn((short)3); - when(tag_values.width()).thenReturn((short)3); } - + @Test public void setStartTime() throws Exception { query.setStartTime(1356998400L); assertEquals(1356998400L, query.getStartTime()); } - + @Test public void setStartTimeZero() throws Exception { query.setStartTime(0L); } - + @Test (expected = IllegalArgumentException.class) public void setStartTimeInvalidNegative() throws Exception { query.setStartTime(-1L); } - + @Test (expected = IllegalArgumentException.class) public void setStartTimeInvalidTooBig() throws Exception { query.setStartTime(17592186044416L); } - - @Test (expected = IllegalArgumentException.class) + + @Test public void setStartTimeEqualtoEndTime() throws Exception { query.setEndTime(1356998400L); query.setStartTime(1356998400L); } - + @Test (expected = IllegalArgumentException.class) public void setStartTimeGreaterThanEndTime() throws Exception { query.setEndTime(1356998400L); query.setStartTime(1356998460L); } - + @Test public void setEndTime() throws Exception { query.setEndTime(1356998400L); assertEquals(1356998400L, query.getEndTime()); } - + + @Test + public void getScanEndTimeSeconds() { + long now = System.currentTimeMillis() / 1000; + long baseTime = now - (now % Const.MAX_TIMESPAN); + long expectedEndScanTime = baseTime + Const.MAX_TIMESPAN; + + for (int i = 0; i < 3600; i++) { + long sec = baseTime + i; + long ms = sec * 1000 + ThreadLocalRandom.current().nextInt(1000); + query.setEndTime(sec); + Assert.assertEquals("EndTime=" + sec, expectedEndScanTime, + query.getScanEndTimeSeconds()); + query.setEndTime(ms); + Assert.assertEquals("EndTime=" + ms, expectedEndScanTime, + query.getScanEndTimeSeconds()); + } + } + @Test (expected = IllegalStateException.class) public void getStartTimeNotSet() throws Exception { query.getStartTime(); } - + @Test (expected = IllegalArgumentException.class) public void setEndTimeInvalidNegative() throws Exception { query.setEndTime(-1L); } - + @Test (expected = IllegalArgumentException.class) public void setEndTimeInvalidTooBig() throws Exception { query.setEndTime(17592186044416L); } - + @Test (expected = IllegalArgumentException.class) public void setEndTimeEqualtoEndTime() throws Exception { query.setStartTime(1356998400L); query.setEndTime(1356998400L); } - + @Test (expected = IllegalArgumentException.class) public void setEndTimeGreaterThanEndTime() throws Exception { query.setStartTime(1356998460L); query.setEndTime(1356998400L); } - + @Test public void getEndTimeNotSet() throws Exception { - PowerMockito.mockStatic(System.class); - when(System.currentTimeMillis()).thenReturn(1357300800000L); + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1357300800000L); assertEquals(1357300800000L, query.getEndTime()); } - + @Test public void setTimeSeries() throws Exception { - setQueryStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); assertNotNull(query); + assertArrayEquals(METRIC_BYTES, ForTesting.getMetric(query)); + assertEquals(1, ForTesting.getFilters(query).size()); + assertEquals(1, ForTesting.getGroupBys(query).size()); + assertArrayEquals(TAGK_BYTES, ForTesting.getGroupBys(query).get(0)); + assertEquals(1, ForTesting.getRowKeyLiterals(query).size()); + assertEquals(1, ForTesting.getRowKeyLiterals(query).get(TAGV_BYTES).length); + assertArrayEquals(TAGV_BYTES, + ForTesting.getRowKeyLiterals(query).get(TAGV_BYTES)[0]); } - + @Test (expected = NullPointerException.class) public void setTimeSeriesNullTags() throws Exception { - query.setTimeSeries("sys.cpu.user", null, Aggregators.SUM, false); + query.setTimeSeries(METRIC_STRING, null, Aggregators.SUM, false); } - + @Test public void setTimeSeriesEmptyTags() throws Exception { - HashMap<String, String> tags = new HashMap<String, String>(1); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); + tags.clear(); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); assertNotNull(query); } - + @Test (expected = NoSuchUniqueName.class) public void setTimeSeriesNosuchMetric() throws Exception { - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - query.setTimeSeries("sys.cpu.system", tags, Aggregators.SUM, false); + query.setTimeSeries(NSUN_METRIC, tags, Aggregators.SUM, false); } - + @Test (expected = NoSuchUniqueName.class) public void setTimeSeriesNosuchTagk() throws Exception { - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("dc", "web01"); - query.setTimeSeries("sys.cpu.system", tags, Aggregators.SUM, false); + tags.clear(); + tags.put(NSUN_TAGK, TAGV_STRING); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); } - + @Test (expected = NoSuchUniqueName.class) public void setTimeSeriesNosuchTagv() throws Exception { - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web03"); - query.setTimeSeries("sys.cpu.system", tags, Aggregators.SUM, false); + tags.put(TAGK_STRING, NSUN_TAGV); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); } @Test @@ -266,18 +208,18 @@ public void setTimeSeriesTS() throws Exception { query.setTimeSeries(tsuids, Aggregators.SUM, false); assertNotNull(query); } - + @Test (expected = IllegalArgumentException.class) public void setTimeSeriesTSNullList() throws Exception { query.setTimeSeries(null, Aggregators.SUM, false); } - + @Test (expected = IllegalArgumentException.class) public void setTimeSeriesTSEmptyList() throws Exception { final List<String> tsuids = new ArrayList<String>(); query.setTimeSeries(tsuids, Aggregators.SUM, false); } - + @Test (expected = IllegalArgumentException.class) public void setTimeSeriesTSDifferentMetrics() throws Exception { final List<String> tsuids = new ArrayList<String>(2); @@ -285,2709 +227,564 @@ public void setTimeSeriesTSDifferentMetrics() throws Exception { tsuids.add("000002000001000002"); query.setTimeSeries(tsuids, Aggregators.SUM, false); } - + @Test - public void runLongSingleTS() throws Exception { - storeLongTimeSeriesSeconds(true, false);; - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); + public void configureFromQuery() throws Exception { + setDataPointStorage(); + final TSQuery ts_query = getTSQuery(); + ts_query.validateAndSetQuery(); + query = new TsdbQuery(tsdb); + query.configureFromQuery(ts_query, 0).joinUninterruptibly(); - final DataPoints[] dps = query.run(); - - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); - - int value = 1; - for (DataPoint dp : dps[0]) { - assertEquals(value, dp.longValue()); - value++; - } - assertEquals(300, dps[0].aggregatedSize()); + assertArrayEquals(METRIC_BYTES, ForTesting.getMetric(query)); + assertEquals(1, ForTesting.getFilters(query).size()); + assertArrayEquals(TAGK_BYTES, ForTesting.getGroupBys(query).get(0)); + assertEquals(1, ForTesting.getGroupBys(query).size()); + assertNotNull(ForTesting.getRateOptions(query)); } - + @Test - public void runLongSingleTSMs() throws Exception { - storeLongTimeSeriesMs(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); - - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); - - int value = 1; - for (DataPoint dp : dps[0]) { - assertEquals(value, dp.longValue()); - value++; - } - assertEquals(300, dps[0].aggregatedSize()); + public void configureFromQueryWithRate() throws Exception { + setDataPointStorage(); + final TSQuery ts_query = getTSQuery(); + final RateOptions rate_options = new RateOptions(); + rate_options.setResetValue(1024); + ts_query.getQueries().get(0).setRateOptions(rate_options); + ts_query.validateAndSetQuery(); + query = new TsdbQuery(tsdb); + query.configureFromQuery(ts_query, 0).joinUninterruptibly(); + + assertArrayEquals(METRIC_BYTES, ForTesting.getMetric(query)); + assertEquals(1, ForTesting.getFilters(query).size()); + assertArrayEquals(TAGK_BYTES, ForTesting.getGroupBys(query).get(0)); + assertEquals(1, ForTesting.getGroupBys(query).size()); + assertTrue(rate_options == ForTesting.getRateOptions(query)); } - + @Test - public void runLongSingleTSNoData() throws Exception { - setQueryStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals(0, dps.length); + public void configureFromQueryNoTags() throws Exception { + + setDataPointStorage(); + final TSQuery ts_query = getTSQuery(); + ts_query.getQueries().get(0).setTags(Collections.EMPTY_MAP); + ts_query.validateAndSetQuery(); + query = new TsdbQuery(tsdb); + query.configureFromQuery(ts_query, 0).joinUninterruptibly(); + + assertArrayEquals(METRIC_BYTES, ForTesting.getMetric(query)); + assertEquals(0, ForTesting.getFilters(query).size()); + assertNull(ForTesting.getGroupBys(query)); + assertNull(ForTesting.getRowKeyLiterals(query)); } - + @Test - public void runLongTwoAggSum() throws Exception { - storeLongTimeSeriesSeconds(true, false);; - HashMap<String, String> tags = new HashMap<String, String>(); - query.setStartTime(1356998400L); - query.setEndTime(1357041600L); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - for (DataPoint dp : dps[0]) { - assertEquals(301, dp.longValue()); - } - assertEquals(300, dps[0].size()); + public void configureFromQueryGroupByAll() throws Exception { + setDataPointStorage(); + final TSQuery ts_query = getTSQuery(); + tags.clear(); + tags.put(TAGK_STRING, "*"); + ts_query.getQueries().get(0).setTags(tags); + ts_query.validateAndSetQuery(); + query = new TsdbQuery(tsdb); + query.configureFromQuery(ts_query, 0).joinUninterruptibly(); + + assertArrayEquals(METRIC_BYTES, ForTesting.getMetric(query)); + assertEquals(1, ForTesting.getFilters(query).size()); + assertEquals(1, ForTesting.getGroupBys(query).size()); + assertArrayEquals(TAGK_BYTES, + ForTesting.getGroupBys(query).get(0)); + assertEquals(1, ForTesting.getRowKeyLiterals(query).size()); + assertNull(ForTesting.getRowKeyLiterals(query).get(TAGV_BYTES)); } - + @Test - public void runLongTwoAggSumMs() throws Exception { - storeLongTimeSeriesMs(); - HashMap<String, String> tags = new HashMap<String, String>(); - query.setStartTime(1356998400L); - query.setEndTime(1357041600L); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - for (DataPoint dp : dps[0]) { - assertEquals(301, dp.longValue()); - } - assertEquals(300, dps[0].size()); + public void configureFromQueryGroupByPipe() throws Exception { + setDataPointStorage(); + final TSQuery ts_query = getTSQuery(); + tags.clear(); + tags.put(TAGK_STRING, TAGV_STRING + "|" + TAGV_B_STRING); + ts_query.getQueries().get(0).setTags(tags); + ts_query.validateAndSetQuery(); + query = new TsdbQuery(tsdb); + query.configureFromQuery(ts_query, 0).joinUninterruptibly(); + + assertArrayEquals(METRIC_BYTES, ForTesting.getMetric(query)); + assertEquals(1, ForTesting.getFilters(query).size()); + assertEquals(1, ForTesting.getGroupBys(query).size()); + assertArrayEquals(TAGK_BYTES, ForTesting.getGroupBys(query).get(0)); + assertEquals(1, ForTesting.getRowKeyLiterals(query).size()); + assertEquals(2, ForTesting.getRowKeyLiterals(query).get(TAGV_BYTES).length); + assertArrayEquals(TAGV_BYTES, + ForTesting.getRowKeyLiterals(query).get(TAGV_BYTES)[0]); + assertArrayEquals(TAGV_B_BYTES, + ForTesting.getRowKeyLiterals(query).get(TAGV_BYTES)[1]); } - + @Test - public void runLongTwoGroup() throws Exception { - storeLongTimeSeriesSeconds(true, false);; - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "*"); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals(2, dps.length); - - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); - - assertEquals("sys.cpu.user", dps[1].metricName()); - assertTrue(dps[1].getAggregatedTags().isEmpty()); - assertNull(dps[1].getAnnotations()); - assertEquals("web02", dps[1].getTags().get("host")); - - int value = 1; - for (DataPoint dp : dps[0]) { - assertEquals(value, dp.longValue()); - value++; - } - assertEquals(300, dps[0].size()); - - value = 300; - for (DataPoint dp : dps[1]) { - assertEquals(value, dp.longValue()); - value--; - } - assertEquals(300, dps[1].size()); + public void configureFromQueryWithGroupByFilter() throws Exception { + setDataPointStorage(); + final TSQuery ts_query = getTSQuery(); + tags.clear(); + tags.put("host", TagVWildcardFilter.FILTER_NAME + "(*imes)"); + ts_query.getQueries().get(0).setTags(tags); + ts_query.validateAndSetQuery(); + query = new TsdbQuery(tsdb); + query.configureFromQuery(ts_query, 0).joinUninterruptibly(); + + assertArrayEquals(METRIC_BYTES, ForTesting.getMetric(query)); + assertEquals(1, ForTesting.getFilters(query).size()); + assertEquals(1, ForTesting.getGroupBys(query).size()); + assertArrayEquals(TAGK_BYTES, ForTesting.getGroupBys(query).get(0)); + assertEquals(1, ForTesting.getRowKeyLiterals(query).size()); + assertNull(ForTesting.getRowKeyLiterals(query).get(TAGV_BYTES)); + assertNotNull(ForTesting.getRateOptions(query)); } - + @Test - public void runLongSingleTSRate() throws Exception { - storeLongTimeSeriesSeconds(true, false);; - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, true); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); - - for (DataPoint dp : dps[0]) { - assertEquals(0.033F, dp.doubleValue(), 0.001); - } - assertEquals(299, dps[0].size()); + public void configureFromQueryWithFilter() throws Exception { + setDataPointStorage(); + final TSQuery ts_query = getTSQuery(); + final List<TagVFilter> filters = new ArrayList<TagVFilter>(1); + filters.add(new TagVWildcardFilter("host", "*imes")); + ts_query.getQueries().get(0).setFilters(filters); + ts_query.validateAndSetQuery(); + query = new TsdbQuery(tsdb); + query.configureFromQuery(ts_query, 0).joinUninterruptibly(); + + assertArrayEquals(METRIC_BYTES, ForTesting.getMetric(query)); + assertEquals(1, ForTesting.getFilters(query).size()); + assertNull(ForTesting.getGroupBys(query)); + assertEquals(1, ForTesting.getRowKeyLiterals(query).size()); + assertNull(ForTesting.getRowKeyLiterals(query).get(TAGV_BYTES)); + assertNotNull(ForTesting.getRateOptions(query)); } - + @Test - public void runLongSingleTSRateMs() throws Exception { - storeLongTimeSeriesMs(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, true); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); - - for (DataPoint dp : dps[0]) { - assertEquals(2.0F, dp.doubleValue(), 0.001); - } - assertEquals(299, dps[0].size()); + public void configureFromQueryWithGroupByAndRegularFilters() throws Exception { + setDataPointStorage(); + final TSQuery ts_query = getTSQuery(); + final List<TagVFilter> filters = new ArrayList<TagVFilter>(1); + filters.add(new TagVWildcardFilter("host", "*imes")); + filters.add(TagVFilter.Builder().setFilter("*").setTagk("host") + .setType("wildcard").setGroupBy(true).build()); + ts_query.getQueries().get(0).setFilters(filters); + ts_query.validateAndSetQuery(); + + query = new TsdbQuery(tsdb); + query.configureFromQuery(ts_query, 0).joinUninterruptibly(); + assertArrayEquals(METRIC_BYTES, ForTesting.getMetric(query)); + assertEquals(2, ForTesting.getFilters(query).size()); + assertEquals(1, ForTesting.getGroupBys(query).size()); + assertArrayEquals(TAGK_BYTES, ForTesting.getGroupBys(query).get(0)); + assertEquals(1, ForTesting.getRowKeyLiterals(query).size()); + assertNull(ForTesting.getRowKeyLiterals(query).get(TAGK_BYTES)); + assertNotNull(ForTesting.getRateOptions(query)); } @Test - public void runLongSingleTSCompacted() throws Exception { - storeLongCompactions(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - - int value = 1; - for (DataPoint dp : dps[0]) { - assertEquals(value, dp.longValue()); - value++; - } - assertEquals(300, dps[0].size()); + public void configureFromQueryWithForceRaw() throws Exception { + setDataPointStorage(); + mockEnableRollupQuerySplitting(); + + final TSQuery ts_query = getTSQuery(TsdbQuery.ROLLUP_USAGE.ROLLUP_NOFALLBACK); + ts_query.validateAndSetQuery(); + query = spy(new TsdbQuery(tsdb)); + query.configureFromQuery(ts_query, 0, true).joinUninterruptibly(); + + assertFalse(query.isRollupQuery()); + verify(query, never()).transformDownSamplerToRollupQuery(any(Aggregator.class), anyString()); + + assertArrayEquals(METRIC_BYTES, ForTesting.getMetric(query)); + assertEquals(1, ForTesting.getFilters(query).size()); + assertArrayEquals(TAGK_BYTES, ForTesting.getGroupBys(query).get(0)); + assertEquals(1, ForTesting.getGroupBys(query).size()); + assertNotNull(ForTesting.getRateOptions(query)); } - - // Can't run this one since the TreeMap will order the compacted row AFTER - // the other data points. A full MockBase implementation would allow this -// @Test -// public void runLongSingleTSCompactedAndNonCompacted() throws Exception { -// storeLongCompactions(); -// HashMap<String, String> tags = new HashMap<String, String>(1); -// tags.put("host", "web01"); -// -// long timestamp = 1357007460; -// for (int i = 301; i <= 310; i++) { -// tsdb.addPoint("sys.cpu.user", timestamp += 30, i, tags).joinUninterruptibly(); -// } -// storage.dumpToSystemOut(false); -// query.setStartTime(1356998400); -// query.setEndTime(1357041600); -// query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); -// final DataPoints[] dps = query.run(); -// assertNotNull(dps); -// -// int value = 1; -// for (DataPoint dp : dps[0]) { -// assertEquals(value, dp.longValue()); -// value++; -// } -// assertEquals(310, dps[0].size()); -// } - - @Test - public void runFloatSingleTS() throws Exception { - storeFloatTimeSeriesSeconds(true, false); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); - - double value = 1.25D; - for (DataPoint dp : dps[0]) { - assertEquals(value, dp.doubleValue(), 0.001); - value += 0.25D; - } - assertEquals(300, dps[0].size()); + + @Test (expected = IllegalArgumentException.class) + public void configureFromQueryNullSubs() throws Exception { + final TSQuery ts_query = new TSQuery(); + new TsdbQuery(tsdb).configureFromQuery(ts_query, 0); } - - @Test - public void runFloatSingleTSMs() throws Exception { - storeFloatTimeSeriesMs(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); - - double value = 1.25D; - for (DataPoint dp : dps[0]) { - assertEquals(value, dp.doubleValue(), 0.001); - value += 0.25D; - } - assertEquals(300, dps[0].size()); + + @Test (expected = IllegalArgumentException.class) + public void configureFromQueryEmptySubs() throws Exception { + final TSQuery ts_query = new TSQuery(); + ts_query.setQueries(new ArrayList<TSSubQuery>(0)); + new TsdbQuery(tsdb).configureFromQuery(ts_query, 0); } - + + @Test (expected = IllegalArgumentException.class) + public void configureFromQueryNegativeIndex() throws Exception { + final TSQuery ts_query = getTSQuery(); + new TsdbQuery(tsdb).configureFromQuery(ts_query, -1); + } + + @Test (expected = IllegalArgumentException.class) + public void configureFromQueryIndexOutOfBounds() throws Exception { + final TSQuery ts_query = getTSQuery(); + new TsdbQuery(tsdb).configureFromQuery(ts_query, 2); + } + + @Test (expected = NoSuchUniqueName.class) + public void configureFromQueryNSUMetric() throws Exception { + setDataPointStorage(); + final TSQuery ts_query = getTSQuery(); + ts_query.getQueries().get(0).setMetric(NSUN_METRIC); + ts_query.validateAndSetQuery(); + query = new TsdbQuery(tsdb); + query.configureFromQuery(ts_query, 0).joinUninterruptibly(); + } + + @Test (expected = DeferredGroupException.class) + public void configureFromQueryNSUTagk() throws Exception { + setDataPointStorage(); + final TSQuery ts_query = getTSQuery(); + tags.clear(); + tags.put(NSUN_TAGK, TAGV_STRING); + ts_query.getQueries().get(0).setTags(tags); + ts_query.validateAndSetQuery(); + query = new TsdbQuery(tsdb); + query.configureFromQuery(ts_query, 0).joinUninterruptibly(); + } + + @Test (expected = DeferredGroupException.class) + public void configureFromQueryNSUTagv() throws Exception { + setDataPointStorage(); + final TSQuery ts_query = getTSQuery(); + tags.clear(); + tags.put(TAGK_STRING, NSUN_TAGV); + ts_query.getQueries().get(0).setTags(tags); + ts_query.validateAndSetQuery(); + query = new TsdbQuery(tsdb); + query.configureFromQuery(ts_query, 0).joinUninterruptibly(); + } + + @Test (expected = DeferredGroupException.class) + public void configureFromQueryGroupByPipeNSUTagk() throws Exception { + setDataPointStorage(); + final TSQuery ts_query = getTSQuery(); + tags.clear(); + tags.put(NSUN_TAGK, TAGV_STRING + "|" + TAGV_B_STRING); + ts_query.getQueries().get(0).setTags(tags); + ts_query.validateAndSetQuery(); + query = new TsdbQuery(tsdb); + query.configureFromQuery(ts_query, 0).joinUninterruptibly(); + } + + @Test (expected = DeferredGroupException.class) + public void configureFromQueryGroupByPipeNSUTagv() throws Exception { + setDataPointStorage(); + final TSQuery ts_query = getTSQuery(); + tags.clear(); + tags.put(TAGK_STRING, TAGV_STRING + "|" + NSUN_TAGV); + ts_query.getQueries().get(0).setTags(tags); + ts_query.validateAndSetQuery(); + query = new TsdbQuery(tsdb); + query.configureFromQuery(ts_query, 0).joinUninterruptibly(); + } + @Test - public void runFloatTwoAggSum() throws Exception { - storeFloatTimeSeriesSeconds(true, false); - HashMap<String, String> tags = new HashMap<String, String>(); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - for (DataPoint dp : dps[0]) { - assertEquals(76.25, dp.doubleValue(), 0.00001); - } - assertEquals(300, dps[0].size()); + public void configureFromQueryGroupByPipeNSUTagvSkipUnresolved() + throws Exception { + config.overrideConfig("tsd.query.skip_unresolved_tagvs", "true"); + setDataPointStorage(); + final TSQuery ts_query = getTSQuery(); + tags.clear(); + tags.put(TAGK_STRING, TAGV_STRING + "|" + NSUN_TAGV); + ts_query.getQueries().get(0).setTags(tags); + ts_query.validateAndSetQuery(); + query = new TsdbQuery(tsdb); + query.configureFromQuery(ts_query, 0).joinUninterruptibly(); + + assertArrayEquals(METRIC_BYTES, ForTesting.getMetric(query)); + assertEquals(1, ForTesting.getFilters(query).size()); + assertEquals(1, ForTesting.getGroupBys(query).size()); + assertArrayEquals(TAGK_BYTES, + ForTesting.getGroupBys(query).get(0)); } - + @Test - public void runFloatTwoAggSumMs() throws Exception { - storeFloatTimeSeriesMs(); - HashMap<String, String> tags = new HashMap<String, String>(); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); + public void configureFromQueryMaxBytes() throws Exception { + TSQuery ts_query = getTSQuery(); + ts_query.validateAndSetQuery(); + query = new TsdbQuery(tsdb); + query.configureFromQuery(ts_query, 0).joinUninterruptibly(); + assertEquals(config.getInt("tsd.query.limits.bytes.default"), + ForTesting.maxBytes(query)); + + config.overrideConfig("tsd.query.limits.bytes.default", "128"); + config.overrideConfig("tsd.query.limits.data_points.default", "16"); + Whitebox.setInternalState(tsdb, "query_limits", + new QueryLimitOverride(tsdb)); + ts_query = getTSQuery(); + ts_query.validateAndSetQuery(); + query = new TsdbQuery(tsdb); + query.configureFromQuery(ts_query, 0).joinUninterruptibly(); + assertEquals(128, ForTesting.maxBytes(query)); - for (DataPoint dp : dps[0]) { - assertEquals(76.25, dp.doubleValue(), 0.00001); - } - assertEquals(300, dps[0].size()); + // disabled by default + ts_query = getTSQuery(); + ts_query.setOverrideByteLimit(true); + ts_query.validateAndSetQuery(); + query = new TsdbQuery(tsdb); + query.configureFromQuery(ts_query, 0).joinUninterruptibly(); + assertEquals(128, ForTesting.maxBytes(query)); + + config.overrideConfig("tsd.query.limits.bytes.allow_override", "true"); + ts_query = getTSQuery(); + ts_query.setOverrideByteLimit(true); + ts_query.validateAndSetQuery(); + query = new TsdbQuery(tsdb); + query.configureFromQuery(ts_query, 0).joinUninterruptibly(); + assertEquals(0, ForTesting.maxBytes(query)); } @Test - public void runFloatTwoGroup() throws Exception { - storeFloatTimeSeriesSeconds(true, false); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "*"); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals(2, dps.length); - - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); - - assertEquals("sys.cpu.user", dps[1].metricName()); - assertTrue(dps[1].getAggregatedTags().isEmpty()); - assertNull(dps[1].getAnnotations()); - assertEquals("web02", dps[1].getTags().get("host")); + public void configureFromQueryMaxDataPoints() throws Exception { + TSQuery ts_query = getTSQuery(); + ts_query.validateAndSetQuery(); + query = new TsdbQuery(tsdb); + query.configureFromQuery(ts_query, 0).joinUninterruptibly(); + assertEquals(config.getInt("tsd.query.limits.data_points.default"), + ForTesting.maxDataPoints(query)); + + config.overrideConfig("tsd.query.limits.bytes.default", "128"); + config.overrideConfig("tsd.query.limits.data_points.default", "16"); + Whitebox.setInternalState(tsdb, "query_limits", + new QueryLimitOverride(tsdb)); + ts_query = getTSQuery(); + ts_query.validateAndSetQuery(); + query = new TsdbQuery(tsdb); + query.configureFromQuery(ts_query, 0).joinUninterruptibly(); + assertEquals(16, ForTesting.maxDataPoints(query)); - double value = 1.25D; - for (DataPoint dp : dps[0]) { - assertEquals(value, dp.doubleValue(), 0.0001); - value += 0.25D; - } - assertEquals(300, dps[0].size()); + // disabled by default + ts_query = getTSQuery(); + ts_query.setOverrideDataPointLimit(true); + ts_query.validateAndSetQuery(); + query = new TsdbQuery(tsdb); + query.configureFromQuery(ts_query, 0).joinUninterruptibly(); + assertEquals(16, ForTesting.maxDataPoints(query)); - value = 75D; - for (DataPoint dp : dps[1]) { - assertEquals(value, dp.doubleValue(), 0.0001); - value -= 0.25d; - } - assertEquals(300, dps[1].size()); + config.overrideConfig("tsd.query.limits.data_points.allow_override", "true"); + ts_query = getTSQuery(); + ts_query.setOverrideDataPointLimit(true); + ts_query.validateAndSetQuery(); + query = new TsdbQuery(tsdb); + query.configureFromQuery(ts_query, 0).joinUninterruptibly(); + assertEquals(0, ForTesting.maxDataPoints(query)); } @Test - public void runFloatSingleTSRate() throws Exception { - storeFloatTimeSeriesSeconds(true, false); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); + public void deleteDatapoints() throws Exception { + setDataPointStorage(); + + tsdb.addPoint(METRIC_STRING, 1356998400, 42, tags).joinUninterruptibly(); query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, true); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); - - for (DataPoint dp : dps[0]) { - assertEquals(0.00833F, dp.doubleValue(), 0.00001); - } - assertEquals(299, dps[0].size()); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + + query.setDelete(true); + final DataPoints[] dps1 = query.run(); + assertEquals(1, dps1.length); + // second run should be empty + final DataPoints[] dps2 = query.run(); + assertEquals(0, dps2.length); } - + @Test - public void runFloatSingleTSRateMs() throws Exception { - storeFloatTimeSeriesMs(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); + public void scannerException() throws Exception { + storeLongTimeSeriesSeconds(true, false); + final RuntimeException ex = new RuntimeException("Boo!"); + storage.throwException(MockBase.stringToBytes( + "00000150E22700000001000001"), ex, true); + + storage.dumpToSystemOut(); query.setStartTime(1356998400); query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, true); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); - - for (DataPoint dp : dps[0]) { - assertEquals(0.5F, dp.doubleValue(), 0.00001); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + try { + query.run(); + fail("Expected a RuntimeException"); + } catch (RuntimeException e) { + assertSame(ex, e); } - assertEquals(299, dps[0].size()); } @Test - public void runFloatSingleTSCompacted() throws Exception { - storeFloatCompactions(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); - - double value = 1.25D; - for (DataPoint dp : dps[0]) { - assertEquals(value, dp.doubleValue(), 0.001); - value += 0.25D; - } - assertEquals(300, dps[0].size()); + public void needsSplittingReturnsFalseIfDisabled() { + Whitebox.setInternalState(tsdb, "rollups_split_queries", false); + assertFalse(query.needsSplitting()); } - + @Test - public void runMixedSingleTS() throws Exception { - storeMixedTimeSeriesSeconds(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.AVG, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); - - double float_value = 1.25D; - int int_value = 76; - // due to aggregation, the only int that will be returned will be the very - // last value of 76 since the agg will convert every point in between to a - // double - for (DataPoint dp : dps[0]) { - if (dp.isInteger()) { - assertEquals(int_value, dp.longValue()); - int_value++; - float_value = int_value; - } else { - assertEquals(float_value, dp.doubleValue(), 0.001); - float_value += 0.25D; - } - } - assertEquals(300, dps[0].size()); + public void needsSplittingReturnsFalseIfNotARollupQuery() { + Whitebox.setInternalState(tsdb, "rollups_split_queries", true); + Whitebox.setInternalState(query, "rollup_query", (RollupQuery) null); + assertFalse(query.needsSplitting()); } - + @Test - public void runMixedSingleTSMsAndS() throws Exception { - storeMixedTimeSeriesMsAndS(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.AVG, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); - - double float_value = 1.25D; - int int_value = 76; - // due to aggregation, the only int that will be returned will be the very - // last value of 76 since the agg will convert every point in between to a - // double - for (DataPoint dp : dps[0]) { - if (dp.isInteger()) { - assertEquals(int_value, dp.longValue()); - int_value++; - float_value = int_value; - } else { - assertEquals(float_value, dp.doubleValue(), 0.001); - float_value += 0.25D; - } - } - assertEquals(300, dps[0].size()); + public void needsSplittingReturnsFalseIfNoSLAConfigured() { + Whitebox.setInternalState(tsdb, "rollups_split_queries", true); + RollupInterval oneHourWithDelay = RollupInterval.builder() + .setTable("fake-rollup-table") + .setPreAggregationTable("fake-preagg-table") + .setInterval("1h") + .setRowSpan("1d") + .setDelaySla(null) + .build(); + RollupQuery rollup_query = new RollupQuery( + oneHourWithDelay, + Aggregators.SUM, + 3600000, + Aggregators.SUM + ); + Whitebox.setInternalState(query, "rollup_query", rollup_query); + + assertTrue(query.isRollupQuery()); + + assertFalse(query.needsSplitting()); } - + @Test - public void runMixedSingleTSPostCompaction() throws Exception { - storeMixedTimeSeriesSeconds(); - - final Field compact = Config.class.getDeclaredField("enable_compactions"); - compact.setAccessible(true); - compact.set(config, true); - - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.AVG, false); - assertNotNull(query.run()); - - // this should only compact the rows for the time series that we fetched and - // leave the others alone - assertEquals(1, storage.numColumns( - MockBase.stringToBytes("00000150E22700000001000001"))); - assertEquals(1, storage.numColumns( - MockBase.stringToBytes("00000150E23510000001000001"))); - assertEquals(1, storage.numColumns( - MockBase.stringToBytes("00000150E24320000001000001"))); - - // run it again to verify the compacted data uncompacts properly - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); - - double float_value = 1.25D; - int int_value = 76; - // due to aggregation, the only int that will be returned will be the very - // last value of 76 since the agg will convert every point in between to a - // double - for (DataPoint dp : dps[0]) { - if (dp.isInteger()) { - assertEquals(int_value, dp.longValue()); - int_value++; - float_value = int_value; - } else { - assertEquals(float_value, dp.doubleValue(), 0.001); - float_value += 0.25D; - } - } - assertEquals(300, dps[0].size()); + public void needsSplittingReturnsFalseIfNotInBlackoutPeriod() { + mockSystemTime(1356998400000L); + mockEnableRollupQuerySplitting(); + + query.setStartTime(0); + query.setEndTime(1); + + assertTrue(query.isRollupQuery()); + + assertFalse(query.needsSplitting()); } - + @Test - public void runMixedSingleTSCompacted() throws Exception { - storeMixedCompactions(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.AVG, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); - - double float_value = 1.25D; - int int_value = 76; - // due to aggregation, the only int that will be returned will be the very - // last value of 76 since the agg will convert every point in between to a - // double - for (DataPoint dp : dps[0]) { - if (dp.isInteger()) { - assertEquals(int_value, dp.longValue()); - int_value++; - float_value = int_value; - } else { - assertEquals(float_value, dp.doubleValue(), 0.001); - float_value += 0.25D; - } - } - assertEquals(300, dps[0].size()); - } - - @Test - public void runEndTime() throws Exception { - storeLongTimeSeriesSeconds(true, false);; - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - query.setStartTime(1356998400); - query.setEndTime(1357001900); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - - int value = 1; - for (DataPoint dp : dps[0]) { - assertEquals(value, dp.longValue()); - value++; - } - assertEquals(236, dps[0].size()); - } - - @Test - public void runCompactPostQuery() throws Exception { - storeLongTimeSeriesSeconds(true, false);; - - final Field compact = Config.class.getDeclaredField("enable_compactions"); - compact.setAccessible(true); - compact.set(config, true); - - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - query.setStartTime(1356998400); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); - assertNotNull(query.run()); - - // this should only compact the rows for the time series that we fetched and - // leave the others alone - assertEquals(1, storage.numColumns( - MockBase.stringToBytes("00000150E22700000001000001"))); - assertEquals(119, storage.numColumns( - MockBase.stringToBytes("00000150E22700000001000002"))); - assertEquals(1, storage.numColumns( - MockBase.stringToBytes("00000150E23510000001000001"))); - assertEquals(120, storage.numColumns( - MockBase.stringToBytes("00000150E23510000001000002"))); - assertEquals(1, storage.numColumns( - MockBase.stringToBytes("00000150E24320000001000001"))); - assertEquals(61, storage.numColumns( - MockBase.stringToBytes("00000150E24320000001000002"))); - - // run it again to verify the compacted data uncompacts properly - final DataPoints[] dps = query.run(); - assertNotNull(dps); - - int value = 1; - for (DataPoint dp : dps[0]) { - assertEquals(value, dp.longValue()); - value++; - } - assertEquals(300, dps[0].size()); - } - - @Test (expected = IllegalStateException.class) - public void runStartNotSet() throws Exception { - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); - query.run(); - } + public void needsSplittingReturnsFalseIfQueryEndsWithLastRollupTimestamp() { + mockSystemTime(1356998400000L); + mockEnableRollupQuerySplitting(); - @Test - public void runFloatAndIntSameTS() throws Exception { - // if a row has an integer and a float for the same timestamp, there will be - // two different qualifiers that will resolve to the same offset. This no - // longer tosses an exception, and keeps the last value - storeLongTimeSeriesSeconds(true, false); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - tsdb.addPoint("sys.cpu.user", 1356998430, 42.5F, tags).joinUninterruptibly(); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, true); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - // TODO: further validate the result - } - - @Test - public void runWithAnnotation() throws Exception { - storeLongTimeSeriesSeconds(true, false);; - - final Annotation note = new Annotation(); - note.setTSUID("000001000001000001"); - note.setStartTime(1356998490); - note.setDescription("Hello World!"); - note.syncToStorage(tsdb, false).joinUninterruptibly(); - - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); + query.setStartTime(0); + query.setEndTime(query.getRollupQuery().getLastRollupTimestampSeconds() * 1000L); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals(1, dps[0].getAnnotations().size()); - assertEquals("Hello World!", dps[0].getAnnotations().get(0).getDescription()); - - int value = 1; - for (DataPoint dp : dps[0]) { - assertEquals(value, dp.longValue()); - value++; - } - assertEquals(300, dps[0].size()); + assertTrue(query.isRollupQuery()); + + assertFalse(query.needsSplitting()); } - + @Test - public void runWithAnnotationPostCompact() throws Exception { - storeLongTimeSeriesSeconds(true, false);; + public void needsSplittingReturnsTrueIfQueryStartsWithLastRollupTimestamp() { + long mockNowTimestamp = 1356998400000L; + mockSystemTime(mockNowTimestamp); mockEnableRollupQuerySplitting(); - final Annotation note = new Annotation(); - note.setTSUID("000001000001000001"); - note.setStartTime(1356998490); - note.setDescription("Hello World!"); - note.syncToStorage(tsdb, false).joinUninterruptibly(); + query.setStartTime(query.getRollupQuery().getLastRollupTimestampSeconds() * 1000L); - final Field compact = Config.class.getDeclaredField("enable_compactions"); - compact.setAccessible(true); - compact.set(config, true); + assertTrue(query.isRollupQuery()); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); - assertNotNull(query.run()); - - // this should only compact the rows for the time series that we fetched and - // leave the others alone - assertEquals(2, storage.numColumns( - MockBase.stringToBytes("00000150E22700000001000001"))); - assertEquals(119, storage.numColumns( - MockBase.stringToBytes("00000150E22700000001000002"))); - assertEquals(1, storage.numColumns( - MockBase.stringToBytes("00000150E23510000001000001"))); - assertEquals(120, storage.numColumns( - MockBase.stringToBytes("00000150E23510000001000002"))); - assertEquals(1, storage.numColumns( - MockBase.stringToBytes("00000150E24320000001000001"))); - assertEquals(61, storage.numColumns( - MockBase.stringToBytes("00000150E24320000001000002"))); - - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals(1, dps[0].getAnnotations().size()); - assertEquals("Hello World!", dps[0].getAnnotations().get(0).getDescription()); - - int value = 1; - for (DataPoint dp : dps[0]) { - assertEquals(value, dp.longValue()); - value++; - } - assertEquals(300, dps[0].size()); + assertTrue(query.needsSplitting()); } @Test - public void runWithOnlyAnnotation() throws Exception { - storeLongTimeSeriesSeconds(true, false);; - - // verifies that we can pickup an annotation stored all by it's lonesome - // in a row without any data - storage.flushRow(MockBase.stringToBytes("00000150E23510000001000001")); - final Annotation note = new Annotation(); - note.setTSUID("000001000001000001"); - note.setStartTime(1357002090); - note.setDescription("Hello World!"); - note.syncToStorage(tsdb, false).joinUninterruptibly(); - - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); - - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals(1, dps[0].getAnnotations().size()); - assertEquals("Hello World!", dps[0].getAnnotations().get(0).getDescription()); - - int value = 1; - for (DataPoint dp : dps[0]) { - assertEquals(value, dp.longValue()); - value++; - // account for the jump - if (value == 120) { - value = 240; - } - } - assertEquals(180, dps[0].size()); - } + public void needsSplittingReturnsTrueIfInBlackoutPeriod() { + long mockNowTimestamp = 1356998400000L; + mockSystemTime(mockNowTimestamp); + mockEnableRollupQuerySplitting(); - @Test - public void runWithSingleAnnotation() throws Exception { - setQueryStorage(); - - // verifies that we can pickup an annotation stored all by it's lonesome - // in a row without any data - storage.flushRow(MockBase.stringToBytes("00000150E23510000001000001")); - final Annotation note = new Annotation(); - note.setTSUID("000001000001000001"); - note.setStartTime(1357002090); - note.setDescription("Hello World!"); - note.syncToStorage(tsdb, false).joinUninterruptibly(); - - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); + query.setStartTime(0L); + query.setEndTime(mockNowTimestamp); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals(1, dps[0].getAnnotations().size()); - assertEquals("Hello World!", dps[0].getAnnotations().get(0).getDescription()); + assertTrue(query.isRollupQuery()); - assertEquals(0, dps[0].size()); + assertTrue(query.needsSplitting()); } @Test - public void runSingleDataPoint() throws Exception { - setQueryStorage(); - // dump a bunch of rows of two metrics so that we can test filtering out - // on the metric - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - long timestamp = 1356998410; - tsdb.addPoint("sys.cpu.user", timestamp, 42, tags).joinUninterruptibly(); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - final List<String> tsuids = new ArrayList<String>(1); - tsuids.add("000001000001000001"); - query.setTimeSeries(tsuids, Aggregators.SUM, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals(1, dps.length); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); - assertEquals(42, dps[0].longValue(0)); - } + public void needsSplittingReturnsTrueIfStartAndEndInBlackoutPeriod() { + long mockNowTimestamp = 1356998400000L; + mockSystemTime(mockNowTimestamp); + mockEnableRollupQuerySplitting(); - @Test - public void runSingleDataPointWithAnnotation() throws Exception { - setQueryStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - long timestamp = 1356998410; - tsdb.addPoint("sys.cpu.user", timestamp, 42, tags).joinUninterruptibly(); - storage.flushRow(MockBase.stringToBytes("00000150E23510000001000001")); - final Annotation note = new Annotation(); - note.setTSUID("000001000001000001"); - note.setStartTime(1357002090); - note.setDescription("Hello World!"); - note.syncToStorage(tsdb, false).joinUninterruptibly(); + int oneHour = 60 * 60 * 1000; - query.setStartTime(1356998400); - query.setEndTime(1357041600); - final List<String> tsuids = new ArrayList<String>(1); - tsuids.add("000001000001000001"); - query.setTimeSeries(tsuids, Aggregators.SUM, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals(1, dps.length); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertEquals("web01", dps[0].getTags().get("host")); - assertEquals(42, dps[0].longValue(0)); - assertEquals(1, dps[0].getAnnotations().size()); - assertEquals("Hello World!", dps[0].getAnnotations().get(0).getDescription()); - } + query.setStartTime(mockNowTimestamp - oneHour); + query.setEndTime(mockNowTimestamp); - @Test - public void runTSUIDQuery() throws Exception { - storeLongTimeSeriesSeconds(true, false);; - query.setStartTime(1356998400); - query.setEndTime(1357041600); - final List<String> tsuids = new ArrayList<String>(1); - tsuids.add("000001000001000001"); - query.setTimeSeries(tsuids, Aggregators.SUM, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); - - int value = 1; - for (DataPoint dp : dps[0]) { - assertEquals(value, dp.longValue()); - value++; - } - assertEquals(300, dps[0].aggregatedSize()); - } - - @Test - public void runTSUIDsAggSum() throws Exception { - storeLongTimeSeriesSeconds(true, false);; - query.setStartTime(1356998400); - query.setEndTime(1357041600); - final List<String> tsuids = new ArrayList<String>(1); - tsuids.add("000001000001000001"); - tsuids.add("000001000001000002"); - query.setTimeSeries(tsuids, Aggregators.SUM, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - for (DataPoint dp : dps[0]) { - assertEquals(301, dp.longValue()); - } - assertEquals(300, dps[0].size()); - } - - @Test - public void runTSUIDQueryNoData() throws Exception { - setQueryStorage(); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - final List<String> tsuids = new ArrayList<String>(1); - tsuids.add("000001000001000001"); - query.setTimeSeries(tsuids, Aggregators.SUM, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals(0, dps.length); - } - - @Test - public void runTSUIDQueryNoDataForTSUID() throws Exception { - // this doesn't throw an exception since the UIDs are only looked for when - // the query completes. - setQueryStorage(); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - final List<String> tsuids = new ArrayList<String>(1); - tsuids.add("000001000001000005"); - query.setTimeSeries(tsuids, Aggregators.SUM, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals(0, dps.length); - } - - @Test (expected = NoSuchUniqueId.class) - public void runTSUIDQueryNSU() throws Exception { - when(metrics.getNameAsync(new byte[] { 0, 0, 1 })) - .thenThrow(new NoSuchUniqueId("metrics", new byte[] { 0, 0, 1 })); - storeLongTimeSeriesSeconds(true, false);; - query.setStartTime(1356998400); - query.setEndTime(1357041600); - final List<String> tsuids = new ArrayList<String>(1); - tsuids.add("000001000001000001"); - query.setTimeSeries(tsuids, Aggregators.SUM, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - dps[0].metricName(); - } - - @Test - public void runRateCounterDefault() throws Exception { - setQueryStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - long timestamp = 1356998400; - tsdb.addPoint("sys.cpu.user", timestamp += 30, Long.MAX_VALUE - 55, tags) - .joinUninterruptibly(); - tsdb.addPoint("sys.cpu.user", timestamp += 30, Long.MAX_VALUE - 25, tags) - .joinUninterruptibly(); - tsdb.addPoint("sys.cpu.user", timestamp += 30, 5, tags).joinUninterruptibly(); - - RateOptions ro = new RateOptions(true, Long.MAX_VALUE, 0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, true, ro); - final DataPoints[] dps = query.run(); + assertTrue(query.isRollupQuery()); - for (DataPoint dp : dps[0]) { - assertEquals(1.0, dp.doubleValue(), 0.001); - } - assertEquals(2, dps[0].size()); + assertTrue(query.needsSplitting()); } - - @Test - public void runRateCounterDefaultNoOp() throws Exception { - setQueryStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - long timestamp = 1356998400; - tsdb.addPoint("sys.cpu.user", timestamp += 30, 30, tags).joinUninterruptibly(); - tsdb.addPoint("sys.cpu.user", timestamp += 30, 60, tags).joinUninterruptibly(); - tsdb.addPoint("sys.cpu.user", timestamp += 30, 90, tags).joinUninterruptibly(); - - RateOptions ro = new RateOptions(true, Long.MAX_VALUE, 0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, true, ro); - final DataPoints[] dps = query.run(); - for (DataPoint dp : dps[0]) { - assertEquals(1.0, dp.doubleValue(), 0.001); - } - assertEquals(2, dps[0].size()); - } - @Test - public void runRateCounterMaxSet() throws Exception { - setQueryStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - long timestamp = 1356998400; - tsdb.addPoint("sys.cpu.user", timestamp += 30, 45, tags).joinUninterruptibly(); - tsdb.addPoint("sys.cpu.user", timestamp += 30, 75, tags).joinUninterruptibly(); - tsdb.addPoint("sys.cpu.user", timestamp += 30, 5, tags).joinUninterruptibly(); - - RateOptions ro = new RateOptions(true, 100, 0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, true, ro); - final DataPoints[] dps = query.run(); + public void split() { + long mockSystemTime = 1356998400000L; + mockSystemTime(mockSystemTime); + mockEnableRollupQuerySplitting(); - for (DataPoint dp : dps[0]) { - assertEquals(1.0, dp.doubleValue(), 0.001); - } - assertEquals(2, dps[0].size()); - } - - @Test - public void runRateCounterAnomally() throws Exception { - setQueryStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - long timestamp = 1356998400; - tsdb.addPoint("sys.cpu.user", timestamp += 30, 45, tags).joinUninterruptibly(); - tsdb.addPoint("sys.cpu.user", timestamp += 30, 75, tags).joinUninterruptibly(); - tsdb.addPoint("sys.cpu.user", timestamp += 30, 25, tags).joinUninterruptibly(); - - RateOptions ro = new RateOptions(true, 10000, 35); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, true, ro); - final DataPoints[] dps = query.run(); + TSQuery tsQuery = getTSQuery(); + TsdbQuery rawQuery = spy(new TsdbQuery(tsdb)); - assertEquals(1.0, dps[0].doubleValue(0), 0.001); - assertEquals(0, dps[0].doubleValue(1), 0.001); - assertEquals(2, dps[0].size()); - } + query.setStartTime(mockSystemTime - 7 * ONE_DAY_MS); - @Test - public void runMultiCompact() throws Exception { - final byte[] qual1 = { 0x00, 0x07 }; - final byte[] val1 = Bytes.fromLong(1L); - final byte[] qual2 = { 0x00, 0x27 }; - final byte[] val2 = Bytes.fromLong(2L); - - // 2nd compaction - final byte[] qual3 = { 0x00, 0x37 }; - final byte[] val3 = Bytes.fromLong(3L); - final byte[] qual4 = { 0x00, 0x47 }; - final byte[] val4 = Bytes.fromLong(4L); - - // 3rd compaction - final byte[] qual5 = { 0x00, 0x57 }; - final byte[] val5 = Bytes.fromLong(5L); - final byte[] qual6 = { 0x00, 0x67 }; - final byte[] val6 = Bytes.fromLong(6L); - - final byte[] KEY = { 0, 0, 1, 0x50, (byte) 0xE2, - 0x27, 0x00, 0, 0, 1, 0, 0, 1 }; - - setQueryStorage(); - storage.addColumn(KEY, - MockBase.concatByteArrays(qual1, qual2), - MockBase.concatByteArrays(val1, val2, new byte[] { 0 })); - storage.addColumn(KEY, - MockBase.concatByteArrays(qual3, qual4), - MockBase.concatByteArrays(val3, val4, new byte[] { 0 })); - storage.addColumn(KEY, - MockBase.concatByteArrays(qual5, qual6), - MockBase.concatByteArrays(val5, val6, new byte[] { 0 })); - - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); - - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); - - int value = 1; - for (DataPoint dp : dps[0]) { - assertEquals(value, dp.longValue()); - value++; - } - assertEquals(6, dps[0].aggregatedSize()); - } + doReturn(Deferred.fromResult(null)).when(rawQuery).configureFromQuery(eq(tsQuery), eq(0), eq(true)); - @Test - public void runMultiCompactAndSingles() throws Exception { - final byte[] qual1 = { 0x00, 0x07 }; - final byte[] val1 = Bytes.fromLong(1L); - final byte[] qual2 = { 0x00, 0x27 }; - final byte[] val2 = Bytes.fromLong(2L); - - // 2nd compaction - final byte[] qual3 = { 0x00, 0x37 }; - final byte[] val3 = Bytes.fromLong(3L); - final byte[] qual4 = { 0x00, 0x47 }; - final byte[] val4 = Bytes.fromLong(4L); - - // 3rd compaction - final byte[] qual5 = { 0x00, 0x57 }; - final byte[] val5 = Bytes.fromLong(5L); - final byte[] qual6 = { 0x00, 0x67 }; - final byte[] val6 = Bytes.fromLong(6L); - - final byte[] KEY = { 0, 0, 1, 0x50, (byte) 0xE2, - 0x27, 0x00, 0, 0, 1, 0, 0, 1 }; - - setQueryStorage(); - storage.addColumn(KEY, - MockBase.concatByteArrays(qual1, qual2), - MockBase.concatByteArrays(val1, val2, new byte[] { 0 })); - storage.addColumn(KEY, qual3, val3); - storage.addColumn(KEY, qual4, val4); - storage.addColumn(KEY, - MockBase.concatByteArrays(qual5, qual6), - MockBase.concatByteArrays(val5, val6, new byte[] { 0 })); - - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); - - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); - - int value = 1; - for (DataPoint dp : dps[0]) { - assertEquals(value, dp.longValue()); - value++; - } - assertEquals(6, dps[0].aggregatedSize()); - } - - @Test - public void runInterpolationSeconds() throws Exception { - setQueryStorage(); - - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - long timestamp = 1356998400; - for (int i = 1; i <= 300; i++) { - tsdb.addPoint("sys.cpu.user", timestamp += 30, i, tags) - .joinUninterruptibly(); - } - - tags.clear(); - tags.put("host", "web02"); - timestamp = 1356998415; - for (int i = 300; i > 0; i--) { - tsdb.addPoint("sys.cpu.user", timestamp += 30, i, tags) - .joinUninterruptibly(); - } - - tags.clear(); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - long v = 1; - long ts = 1356998430000L; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 15000; - assertEquals(v, dp.longValue()); - - if (dp.timestamp() == 1357007400000L) { - v = 1; - } else if (v == 1 || v == 302) { - v = 301; - } else { - v = 302; - } - } - assertEquals(600, dps[0].size()); - } - - @Test - public void runInterpolationMs() throws Exception { - setQueryStorage(); - - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - long timestamp = 1356998400000L; - for (int i = 1; i <= 300; i++) { - tsdb.addPoint("sys.cpu.user", timestamp += 500, i, tags) - .joinUninterruptibly(); - } - - tags.clear(); - tags.put("host", "web02"); - timestamp = 1356998400250L; - for (int i = 300; i > 0; i--) { - tsdb.addPoint("sys.cpu.user", timestamp += 500, i, tags) - .joinUninterruptibly(); - } - - tags.clear(); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - long v = 1; - long ts = 1356998400500L; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 250; - assertEquals(v, dp.longValue()); - - if (dp.timestamp() == 1356998550000L) { - v = 1; - } else if (v == 1 || v == 302) { - v = 301; - } else { - v = 302; - } - } - assertEquals(600, dps[0].size()); - } - - @Test - public void runInterpolationMsDownsampled() throws Exception { - setQueryStorage(); - - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - // ts = 1356998400500, v = 1 - // ts = 1356998401000, v = 2 - // ts = 1356998401500, v = 3 - // ts = 1356998402000, v = 4 - // ts = 1356998402500, v = 5 - // ... - // ts = 1356998449000, v = 98 - // ts = 1356998449500, v = 99 - // ts = 1356998450000, v = 100 - // ts = 1356998455000, v = 101 - // ts = 1356998460000, v = 102 - // ... - // ts = 1356998550000, v = 120 - long timestamp = 1356998400000L; - for (int i = 1; i <= 120; i++) { - timestamp += i <= 100 ? 500 : 5000; - tsdb.addPoint("sys.cpu.user", timestamp, i, tags) - .joinUninterruptibly(); - } - - // ts = 1356998400750, v = 300 - // ts = 1356998401250, v = 299 - // ts = 1356998401750, v = 298 - // ts = 1356998402250, v = 297 - // ts = 1356998402750, v = 296 - // ... - // ts = 1356998549250, v = 3 - // ts = 1356998549750, v = 2 - // ts = 1356998550250, v = 1 - tags.clear(); - tags.put("host", "web02"); - timestamp = 1356998400250L; - for (int i = 300; i > 0; i--) { - tsdb.addPoint("sys.cpu.user", timestamp += 500, i, tags) - .joinUninterruptibly(); - } - - tags.clear(); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); - query.downsample(1000, Aggregators.SUM); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - // TS1 in intervals = (1), (2,3), (4,5) ... (98,99), 100, (), (), (), (), - // (101), ... (120) - // TS2 in intervals = (300), (299,298), (297,296), ... (203, 202) ... - // (3,2), (1) - // TS1 downsample = 1, 5, 9, ... 197, 100, _, _, _, _, 101, ... 120 - // TS1 interpolation = 1, 5, ... 197, 100, 100.2, 100.4, 100.6, 100.8, 101, - // ... 119.6, 119.8, 120 - // TS2 downsample = 300, 597, 593, ... 405, 401, ... 5, 1 - // TS1 + TS2 = 301, 602, 602, ... 501, 497.2, ... 124.8, 121 - int i = 0; - long ts = 1356998400000L; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 1000; - if (i == 0) { - assertEquals(301, dp.doubleValue(), 0.0000001); - } else if (i < 50) { - // TS1 = i * 2 + i * 2 + 1 - // TS2 = (300 - i * 2 + 1) + (300 - i * 2) - // TS1 + TS2 = 602 - assertEquals(602, dp.doubleValue(), 0.0000001); - } else { - // TS1 = 100 + (i - 50) * 0.2 - // TS2 = (300 - i * 2 + 1) + (300 - i * 2) - // TS1 + TS2 = 701 + (i - 50) * 0.2 - i * 4 - double value = 701 + (i - 50) * 0.2 - i * 4; - assertEquals(value, dp.doubleValue(), 0.0000001); - } - ++i; - } - assertEquals(151, dps[0].size()); - } - - //---------------------- // - // Aggregator unit tests // - // --------------------- // - - @Test - public void runZimSum() throws Exception { - storeLongTimeSeriesSeconds(false, false); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.ZIMSUM, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - long ts = 1356998430000L; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 30000; - assertEquals(301, dp.longValue()); - } - assertEquals(300, dps[0].size()); - } - - @Test - public void runZimSumFloat() throws Exception { - storeFloatTimeSeriesSeconds(false, false); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.ZIMSUM, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - long ts = 1356998430000L; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 30000; - assertEquals(76.25, dp.doubleValue(), 0.001); - } - assertEquals(300, dps[0].size()); - } - - @Test - public void runZimSumOffset() throws Exception { - storeLongTimeSeriesSeconds(false, true); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.ZIMSUM, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - long v1 = 1; - long v2 = 300; - long ts = 1356998430000L; - int counter = 0; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 15000; - - if (counter % 2 == 0) { - assertEquals(v1, dp.longValue()); - v1++; - } else { - assertEquals(v2, dp.longValue()); - v2--; - } - counter++; - } - assertEquals(600, dps[0].size()); - } - - @Test - public void runZimSumFloatOffset() throws Exception { - storeFloatTimeSeriesSeconds(false, true); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.ZIMSUM, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - double v1 = 1.25; - double v2 = 75.0; - long ts = 1356998430000L; - int counter = 0; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 15000; - if (counter % 2 == 0) { - assertEquals(v1, dp.doubleValue(), 0.001); - v1 += 0.25; - } else { - assertEquals(v2, dp.doubleValue(), 0.001); - v2 -= 0.25; - } - counter++; - } - assertEquals(600, dps[0].size()); - } - - @Test - public void runMin() throws Exception { - storeLongTimeSeriesSeconds(false, false); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.MIN, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - long v = 1; - long ts = 1356998430000L; - boolean decrement = false; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 30000; - assertEquals(v, dp.longValue()); - - if (decrement) { - v--; - } else { - v++; - } - - if (v == 151){ - v = 150; - decrement = true; - } - } - assertEquals(300, dps[0].size()); - } - - @Test - public void runMinFloat() throws Exception { - storeFloatTimeSeriesSeconds(false, false); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.MIN, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - double v = 1.25; - long ts = 1356998430000L; - boolean decrement = false; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 30000; - assertEquals(v, dp.doubleValue(), 0.0001); - - if (decrement) { - v -= .25; - } else { - v += .25; - } - - if (v > 38){ - v = 38.0; - decrement = true; - } - } - assertEquals(300, dps[0].size()); - } - - @Test - public void runMinOffset() throws Exception { - storeLongTimeSeriesSeconds(false, true); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.MIN, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - long v = 1; - long ts = 1356998430000L; - int counter = 0; - boolean decrement = false; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 15000; - assertEquals(v, dp.longValue()); - if (counter % 2 != 0) { - if (decrement) { - v--; - } else { - v++; - } - } else if (v == 151){ - v = 150; - decrement = true; - counter--; // hack since the hump is 150 150 151 150 150 - } - counter++; - } - assertEquals(600, dps[0].size()); - } - - @Test - public void runMinFloatOffset() throws Exception { - storeFloatTimeSeriesSeconds(false, true); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.MIN, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - double v = 1.25; - long ts = 1356998430000L; - boolean decrement = false; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 15000; - assertEquals(v, dp.doubleValue(), 0.001); - if (decrement) { - v -= 0.125; - } else { - v += 0.125; - } - - if (v > 38.125){ - v = 38.125; - decrement = true; - } - } - assertEquals(600, dps[0].size()); - } - - @Test - public void runMax() throws Exception { - storeLongTimeSeriesSeconds(false, false); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.MAX, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - long v = 300; - long ts = 1356998430000L; - boolean decrement = true; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 30000; - assertEquals(v, dp.longValue()); - - if (decrement) { - v--; - } else { - v++; - } - - if (v == 150){ - v = 151; - decrement = false; - } - } - assertEquals(300, dps[0].size()); - } - - @Test - public void runMaxFloat() throws Exception { - storeFloatTimeSeriesSeconds(false, false); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.MAX, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - double v = 75.0; - long ts = 1356998430000L; - boolean decrement = true; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 30000; - assertEquals(v, dp.doubleValue(), 0.001); - - if (decrement) { - v -= .25; - } else { - v += .25; - } - - if (v < 38.25){ - v = 38.25; - decrement = false; - } - } - assertEquals(300, dps[0].size()); - } - - @Test - public void runMaxOffset() throws Exception { - storeLongTimeSeriesSeconds(false, true); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.MAX, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - long v = 1; - long ts = 1356998430000L; - int counter = 0; - boolean decrement = true; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 15000; - assertEquals(v, dp.longValue()); - if (v == 1) { - v = 300; - } else if (dp.timestamp() == 1357007400000L) { - v = 1; - } else if (counter % 2 == 0) { - if (decrement) { - v--; - } else { - v++; - } - } - - if (v == 150){ - v = 151; - decrement = false; - counter--; // hack since the hump is 151 151 151 - } - counter++; - } - assertEquals(600, dps[0].size()); - } - - @Test - public void runMaxFloatOffset() throws Exception { - storeFloatTimeSeriesSeconds(false, true); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.MAX, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - double v = 1.25; - long ts = 1356998430000L; - boolean decrement = true; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 15000; - assertEquals(v, dp.doubleValue(), .0001); - if (v == 1.25) { - v = 75.0; - } else if (dp.timestamp() == 1357007400000L) { - v = 0.25; - } else { - if (decrement) { - v -= .125; - } else { - v += .125; - } - - if (v < 38.25){ - v = 38.25; - decrement = false; - } - } - } - assertEquals(600, dps[0].size()); - } - - @Test - public void runAvg() throws Exception { - storeLongTimeSeriesSeconds(false, false); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.AVG, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - long ts = 1356998430000L; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 30000; - assertEquals(150, dp.longValue()); - } - assertEquals(300, dps[0].size()); - } - - @Test - public void runAvgFloat() throws Exception { - storeFloatTimeSeriesSeconds(false, false); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.AVG, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - long ts = 1356998430000L; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 30000; - assertEquals(38.125, dp.doubleValue(), 0.001); - } - assertEquals(300, dps[0].size()); - } - - @Test - public void runAvgOffset() throws Exception { - storeLongTimeSeriesSeconds(false, true); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.AVG, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - long v = 1; - long ts = 1356998430000L; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 15000; - assertEquals(v, dp.longValue()); - if (v == 1) { - v = 150; - } else if (dp.timestamp() == 1357007400000L) { - v = 1; - } else if (v == 150) { - v = 151; - } else { - v = 150; - } - } - assertEquals(600, dps[0].size()); - } - - @Test - public void runAvgFloatOffset() throws Exception { - storeFloatTimeSeriesSeconds(false, true); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.AVG, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - double v = 1.25; - long ts = 1356998430000L; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 15000; - assertEquals(v, dp.doubleValue(), 0.0001); - if (v == 1.25) { - v = 38.1875; - } else if (dp.timestamp() == 1357007400000L) { - v = .25; - } - } - assertEquals(600, dps[0].size()); - } - - @Test - public void runDev() throws Exception { - storeLongTimeSeriesSeconds(false, false); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.DEV, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - long v = 149; - long ts = 1356998430000L; - boolean decrement = true; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 30000; - assertEquals(v, dp.longValue()); - - if (decrement) { - v--; - } else { - v++; - } - - if (v < 0){ - v = 0; - decrement = false; - } - } - assertEquals(300, dps[0].size()); - } - - @Test - public void runDevFloat() throws Exception { - storeFloatTimeSeriesSeconds(false, false); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.DEV, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - double v = 36.875; - long ts = 1356998430000L; - boolean decrement = true; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 30000; - assertEquals(v, dp.doubleValue(), 0.001); - - if (decrement) { - v -= 0.25; - } else { - v += 0.25; - } - - if (v < 0.125){ - v = 0.125; - decrement = false; - } - } - assertEquals(300, dps[0].size()); - } - - @Test - public void runDevOffset() throws Exception { - storeLongTimeSeriesSeconds(false, true); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.DEV, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - long v = 0; - long ts = 1356998430000L; - int counter = 0; - boolean decrement = true; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 15000; - assertEquals(v, dp.longValue()); - if (dp.timestamp() == 1356998430000L) { - v = 149; - } else if (dp.timestamp() == 1357007400000L) { - v = 0; - } else if (counter % 2 == 0) { - if (decrement) { - v--; - } else { - v++; - } - if (v < 0) { - v = 0; - decrement = false; - counter++; - } - } - counter++; - } - assertEquals(600, dps[0].size()); - } - - @Test - public void runDevFloatOffset() throws Exception { - storeFloatTimeSeriesSeconds(false, true); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.DEV, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - double v = 0; - long ts = 1356998430000L; - boolean decrement = true; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 15000; - assertEquals(v, dp.doubleValue(), 0.0001); - if (dp.timestamp() == 1356998430000L) { - v = 36.8125; - } else if (dp.timestamp() == 1357007400000L) { - v = 0; - } else { - if (decrement) { - v -= 0.125; - } else { - v += 0.125; - } - if (v < 0.0625) { - v = 0.0625; - decrement = false; - } - } - } - assertEquals(600, dps[0].size()); - } - - @Test - public void runMimMin() throws Exception { - storeLongTimeSeriesSeconds(false, false); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.MIMMIN, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - long v = 1; - long ts = 1356998430000L; - boolean decrement = false; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 30000; - assertEquals(v, dp.longValue()); - - if (decrement) { - v--; - } else { - v++; - } - - if (v == 151){ - v = 150; - decrement = true; - } - } - assertEquals(300, dps[0].size()); - } - - @Test - public void runMimMinOffset() throws Exception { - storeLongTimeSeriesSeconds(false, true); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.MIMMIN, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - long v1 = 1; - long v2 = 300; - long ts = 1356998430000L; - int counter = 0; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 15000; - - if (counter % 2 == 0) { - assertEquals(v1, dp.longValue()); - v1++; - } else { - assertEquals(v2, dp.longValue()); - v2--; - } - counter++; - } - assertEquals(600, dps[0].size()); - } - - @Test - public void runMimMinFloat() throws Exception { - storeFloatTimeSeriesSeconds(false, false); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.MIMMIN, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - double v = 1.25; - long ts = 1356998430000L; - boolean decrement = false; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 30000; - assertEquals(v, dp.doubleValue(), 0.0001); - - if (decrement) { - v -= .25; - } else { - v += .25; - } - - if (v > 38){ - v = 38.0; - decrement = true; - } - } - assertEquals(300, dps[0].size()); - } - - @Test - public void runMimMinFloatOffset() throws Exception { - storeFloatTimeSeriesSeconds(false, true); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.MIMMIN, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - double v1 = 1.25; - double v2 = 75.0; - long ts = 1356998430000L; - int counter = 0; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 15000; - if (counter % 2 == 0) { - assertEquals(v1, dp.doubleValue(), 0.001); - v1 += 0.25; - } else { - assertEquals(v2, dp.doubleValue(), 0.001); - v2 -= 0.25; - } - counter++; - } - assertEquals(600, dps[0].size()); - } - - @Test - public void runMimMax() throws Exception { - storeLongTimeSeriesSeconds(false, false); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.MIMMAX, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - long v = 300; - long ts = 1356998430000L; - boolean decrement = true; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 30000; - assertEquals(v, dp.longValue()); - - if (decrement) { - v--; - } else { - v++; - } - - if (v == 150){ - v = 151; - decrement = false; - } - } - assertEquals(300, dps[0].size()); - } - - @Test - public void runMimMaxFloat() throws Exception { - storeFloatTimeSeriesSeconds(false, false); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.MIMMAX, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - double v = 75.0; - long ts = 1356998430000L; - boolean decrement = true; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 30000; - assertEquals(v, dp.doubleValue(), 0.001); - - if (decrement) { - v -= .25; - } else { - v += .25; - } - - if (v < 38.25){ - v = 38.25; - decrement = false; - } - } - assertEquals(300, dps[0].size()); - } - - @Test - public void runMimMaxOffset() throws Exception { - storeLongTimeSeriesSeconds(false, true); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.MIMMAX, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - long v1 = 1; - long v2 = 300; - long ts = 1356998430000L; - int counter = 0; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 15000; - - if (counter % 2 == 0) { - assertEquals(v1, dp.longValue()); - v1++; - } else { - assertEquals(v2, dp.longValue()); - v2--; - } - counter++; - } - assertEquals(600, dps[0].size()); - } - - @Test - public void runMimMaxFloatOffset() throws Exception { - storeFloatTimeSeriesSeconds(false, true); - - HashMap<String, String> tags = new HashMap<String, String>(0); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.MIMMAX, false); - final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertEquals("host", dps[0].getAggregatedTags().get(0)); - assertNull(dps[0].getAnnotations()); - assertTrue(dps[0].getTags().isEmpty()); - - double v1 = 1.25; - double v2 = 75.0; - long ts = 1356998430000L; - int counter = 0; - for (DataPoint dp : dps[0]) { - assertEquals(ts, dp.timestamp()); - ts += 15000; - if (counter % 2 == 0) { - assertEquals(v1, dp.doubleValue(), 0.001); - v1 += 0.25; - } else { - assertEquals(v2, dp.doubleValue(), 0.001); - v2 -= 0.25; - } - counter++; - } - assertEquals(600, dps[0].size()); - } - - // ----------------- // - // Helper functions. // - // ----------------- // - - @SuppressWarnings("unchecked") - private void setQueryStorage() throws Exception { - storage = new MockBase(tsdb, client, true, true, true, true); - storage.setFamily("t".getBytes(MockBase.ASCII())); - - PowerMockito.mockStatic(IncomingDataPoints.class); - PowerMockito.doAnswer( - new Answer<byte[]>() { - @Override - public byte[] answer(final InvocationOnMock args) - throws Exception { - final String metric = (String)args.getArguments()[1]; - final Map<String, String> tags = - (Map<String, String>)args.getArguments()[2]; - - if (metric.equals("sys.cpu.user")) { - if (tags.get("host").equals("web01")) { - return new byte[] { 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1}; - } else { - return new byte[] { 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2}; - } - } else { - if (tags.get("host").equals("web01")) { - return new byte[] { 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1}; - } else { - return new byte[] { 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2}; - } - } - } - } - ).when(IncomingDataPoints.class, "rowKeyTemplate", any(), anyString(), - any()); - } - - private void storeLongTimeSeriesSeconds(final boolean two_metrics, - final boolean offset) throws Exception { - setQueryStorage(); - // dump a bunch of rows of two metrics so that we can test filtering out - // on the metric - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - long timestamp = 1356998400; - for (int i = 1; i <= 300; i++) { - tsdb.addPoint("sys.cpu.user", timestamp += 30, i, tags).joinUninterruptibly(); - if (two_metrics) { - tsdb.addPoint("sys.cpu.nice", timestamp, i, tags).joinUninterruptibly(); - } - } + query.split(tsQuery, 0, rawQuery); - // dump a parallel set but invert the values - tags.clear(); - tags.put("host", "web02"); - timestamp = offset ? 1356998415 : 1356998400; - for (int i = 300; i > 0; i--) { - tsdb.addPoint("sys.cpu.user", timestamp += 30, i, tags).joinUninterruptibly(); - if (two_metrics) { - tsdb.addPoint("sys.cpu.nice", timestamp, i, tags).joinUninterruptibly(); - } - } + verify(rawQuery).configureFromQuery(eq(tsQuery), eq(0), eq(true)); + + assertEquals(mockSystemTime - 7 * ONE_DAY_MS, query.getStartTime()); + assertEquals(mockSystemTime - 2 * ONE_DAY_MS, query.getEndTime()); + assertEquals(mockSystemTime - 2 * ONE_DAY_MS, rawQuery.getStartTime()); + assertEquals(mockSystemTime, rawQuery.getEndTime()); } - private void storeLongTimeSeriesMs() throws Exception { - setQueryStorage(); - // dump a bunch of rows of two metrics so that we can test filtering out - // on the metric - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - long timestamp = 1356998400000L; - for (int i = 1; i <= 300; i++) { - tsdb.addPoint("sys.cpu.user", timestamp += 500, i, tags).joinUninterruptibly(); - tsdb.addPoint("sys.cpu.nice", timestamp, i, tags).joinUninterruptibly(); - } + @Test(expected = IllegalStateException.class) + public void splitThrowsIfNotSplittable() { + Whitebox.setInternalState(tsdb, "rollups_split_queries", false); - // dump a parallel set but invert the values - tags.clear(); - tags.put("host", "web02"); - timestamp = 1356998400000L; - for (int i = 300; i > 0; i--) { - tsdb.addPoint("sys.cpu.user", timestamp += 500, i, tags).joinUninterruptibly(); - tsdb.addPoint("sys.cpu.nice", timestamp, i, tags).joinUninterruptibly(); - } + query.split(getTSQuery(), 0, new TsdbQuery(tsdb)); } - - private void storeFloatTimeSeriesSeconds(final boolean two_metrics, - final boolean offset) throws Exception { - setQueryStorage(); - // dump a bunch of rows of two metrics so that we can test filtering out - // on the metric - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - long timestamp = 1356998400; - for (float i = 1.25F; i <= 76; i += 0.25F) { - tsdb.addPoint("sys.cpu.user", timestamp += 30, i, tags).joinUninterruptibly(); - if (two_metrics) { - tsdb.addPoint("sys.cpu.nice", timestamp, i, tags).joinUninterruptibly(); - } - } - // dump a parallel set but invert the values - tags.clear(); - tags.put("host", "web02"); - timestamp = offset ? 1356998415 : 1356998400; - for (float i = 75F; i > 0; i -= 0.25F) { - tsdb.addPoint("sys.cpu.user", timestamp += 30, i, tags).joinUninterruptibly(); - if (two_metrics) { - tsdb.addPoint("sys.cpu.nice", timestamp, i, tags).joinUninterruptibly(); - } - } + /** @return a simple TSQuery object for testing */ + private TSQuery getTSQuery() { + return getTSQuery(null); } - - private void storeFloatTimeSeriesMs() throws Exception { - setQueryStorage(); - // dump a bunch of rows of two metrics so that we can test filtering out - // on the metric - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - long timestamp = 1356998400000L; - for (float i = 1.25F; i <= 76; i += 0.25F) { - tsdb.addPoint("sys.cpu.user", timestamp += 500, i, tags).joinUninterruptibly(); - tsdb.addPoint("sys.cpu.nice", timestamp, i, tags).joinUninterruptibly(); - } - // dump a parallel set but invert the values - tags.clear(); - tags.put("host", "web02"); - timestamp = 1356998400000L; - for (float i = 75F; i > 0; i -= 0.25F) { - tsdb.addPoint("sys.cpu.user", timestamp += 500, i, tags).joinUninterruptibly(); - tsdb.addPoint("sys.cpu.nice", timestamp, i, tags).joinUninterruptibly(); - } - } - - private void storeMixedTimeSeriesSeconds() throws Exception { - setQueryStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - long timestamp = 1356998400; - for (float i = 1.25F; i <= 76; i += 0.25F) { - if (i % 2 == 0) { - tsdb.addPoint("sys.cpu.user", timestamp += 30, (long)i, tags) - .joinUninterruptibly(); - } else { - tsdb.addPoint("sys.cpu.user", timestamp += 30, i, tags) - .joinUninterruptibly(); - } - } - } - - // dumps ints, floats, seconds and ms - private void storeMixedTimeSeriesMsAndS() throws Exception { - setQueryStorage(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - long timestamp = 1356998400000L; - for (float i = 1.25F; i <= 76; i += 0.25F) { - long ts = timestamp += 500; - if (ts % 1000 == 0) { - ts /= 1000; - } - if (i % 2 == 0) { - tsdb.addPoint("sys.cpu.user", ts, (long)i, tags).joinUninterruptibly(); - } else { - tsdb.addPoint("sys.cpu.user", ts, i, tags).joinUninterruptibly(); - } - } + private TSQuery getTSQuery(TsdbQuery.ROLLUP_USAGE rollupUsage) { + final TSQuery ts_query = new TSQuery(); + ts_query.setStart("1356998400"); + + final ArrayList<TSSubQuery> sub_queries = new ArrayList<TSSubQuery>(1); + sub_queries.add(getSubQuery(rollupUsage)); + + ts_query.setQueries(sub_queries); + return ts_query; } - - private void storeLongCompactions() throws Exception { - setQueryStorage(); - long base_timestamp = 1356998400; - long value = 1; - byte[] qualifier = new byte[119 * 2]; - long timestamp = 1356998430; - for (int index = 0; index < qualifier.length; index += 2) { - final int offset = (int) (timestamp - base_timestamp); - final byte[] column = - Bytes.fromShort((short)(offset << Const.FLAG_BITS | 0x7)); - System.arraycopy(column, 0, qualifier, index, 2); - timestamp += 30; - } - - byte[] column_qualifier = new byte[119 * 8]; - for (int index = 0; index < column_qualifier.length; index += 8) { - System.arraycopy(Bytes.fromLong(value), 0, column_qualifier, index, 8); - value++; - } - storage.addColumn(MockBase.stringToBytes("00000150E22700000001000001"), - qualifier, column_qualifier); - - base_timestamp = 1357002000; - qualifier = new byte[120 * 2]; - timestamp = 1357002000; - for (int index = 0; index < qualifier.length; index += 2) { - final int offset = (int) (timestamp - base_timestamp); - final byte[] column = - Bytes.fromShort((short)(offset << Const.FLAG_BITS | 0x7)); - System.arraycopy(column, 0, qualifier, index, 2); - timestamp += 30; - } - - column_qualifier = new byte[120 * 8]; - for (int index = 0; index < column_qualifier.length; index += 8) { - System.arraycopy(Bytes.fromLong(value), 0, column_qualifier, index, 8); - value++; - } - storage.addColumn(MockBase.stringToBytes("00000150E23510000001000001"), - qualifier, column_qualifier); - - base_timestamp = 1357005600; - qualifier = new byte[61 * 2]; - timestamp = 1357005600; - for (int index = 0; index < qualifier.length; index += 2) { - final int offset = (int) (timestamp - base_timestamp); - final byte[] column = - Bytes.fromShort((short)(offset << Const.FLAG_BITS | 0x7)); - System.arraycopy(column, 0, qualifier, index, 2); - timestamp += 30; - } - - column_qualifier = new byte[61 * 8]; - for (int index = 0; index < column_qualifier.length; index += 8) { - System.arraycopy(Bytes.fromLong(value), 0, column_qualifier, index, 8); - value++; + + private TSSubQuery getSubQuery(TsdbQuery.ROLLUP_USAGE rollupUsage) { + final TSSubQuery sub_query = new TSSubQuery(); + sub_query.setMetric(METRIC_STRING); + sub_query.setAggregator("sum"); + + sub_query.setTags(tags); + + if (rollupUsage != null) { + sub_query.setRollupUsage(rollupUsage.name()); } - storage.addColumn(MockBase.stringToBytes("00000150E24320000001000001"), - qualifier, column_qualifier); + + return sub_query; } - - private void storeFloatCompactions() throws Exception { - setQueryStorage(); - long base_timestamp = 1356998400; - float value = 1.25F; - byte[] qualifier = new byte[119 * 2]; - long timestamp = 1356998430; - for (int index = 0; index < qualifier.length; index += 2) { - final int offset = (int) (timestamp - base_timestamp); - final byte[] column = - Bytes.fromShort((short)(offset << Const.FLAG_BITS | Const.FLAG_FLOAT | 0x3)); - System.arraycopy(column, 0, qualifier, index, 2); - timestamp += 30; - } - - byte[] column_qualifier = new byte[119 * 4]; - for (int index = 0; index < column_qualifier.length; index += 4) { - System.arraycopy(Bytes.fromInt(Float.floatToRawIntBits(value)), 0, - column_qualifier, index, 4); - value += 0.25F; - } - storage.addColumn(MockBase.stringToBytes("00000150E22700000001000001"), - qualifier, column_qualifier); - - base_timestamp = 1357002000; - qualifier = new byte[120 * 2]; - timestamp = 1357002000; - for (int index = 0; index < qualifier.length; index += 2) { - final int offset = (int) (timestamp - base_timestamp); - final byte[] column = - Bytes.fromShort((short)(offset << Const.FLAG_BITS | Const.FLAG_FLOAT | 0x3)); - System.arraycopy(column, 0, qualifier, index, 2); - timestamp += 30; - } - - column_qualifier = new byte[120 * 4]; - for (int index = 0; index < column_qualifier.length; index += 4) { - System.arraycopy(Bytes.fromInt(Float.floatToRawIntBits(value)), 0, - column_qualifier, index, 4); - value += 0.25F; - } - storage.addColumn(MockBase.stringToBytes("00000150E23510000001000001"), - qualifier, column_qualifier); - - base_timestamp = 1357005600; - qualifier = new byte[61 * 2]; - timestamp = 1357005600; - for (int index = 0; index < qualifier.length; index += 2) { - final int offset = (int) (timestamp - base_timestamp); - final byte[] column = - Bytes.fromShort((short)(offset << Const.FLAG_BITS | Const.FLAG_FLOAT | 0x3)); - System.arraycopy(column, 0, qualifier, index, 2); - timestamp += 30; - } - - column_qualifier = new byte[61 * 4]; - for (int index = 0; index < column_qualifier.length; index += 4) { - System.arraycopy(Bytes.fromInt(Float.floatToRawIntBits(value)), 0, - column_qualifier, index, 4); - value += 0.25F; - } - storage.addColumn(MockBase.stringToBytes("00000150E24320000001000001"), - qualifier, column_qualifier); + + private void mockSystemTime(long newTimestamp) { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(newTimestamp); + PowerMockito.when(DateTime.getDurationUnits(anyString())).thenCallRealMethod(); + PowerMockito.when(DateTime.getDurationInterval(anyString())).thenCallRealMethod(); + PowerMockito.when(DateTime.parseDuration(anyString())).thenCallRealMethod(); } - - private void storeMixedCompactions() throws Exception { - setQueryStorage(); - long base_timestamp = 1356998400; - float q_counter = 1.25F; - byte[] qualifier = new byte[119 * 2]; - long timestamp = 1356998430; - for (int index = 0; index < qualifier.length; index += 2) { - final int offset = (int) (timestamp - base_timestamp); - final byte[] column; - if (q_counter % 1 == 0) { - column = Bytes.fromShort((short)(offset << Const.FLAG_BITS | 0x7)); - } else { - column = Bytes.fromShort( - (short)(offset << Const.FLAG_BITS | Const.FLAG_FLOAT | 0x3)); - } - System.arraycopy(column, 0, qualifier, index, 2); - timestamp += 30; - q_counter += 0.25F; - } - - float value = 1.25F; - int num = 119; - byte[] column_qualifier = new byte[((num / 4) * 8) + ((num - (num / 4)) * 4)]; - int idx = 0; - while (idx < column_qualifier.length) { - if (value % 1 == 0) { - System.arraycopy(Bytes.fromLong((long)value), 0, column_qualifier, idx, 8); - idx += 8; - } else { - System.arraycopy(Bytes.fromInt(Float.floatToRawIntBits(value)), 0, - column_qualifier, idx, 4); - idx += 4; - } - value += 0.25F; - } - storage.addColumn(MockBase.stringToBytes("00000150E22700000001000001"), - qualifier, column_qualifier); - - base_timestamp = 1357002000; - qualifier = new byte[120 * 2]; - timestamp = 1357002000; - for (int index = 0; index < qualifier.length; index += 2) { - final int offset = (int) (timestamp - base_timestamp); - final byte[] column; - if (q_counter % 1 == 0) { - column = Bytes.fromShort((short)(offset << Const.FLAG_BITS | 0x7)); - } else { - column = Bytes.fromShort( - (short)(offset << Const.FLAG_BITS | Const.FLAG_FLOAT | 0x3)); - } - System.arraycopy(column, 0, qualifier, index, 2); - timestamp += 30; - q_counter += 0.25F; - } - - num = 120; - column_qualifier = new byte[((num / 4) * 8) + ((num - (num / 4)) * 4)]; - idx = 0; - while (idx < column_qualifier.length) { - if (value % 1 == 0) { - System.arraycopy(Bytes.fromLong((long)value), 0, column_qualifier, idx, 8); - idx += 8; - } else { - System.arraycopy(Bytes.fromInt(Float.floatToRawIntBits(value)), 0, - column_qualifier, idx, 4); - idx += 4; - } - value += 0.25F; - } - storage.addColumn(MockBase.stringToBytes("00000150E23510000001000001"), - qualifier, column_qualifier); - - base_timestamp = 1357005600; - qualifier = new byte[61 * 2]; - timestamp = 1357005600; - for (int index = 0; index < qualifier.length; index += 2) { - final int offset = (int) (timestamp - base_timestamp); - final byte[] column; - if (q_counter % 1 == 0) { - column = Bytes.fromShort((short)(offset << Const.FLAG_BITS | 0x7)); - } else { - column = Bytes.fromShort( - (short)(offset << Const.FLAG_BITS | Const.FLAG_FLOAT | 0x3)); - } - System.arraycopy(column, 0, qualifier, index, 2); - timestamp += 30; - q_counter += 0.25F; - } - - num = 61; - column_qualifier = - new byte[(((num / 4) + 1) * 8) + ((num - ((num / 4) + 1)) * 4)]; - idx = 0; - while (idx < column_qualifier.length) { - if (value % 1 == 0) { - System.arraycopy(Bytes.fromLong((long)value), 0, column_qualifier, idx, 8); - idx += 8; - } else { - System.arraycopy(Bytes.fromInt(Float.floatToRawIntBits(value)), 0, - column_qualifier, idx, 4); - idx += 4; - } - value += 0.25F; - } - storage.addColumn(MockBase.stringToBytes("00000150E24320000001000001"), - qualifier, column_qualifier); + + private void mockEnableRollupQuerySplitting() { + Whitebox.setInternalState(tsdb, "rollups_split_queries", true); + Whitebox.setInternalState(query, "rollup_query", makeRollupQuery()); } } diff --git a/test/core/TestTsdbQueryAggregators.java b/test/core/TestTsdbQueryAggregators.java new file mode 100644 index 0000000000..eed999fc66 --- /dev/null +++ b/test/core/TestTsdbQueryAggregators.java @@ -0,0 +1,1117 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.HashMap; + +import org.hbase.async.Scanner; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +/** + * Integration testing for the various aggregators. We write data points to + * MockBase and then pull them out, following the full path for a TSDB query. + */ +@RunWith(PowerMockRunner.class) +@PrepareForTest({ Scanner.class }) +public class TestTsdbQueryAggregators extends BaseTsdbTest { + protected TsdbQuery query = null; + + @Before + public void beforeLocal() throws Exception { + query = new TsdbQuery(tsdb); + } + + @Test + public void runZimSum() throws Exception { + storeLongTimeSeriesSeconds(false, false); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.ZIMSUM, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long ts = 1356998430000L; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 30000; + assertEquals(301, dp.longValue()); + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runZimSumFloat() throws Exception { + storeFloatTimeSeriesSeconds(false, false); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.ZIMSUM, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long ts = 1356998430000L; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 30000; + assertEquals(76.25, dp.doubleValue(), 0.001); + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runZimSumOffset() throws Exception { + storeLongTimeSeriesSeconds(false, true); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.ZIMSUM, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long v1 = 1; + long v2 = 300; + long ts = 1356998430000L; + int counter = 0; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 15000; + if (counter % 2 == 0) { + assertEquals(v1, dp.longValue()); + v1++; + } else { + assertEquals(v2, dp.longValue()); + v2--; + } + counter++; + } + assertEquals(600, dps[0].size()); + } + + @Test + public void runZimSumFloatOffset() throws Exception { + storeFloatTimeSeriesSeconds(false, true); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.ZIMSUM, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + double v1 = 1.25; + double v2 = 75.0; + long ts = 1356998430000L; + int counter = 0; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 15000; + if (counter % 2 == 0) { + assertEquals(v1, dp.doubleValue(), 0.001); + v1 += 0.25; + } else { + assertEquals(v2, dp.doubleValue(), 0.001); + v2 -= 0.25; + } + counter++; + } + assertEquals(600, dps[0].size()); + } + + @Test + public void runZimSumWithMissingData() throws Exception { + storeLongTimeSeriesWithMissingData(); + + HashMap<String, String> tags = new HashMap<String, String>(0); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.ZIMSUM, false); + final DataPoints[] dps = query.run(); + assertNotNull(dps); + assertEquals(METRIC_STRING, dps[0].metricName()); + assertEquals(TAGK_STRING, dps[0].getAggregatedTags().get(0)); + assertNull(dps[0].getAnnotations()); + assertTrue(dps[0].getTags().isEmpty()); + + /* INPUT: + * t0 t1 t2 t3 t4 t5 ... + * web01 X 2 3 X 5 6 ... + * web02 X 299 X 297 X 295 ... + * + * OUTPUT: + * zimsum X 301 3 297 5 301 ... + */ + + int i = 0; + long ts = 1356998400000L; + for (final DataPoint dp : dps[0]) { + // Every sixth position, both elements are missing, so the aggregation + // will have a gap. + int offset = i % 6; + if (0 == offset) { + // We have skipped a timestamp, so we should update the state. + ts += 10000; + ++i; + ++offset; + } + + if (1 == offset || 5 == offset) { + // The second and last elements in each cycle should be the expected + // value, which is 301. + assertEquals(301, dp.longValue()); + } else if (2 == offset || 4 == offset) { + // The third and fifth elements in each cycle should be taken from the + // ascending series. + assertEquals(i + 1, dp.longValue()); + } else { + // Otherwise, the element should be taken from the descending series. + assertEquals(300 - i, dp.longValue()); + } + + assertEquals(ts, dp.timestamp()); + ts += 10000; + ++i; + } + + assertEquals(250, dps[0].size()); + } + + @Test + public void runMin() throws Exception { + storeLongTimeSeriesSeconds(false, false); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.MIN, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long v = 1; + long ts = 1356998430000L; + boolean decrement = false; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 30000; + assertEquals(v, dp.longValue()); + + if (decrement) { + v--; + } else { + v++; + } + + if (v == 151){ + v = 150; + decrement = true; + } + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runMinFloat() throws Exception { + storeFloatTimeSeriesSeconds(false, false); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.MIN, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + double v = 1.25; + long ts = 1356998430000L; + boolean decrement = false; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 30000; + assertEquals(v, dp.doubleValue(), 0.0001); + + if (decrement) { + v -= .25; + } else { + v += .25; + } + + if (v > 38){ + v = 38.0; + decrement = true; + } + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runMinOffset() throws Exception { + storeLongTimeSeriesSeconds(false, true); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.MIN, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long v = 1; + long ts = 1356998430000L; + int counter = 0; + boolean decrement = false; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 15000; + assertEquals(v, dp.longValue()); + if (counter % 2 != 0) { + if (decrement) { + v--; + } else { + v++; + } + } else if (v == 151){ + v = 150; + decrement = true; + counter--; // hack since the hump is 150 150 151 150 150 + } + counter++; + } + assertEquals(600, dps[0].size()); + } + + @Test + public void runMinFloatOffset() throws Exception { + storeFloatTimeSeriesSeconds(false, true); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.MIN, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + double v = 1.25; + long ts = 1356998430000L; + boolean decrement = false; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 15000; + assertEquals(v, dp.doubleValue(), 0.001); + if (decrement) { + v -= 0.125; + } else { + v += 0.125; + } + + if (v > 38.125){ + v = 38.125; + decrement = true; + } + } + assertEquals(600, dps[0].size()); + } + + @Test + public void runMax() throws Exception { + storeLongTimeSeriesSeconds(false, false); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.MAX, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long v = 300; + long ts = 1356998430000L; + boolean decrement = true; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 30000; + assertEquals(v, dp.longValue()); + + if (decrement) { + v--; + } else { + v++; + } + + if (v == 150){ + v = 151; + decrement = false; + } + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runMaxFloat() throws Exception { + storeFloatTimeSeriesSeconds(false, false); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.MAX, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + double v = 75.0; + long ts = 1356998430000L; + boolean decrement = true; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 30000; + assertEquals(v, dp.doubleValue(), 0.001); + + if (decrement) { + v -= .25; + } else { + v += .25; + } + + if (v < 38.25){ + v = 38.25; + decrement = false; + } + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runMaxOffset() throws Exception { + storeLongTimeSeriesSeconds(false, true); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.MAX, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long v = 1; + long ts = 1356998430000L; + int counter = 0; + boolean decrement = true; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 15000; + assertEquals(v, dp.longValue()); + if (v == 1) { + v = 300; + } else if (dp.timestamp() == 1357007400000L) { + v = 1; + } else if (counter % 2 == 0) { + if (decrement) { + v--; + } else { + v++; + } + } + + if (v == 150){ + v = 151; + decrement = false; + counter--; // hack since the hump is 151 151 151 + } + counter++; + } + assertEquals(600, dps[0].size()); + } + + @Test + public void runMaxFloatOffset() throws Exception { + storeFloatTimeSeriesSeconds(false, true); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.MAX, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + double v = 1.25; + long ts = 1356998430000L; + boolean decrement = true; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 15000; + assertEquals(v, dp.doubleValue(), .0001); + if (v == 1.25) { + v = 75.0; + } else if (dp.timestamp() == 1357007400000L) { + v = 0.25; + } else { + if (decrement) { + v -= .125; + } else { + v += .125; + } + + if (v < 38.25){ + v = 38.25; + decrement = false; + } + } + } + assertEquals(600, dps[0].size()); + } + + @Test + public void runAvg() throws Exception { + storeLongTimeSeriesSeconds(false, false); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.AVG, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long ts = 1356998430000L; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 30000; + assertEquals(150, dp.longValue()); + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runAvgFloat() throws Exception { + storeFloatTimeSeriesSeconds(false, false); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.AVG, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long ts = 1356998430000L; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 30000; + assertEquals(38.125, dp.doubleValue(), 0.001); + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runAvgOffset() throws Exception { + storeLongTimeSeriesSeconds(false, true); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.AVG, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long v = 1; + long ts = 1356998430000L; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 15000; + assertEquals(v, dp.longValue()); + if (v == 1) { + v = 150; + } else if (dp.timestamp() == 1357007400000L) { + v = 1; + } else if (v == 150) { + v = 151; + } else { + v = 150; + } + } + assertEquals(600, dps[0].size()); + } + + @Test + public void runAvgFloatOffset() throws Exception { + storeFloatTimeSeriesSeconds(false, true); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.AVG, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + double v = 1.25; + long ts = 1356998430000L; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 15000; + assertEquals(v, dp.doubleValue(), 0.0001); + if (v == 1.25) { + v = 38.1875; + } else if (dp.timestamp() == 1357007400000L) { + v = .25; + } + } + assertEquals(600, dps[0].size()); + } + + @Test + public void runDev() throws Exception { + storeLongTimeSeriesSeconds(false, false); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.DEV, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long v = 149; + long ts = 1356998430000L; + boolean decrement = true; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 30000; + assertEquals(v, dp.longValue()); + + if (decrement) { + v--; + } else { + v++; + } + + if (v < 0){ + v = 0; + decrement = false; + } + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runDevFloat() throws Exception { + storeFloatTimeSeriesSeconds(false, false); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.DEV, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + double v = 36.875; + long ts = 1356998430000L; + boolean decrement = true; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 30000; + assertEquals(v, dp.doubleValue(), 0.001); + + if (decrement) { + v -= 0.25; + } else { + v += 0.25; + } + + if (v < 0.125){ + v = 0.125; + decrement = false; + } + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runDevOffset() throws Exception { + storeLongTimeSeriesSeconds(false, true); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.DEV, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long v = 0; + long ts = 1356998430000L; + int counter = 0; + boolean decrement = true; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 15000; + assertEquals(v, dp.longValue()); + if (dp.timestamp() == 1356998430000L) { + v = 149; + } else if (dp.timestamp() == 1357007400000L) { + v = 0; + } else if (counter % 2 == 0) { + if (decrement) { + v--; + } else { + v++; + } + if (v < 0) { + v = 0; + decrement = false; + counter++; + } + } + counter++; + } + assertEquals(600, dps[0].size()); + } + + @Test + public void runDevFloatOffset() throws Exception { + storeFloatTimeSeriesSeconds(false, true); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.DEV, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + double v = 0; + long ts = 1356998430000L; + boolean decrement = true; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 15000; + assertEquals(v, dp.doubleValue(), 0.0001); + if (dp.timestamp() == 1356998430000L) { + v = 36.8125; + } else if (dp.timestamp() == 1357007400000L) { + v = 0; + } else { + if (decrement) { + v -= 0.125; + } else { + v += 0.125; + } + if (v < 0.0625) { + v = 0.0625; + decrement = false; + } + } + } + assertEquals(600, dps[0].size()); + } + + @Test + public void runMimMin() throws Exception { + storeLongTimeSeriesSeconds(false, false); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.MIMMIN, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long v = 1; + long ts = 1356998430000L; + boolean decrement = false; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 30000; + assertEquals(v, dp.longValue()); + + if (decrement) { + v--; + } else { + v++; + } + + if (v == 151){ + v = 150; + decrement = true; + } + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runMimMinOffset() throws Exception { + storeLongTimeSeriesSeconds(false, true); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.MIMMIN, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long v1 = 1; + long v2 = 300; + long ts = 1356998430000L; + int counter = 0; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 15000; + + if (counter % 2 == 0) { + assertEquals(v1, dp.longValue()); + v1++; + } else { + assertEquals(v2, dp.longValue()); + v2--; + } + counter++; + } + assertEquals(600, dps[0].size()); + } + + @Test + public void runMimMinFloat() throws Exception { + storeFloatTimeSeriesSeconds(false, false); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.MIMMIN, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + double v = 1.25; + long ts = 1356998430000L; + boolean decrement = false; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 30000; + assertEquals(v, dp.doubleValue(), 0.0001); + + if (decrement) { + v -= .25; + } else { + v += .25; + } + + if (v > 38){ + v = 38.0; + decrement = true; + } + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runMimMinFloatOffset() throws Exception { + storeFloatTimeSeriesSeconds(false, true); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.MIMMIN, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + double v1 = 1.25; + double v2 = 75.0; + long ts = 1356998430000L; + int counter = 0; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 15000; + if (counter % 2 == 0) { + assertEquals(v1, dp.doubleValue(), 0.001); + v1 += 0.25; + } else { + assertEquals(v2, dp.doubleValue(), 0.001); + v2 -= 0.25; + } + counter++; + } + assertEquals(600, dps[0].size()); + } + + @Test + public void runMimMax() throws Exception { + storeLongTimeSeriesSeconds(false, false); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.MIMMAX, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long v = 300; + long ts = 1356998430000L; + boolean decrement = true; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 30000; + assertEquals(v, dp.longValue()); + + if (decrement) { + v--; + } else { + v++; + } + + if (v == 150){ + v = 151; + decrement = false; + } + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runMimMaxFloat() throws Exception { + storeFloatTimeSeriesSeconds(false, false); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.MIMMAX, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + double v = 75.0; + long ts = 1356998430000L; + boolean decrement = true; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 30000; + assertEquals(v, dp.doubleValue(), 0.001); + + if (decrement) { + v -= .25; + } else { + v += .25; + } + + if (v < 38.25){ + v = 38.25; + decrement = false; + } + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runMimMaxOffset() throws Exception { + storeLongTimeSeriesSeconds(false, true); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.MIMMAX, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long v1 = 1; + long v2 = 300; + long ts = 1356998430000L; + int counter = 0; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 15000; + + if (counter % 2 == 0) { + assertEquals(v1, dp.longValue()); + v1++; + } else { + assertEquals(v2, dp.longValue()); + v2--; + } + counter++; + } + assertEquals(600, dps[0].size()); + } + + @Test + public void runMimMaxFloatOffset() throws Exception { + storeFloatTimeSeriesSeconds(false, true); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.MIMMAX, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + double v1 = 1.25; + double v2 = 75.0; + long ts = 1356998430000L; + int counter = 0; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 15000; + if (counter % 2 == 0) { + assertEquals(v1, dp.doubleValue(), 0.001); + v1 += 0.25; + } else { + assertEquals(v2, dp.doubleValue(), 0.001); + v2 -= 0.25; + } + counter++; + } + assertEquals(600, dps[0].size()); + } + + @Test + public void runPercentiles() throws Exception { + storeLongTimeSeriesSeconds(false, true); + + // These are not accurate at all when data points only contain 2 values + // so we are just testing constructor logic, rather than precision + testPercentile(Aggregators.p50, 150, 150); + testPercentile(Aggregators.p75, 150, 150); + testPercentile(Aggregators.p90, 150, 150); + testPercentile(Aggregators.p95, 150, 150); + testPercentile(Aggregators.p99, 150, 150); + testPercentile(Aggregators.p999, 150, 150); + testPercentile(Aggregators.ep50r3, 150, 150); + testPercentile(Aggregators.ep75r3, 150, 150); + testPercentile(Aggregators.ep90r3, 150, 150); + testPercentile(Aggregators.ep95r3, 150, 150); + testPercentile(Aggregators.ep99r3, 150, 150); + testPercentile(Aggregators.ep999r3, 150, 150); + testPercentile(Aggregators.ep50r7, 150, 150); + testPercentile(Aggregators.ep75r7, 150, 150); + testPercentile(Aggregators.ep90r7, 150, 150); + testPercentile(Aggregators.ep95r7, 150, 150); + testPercentile(Aggregators.ep99r7, 150, 150); + testPercentile(Aggregators.ep999r7, 150, 150); + } + + public void runCount() throws Exception { + storeLongTimeSeriesSeconds(false, false); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.COUNT, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long ts = 1356998430000L; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 30000; + assertEquals(2, dp.longValue()); + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runCountFloat() throws Exception { + storeFloatTimeSeriesSeconds(false, false); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.COUNT, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long ts = 1356998430000L; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 30000; + assertEquals(2, dp.doubleValue(), 0.001); + } + assertEquals(300, dps[0].size()); + } + + // TODO - The count agg is inaccurate until we implement NaNs. + @Test + public void runCountOffset() throws Exception { + storeLongTimeSeriesSeconds(false, true); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.COUNT, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long ts = 1356998430000L; + int counter = 0; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 15000; + if (counter == 0 || counter == 599) { + assertEquals(1, dp.longValue()); + } else { + assertEquals(2, dp.longValue()); + } + counter++; + } + assertEquals(600, dps[0].size()); + } + + // TODO - The count agg is inaccurate until we implement NaNs. + @Test + public void runCountFloatOffset() throws Exception { + storeFloatTimeSeriesSeconds(false, true); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.COUNT, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long ts = 1356998430000L; + int counter = 0; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 15000; + if (counter == 0 || counter == 599) { + assertEquals(1, dp.doubleValue(), 0.0001); + } else { + assertEquals(2, dp.doubleValue(), 0.0001); + } + counter++; + } + assertEquals(600, dps[0].size()); + } + + /** + * Helper to test the various percentiles + * @param agg The aggregator + * @param value The value to expect + * @param delta The variance to expect + */ + private void testPercentile(final Aggregator agg, final long value, + final double delta) { + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, agg, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long ts = 1356998430000L; + int counter = 0; + int size = dps[0].size(); + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 15000; + assertEquals("counter " + counter, value, dp.longValue(), delta); + counter++; + } + assertEquals(600, size); + } +} diff --git a/test/core/TestTsdbQueryAggregatorsSalted.java b/test/core/TestTsdbQueryAggregatorsSalted.java new file mode 100644 index 0000000000..5ad0cf8d33 --- /dev/null +++ b/test/core/TestTsdbQueryAggregatorsSalted.java @@ -0,0 +1,38 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import org.junit.Before; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.modules.junit4.PowerMockRunner; + +/** + * Integration test that runs all of the tests in {@see TestTsdbQueryAggregators} + * but with salting enabled just to verify nothing goes wrong with the extra + * salt bytes. + */ +@RunWith(PowerMockRunner.class) +public class TestTsdbQueryAggregatorsSalted extends TestTsdbQueryAggregators { + + @Before + public void beforeLocal() throws Exception { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(2); + PowerMockito.when(Const.MAX_NUM_TAGS()).thenReturn((short) 8); + + query = new TsdbQuery(tsdb); + } + +} diff --git a/test/core/TestTsdbQueryAppend.java b/test/core/TestTsdbQueryAppend.java new file mode 100644 index 0000000000..c7b4928b09 --- /dev/null +++ b/test/core/TestTsdbQueryAppend.java @@ -0,0 +1,25 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2010-2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import org.junit.Before; +import org.powermock.reflect.Whitebox; + +public class TestTsdbQueryAppend extends TestTsdbQueryQueries { + + @Before + public void beforeLocal() { + Whitebox.setInternalState(config, "enable_appends", true); + query = new TsdbQuery(tsdb); + } +} diff --git a/test/core/TestTsdbQueryDownsample.java b/test/core/TestTsdbQueryDownsample.java index cc0007fcd8..02bdce9469 100644 --- a/test/core/TestTsdbQueryDownsample.java +++ b/test/core/TestTsdbQueryDownsample.java @@ -17,153 +17,114 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyString; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import static org.powermock.api.mockito.PowerMockito.mock; -import java.lang.reflect.Field; import java.util.HashMap; -import java.util.Map; -import com.stumbleupon.async.Deferred; - -import net.opentsdb.meta.Annotation; -import net.opentsdb.storage.MockBase; -import net.opentsdb.uid.NoSuchUniqueName; -import net.opentsdb.uid.UniqueId; -import net.opentsdb.utils.Config; import net.opentsdb.utils.DateTime; -import org.apache.zookeeper.proto.DeleteRequest; -import org.hbase.async.GetRequest; -import org.hbase.async.HBaseClient; -import org.hbase.async.KeyValue; -import org.hbase.async.PutRequest; import org.hbase.async.Scanner; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +import com.google.common.collect.Lists; +import com.google.common.math.DoubleMath; /** * Tests downsampling with query. */ @RunWith(PowerMockRunner.class) -@PowerMockIgnore({"javax.management.*", "javax.xml.*", - "ch.qos.*", "org.slf4j.*", - "com.sum.*", "org.xml.*"}) -@PrepareForTest({TSDB.class, Config.class, UniqueId.class, HBaseClient.class, - CompactionQueue.class, GetRequest.class, PutRequest.class, KeyValue.class, - Scanner.class, TsdbQuery.class, DeleteRequest.class, Annotation.class, - RowKey.class, Span.class, SpanGroup.class, IncomingDataPoints.class }) -public class TestTsdbQueryDownsample { - - private Config config; - private TSDB tsdb = null; - private HBaseClient client = mock(HBaseClient.class); - private UniqueId metrics = mock(UniqueId.class); - private UniqueId tag_names = mock(UniqueId.class); - private UniqueId tag_values = mock(UniqueId.class); - private TsdbQuery query = null; - private MockBase storage = null; +@PrepareForTest({ Scanner.class }) +public class TestTsdbQueryDownsample extends BaseTsdbTest { + protected TsdbQuery query = null; @Before - public void before() throws Exception { - config = new Config(false); - tsdb = new TSDB(config); + public void beforeLocal() throws Exception { query = new TsdbQuery(tsdb); + } - // replace the "real" field objects with mocks - Field cl = tsdb.getClass().getDeclaredField("client"); - cl.setAccessible(true); - cl.set(tsdb, client); - - Field met = tsdb.getClass().getDeclaredField("metrics"); - met.setAccessible(true); - met.set(tsdb, metrics); - - Field tagk = tsdb.getClass().getDeclaredField("tag_names"); - tagk.setAccessible(true); - tagk.set(tsdb, tag_names); - - Field tagv = tsdb.getClass().getDeclaredField("tag_values"); - tagv.setAccessible(true); - tagv.set(tsdb, tag_values); - - // mock UniqueId - when(metrics.getId("sys.cpu.user")).thenReturn(new byte[] { 0, 0, 1 }); - when(metrics.getNameAsync(new byte[] { 0, 0, 1 })) - .thenReturn(Deferred.fromResult("sys.cpu.user")); - when(metrics.getId("sys.cpu.system")) - .thenThrow(new NoSuchUniqueName("sys.cpu.system", "metric")); - when(metrics.getId("sys.cpu.nice")).thenReturn(new byte[] { 0, 0, 2 }); - when(metrics.getNameAsync(new byte[] { 0, 0, 2 })) - .thenReturn(Deferred.fromResult("sys.cpu.nice")); - when(tag_names.getId("host")).thenReturn(new byte[] { 0, 0, 1 }); - when(tag_names.getIdAsync("host")).thenReturn( - Deferred.fromResult(new byte[] { 0, 0, 1 })); - when(tag_names.getNameAsync(new byte[] { 0, 0, 1 })) - .thenReturn(Deferred.fromResult("host")); - when(tag_names.getOrCreateIdAsync("host")).thenReturn( - Deferred.fromResult(new byte[] { 0, 0, 1 })); - when(tag_names.getIdAsync("dc")) - .thenThrow(new NoSuchUniqueName("dc", "metric")); - when(tag_values.getId("web01")).thenReturn(new byte[] { 0, 0, 1 }); - when(tag_values.getIdAsync("web01")).thenReturn( - Deferred.fromResult(new byte[] { 0, 0, 1 })); - when(tag_values.getNameAsync(new byte[] { 0, 0, 1 })) - .thenReturn(Deferred.fromResult("web01")); - when(tag_values.getOrCreateIdAsync("web01")).thenReturn( - Deferred.fromResult(new byte[] { 0, 0, 1 })); - when(tag_values.getId("web02")).thenReturn(new byte[] { 0, 0, 2 }); - when(tag_values.getIdAsync("web02")).thenReturn( - Deferred.fromResult(new byte[] { 0, 0, 2 })); - when(tag_values.getNameAsync(new byte[] { 0, 0, 2 })) - .thenReturn(Deferred.fromResult("web02")); - when(tag_values.getOrCreateIdAsync("web02")).thenReturn( - Deferred.fromResult(new byte[] { 0, 0, 2 })); - when(tag_values.getId("web03")) - .thenThrow(new NoSuchUniqueName("web03", "metric")); - - when(metrics.width()).thenReturn((short)3); - when(tag_names.width()).thenReturn((short)3); - when(tag_values.width()).thenReturn((short)3); + @Test + public void downsampleFullyAligned() { + testDownsampleScanBounds( + 60000L, + 1356998400L, 1357041600L, + + // The scan start time should be exactly the same as the query start time + // because it is aligned on both boundaries, timespan and interval. + 1356998400L, + + // However, because the query end time is already aligned on an interval, + // it should be snapped forward yet another interval and then snapped + // forward to the next timespan, which is an entire extra hour. + 1357045200L); } @Test - public void downsample() throws Exception { - int downsampleInterval = (int)DateTime.parseDuration("60s"); - query.downsample(downsampleInterval, Aggregators.SUM); - query.setStartTime(1356998400); - query.setEndTime(1357041600); - assertEquals(60000, TsdbQuery.ForTesting.getDownsampleIntervalMs(query)); - long scanStartTime = 1356998400 - Const.MAX_TIMESPAN * 2 - 60; - assertEquals(scanStartTime, TsdbQuery.ForTesting.getScanStartTimeSeconds(query)); - long scanEndTime = 1357041600 + Const.MAX_TIMESPAN + 1 + 60; - assertEquals(scanEndTime, TsdbQuery.ForTesting.getScanEndTimeSeconds(query)); + public void downsampleUnaligned() { + final long fifteen_minutes = 60L * 15L; + final long twelve_hours = 3600L * 12L; + final long now = 1427415547L; // Thu Mar 26 17:19:07 2015 GMT-7:00 DST + + testDownsampleScanBounds( + 1000L * fifteen_minutes, + now - twelve_hours, now, + + // Start time should have been snapped back to 1427415300 (5:15a) for the + // interval, then back to 1427371200 (5:00a) for the timespan. + 1427371200L, + + // End time should have been snapped forward to 1427416200 (5:30p) for + // the interval, then forward to 1427418000 (6:00p) for the timespan. + 1427418000L); + } + + @Test + public void downsampleWeirdly() { + final long day = 3600L * 24L; + final long twelve_hours = 3600L * 12L; + + // Thu Mar 26 17:19:07 2015 GMT-7:00 DST + // Fri, 27 Mar 2015 00:19:07 GMT + final long now = 1427415547L; + + testDownsampleScanBounds( + 1000L * day, + now - twelve_hours, now, + + // Start time should be midnight UTC, 26 March. + 1427328000L, + + // End time should be midnight UTC, 28 March. + 1427500800L); } @Test public void downsampleMilliseconds() throws Exception { - int downsampleInterval = (int)DateTime.parseDuration("60s"); - query.downsample(downsampleInterval, Aggregators.SUM); - query.setStartTime(1356998400000L); - query.setEndTime(1357041600000L); - assertEquals(60000, TsdbQuery.ForTesting.getDownsampleIntervalMs(query)); - long scanStartTime = 1356998400 - Const.MAX_TIMESPAN * 2 - 60; - assertEquals(scanStartTime, TsdbQuery.ForTesting.getScanStartTimeSeconds(query)); - long scanEndTime = 1357041600 + Const.MAX_TIMESPAN + 1 + 60; - assertEquals(scanEndTime, TsdbQuery.ForTesting.getScanEndTimeSeconds(query)); - } - - @Test (expected = NullPointerException.class) + final long start_time = 1356998400000L; + final long end_time = 1357041600000L; + final long downsample_interval = DateTime.parseDuration("60s"); + + query.downsample(downsample_interval, Aggregators.SUM); + query.setStartTime(start_time); + query.setEndTime(end_time); + assertEquals(60000L, TsdbQuery.ForTesting.getDownsampleIntervalMs(query)); + + // The scan start time should be exactly the same as the query start time + // because it is aligned on both boundaries, timespan and interval. + assertEquals(start_time / 1000L, + TsdbQuery.ForTesting.getScanStartTimeSeconds(query)); + + // However, because the query end time is already aligned on an interval, + // it should be snapped forward yet another interval and then snapped + // forward to the next timespan, which is an entire extra hour. + assertEquals((end_time + 3600000L) / 1000L, + TsdbQuery.ForTesting.getScanEndTimeSeconds(query)); + } + + @Test (expected = IllegalArgumentException.class) public void downsampleNullAgg() throws Exception { query.downsample(60, null); } @@ -175,19 +136,14 @@ public void downsampleInvalidInterval() throws Exception { @Test public void runLongSingleTSDownsample() throws Exception { - storeLongTimeSeriesSeconds(true, false);; - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); + storeLongTimeSeriesSeconds(true, false); + query.setStartTime(1356998400); query.setEndTime(1357041600); query.downsample(60000, Aggregators.AVG); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); + assertMeta(dps, 0, false); // Timeseries in intervals: (1), (2, 3), (4, 5), ... (298, 299), (300) int i = 0; @@ -218,19 +174,13 @@ public void runLongSingleTSDownsample() throws Exception { @Test public void runLongSingleTSDownsampleMs() throws Exception { storeLongTimeSeriesMs(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); + query.setStartTime(1356998400); query.setEndTime(1357041600); query.downsample(1000, Aggregators.AVG); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); final DataPoints[] dps = query.run(); - verify(client).newScanner(tsdb.table); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); + assertMeta(dps, 0, false); // Timeseries in intervals: (1), (2, 3), (4, 5), ... (298, 299), (300) int i = 0; @@ -260,19 +210,14 @@ public void runLongSingleTSDownsampleMs() throws Exception { @Test public void runLongSingleTSDownsampleAndRate() throws Exception { - storeLongTimeSeriesSeconds(true, false);; - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); + storeLongTimeSeriesSeconds(true, false); + query.setStartTime(1356998400); query.setEndTime(1357041600); query.downsample(60000, Aggregators.AVG); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, true); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, true); final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); + assertMeta(dps, 0, false); // Timeseries in intervals: (1), (2, 3), (4, 5), ... (298, 299), (300) // After downsampling: 1, 2.5, 4.5, ... 298.5, 300 @@ -305,18 +250,13 @@ public void runLongSingleTSDownsampleAndRate() throws Exception { @Test public void runLongSingleTSDownsampleAndRateMs() throws Exception { storeLongTimeSeriesMs(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); + query.setStartTime(1356998400); query.setEndTime(1357041600); query.downsample(1000, Aggregators.AVG); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, true); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, true); final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); + assertMeta(dps, 0, false); // Timeseries in intervals: (1), (2, 3), (4, 5), ... (298, 299), (300) // After downsampling: 1, 2.5, 4.5, ... 298.5, 300 @@ -346,18 +286,13 @@ public void runLongSingleTSDownsampleAndRateMs() throws Exception { @Test public void runFloatSingleTSDownsample() throws Exception { storeFloatTimeSeriesSeconds(true, false); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); + query.setStartTime(1356998400); query.setEndTime(1357041600); query.downsample(60000, Aggregators.AVG); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); + assertMeta(dps, 0, false); // Timeseries in intervals: (1.25), (1.5, 1.75), (2, 2.25), ... // (75.5, 75.75), (76). @@ -388,18 +323,13 @@ public void runFloatSingleTSDownsample() throws Exception { @Test public void runFloatSingleTSDownsampleMs() throws Exception { storeFloatTimeSeriesMs(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); + query.setStartTime(1356998400); query.setEndTime(1357041600); query.downsample(1000, Aggregators.AVG); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); + assertMeta(dps, 0, false); // Timeseries in intervals: (1.25), (1.5, 1.75), (2, 2.25), ... // (75.5, 75.75), (76). @@ -430,18 +360,13 @@ public void runFloatSingleTSDownsampleMs() throws Exception { @Test public void runFloatSingleTSDownsampleAndRate() throws Exception { storeFloatTimeSeriesSeconds(true, false); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); + query.setStartTime(1356998400); query.setEndTime(1357041600); query.downsample(60000, Aggregators.AVG); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, true); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, true); final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); + assertMeta(dps, 0, false); // Timeseries in intervals: (1.25), (1.5, 1.75), (2, 2.25), ... // (75.5, 75.75), (76). @@ -475,18 +400,13 @@ public void runFloatSingleTSDownsampleAndRate() throws Exception { @Test public void runFloatSingleTSDownsampleAndRateMs() throws Exception { storeFloatTimeSeriesMs(); - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); + query.setStartTime(1356998400); query.setEndTime(1357041600); query.downsample(1000, Aggregators.AVG); - query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, true); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, true); final DataPoints[] dps = query.run(); - assertNotNull(dps); - assertEquals("sys.cpu.user", dps[0].metricName()); - assertTrue(dps[0].getAggregatedTags().isEmpty()); - assertNull(dps[0].getAnnotations()); - assertEquals("web01", dps[0].getTags().get("host")); + assertMeta(dps, 0, false); // Timeseries in intervals: (1.25), (1.5, 1.75), (2, 2.25), ... // (75.5, 75.75), (76). @@ -513,143 +433,501 @@ public void runFloatSingleTSDownsampleAndRateMs() throws Exception { assertEquals(150, dps[0].size()); } - // ----------------- // - // Helper functions. // - // ----------------- // + @Test + public void runLongSingleTSDownsampleCount() throws Exception { + storeLongTimeSeriesSeconds(true, false); - private void storeLongTimeSeriesSeconds(final boolean two_metrics, - final boolean offset) throws Exception { - storeLongTimeSeriesSecondsWithBasetime(1356998400L, two_metrics, offset); - } + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.downsample(60000, Aggregators.COUNT); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); - private void storeLongTimeSeriesSecondsWithBasetime(final long baseTimestamp, - final boolean two_metrics, final boolean offset) throws Exception { - setQueryStorage(); - // dump a bunch of rows of two metrics so that we can test filtering out - // on the metric - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - long timestamp = baseTimestamp; - for (int i = 1; i <= 300; i++) { - tsdb.addPoint("sys.cpu.user", timestamp += 30, i, tags).joinUninterruptibly(); - if (two_metrics) { - tsdb.addPoint("sys.cpu.nice", timestamp, i, tags).joinUninterruptibly(); + // Timeseries in intervals: (1), (2, 3), (4, 5), ... (298, 299), (300) + int i = 0; + for (DataPoint dp : dps[0]) { + // Downsampler outputs just doubles. + assertFalse(dp.isInteger()); + if (i == 0 || i == 150) { + assertEquals(1, dp.doubleValue(), 0.00001); + } else { + assertEquals(2, dp.doubleValue(), 0.00001); } + ++i; } + // Out of 300 values, the first and the last intervals have one value each, + // and the 149 intervals in the middle have two values for each. + assertEquals(151, dps[0].size()); + } + + @Test + public void runLongSingleTSDownsampleAll() throws Exception { + storeLongTimeSeriesSeconds(true, false); + final TSQuery ts_query = new TSQuery(); + ts_query.setStart("1356998400"); + ts_query.setEnd("1357041600"); + + final HashMap<String, String> tags = new HashMap<String, String>(1); + tags.put("host", "web01"); + final TSSubQuery sub = new TSSubQuery(); + sub.setTags(tags); + sub.setMetric("sys.cpu.user"); + sub.setAggregator("sum"); + sub.setDownsample("0all-sum"); + + ts_query.setQueries(Lists.newArrayList(sub)); + ts_query.validateAndSetQuery(); + query.configureFromQuery(ts_query, 0); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); - // dump a parallel set but invert the values - tags.clear(); - tags.put("host", "web02"); - timestamp = baseTimestamp + (offset ? 15 : 0); - for (int i = 300; i > 0; i--) { - tsdb.addPoint("sys.cpu.user", timestamp += 30, i, tags).joinUninterruptibly(); - if (two_metrics) { - tsdb.addPoint("sys.cpu.nice", timestamp, i, tags).joinUninterruptibly(); - } + for (DataPoint dp : dps[0]) { + // Downsampler outputs just doubles. + assertFalse(dp.isInteger()); + assertEquals(45150, dp.doubleValue(), 0.00001); + assertEquals(1356998400000L, dp.timestamp()); } + // Out of 300 values, the first and the last intervals have one value each, + // and the 149 intervals in the middle have two values for each. + assertEquals(1, dps[0].size()); } + + @Test + public void runLongSingleTSDownsampleAllSubSet() throws Exception { + storeLongTimeSeriesSeconds(true, false); + final TSQuery ts_query = new TSQuery(); + ts_query.setStart("1356998500"); + ts_query.setEnd("1356998600"); + + final HashMap<String, String> tags = new HashMap<String, String>(1); + tags.put("host", "web01"); + final TSSubQuery sub = new TSSubQuery(); + sub.setTags(tags); + sub.setMetric("sys.cpu.user"); + sub.setAggregator("sum"); + sub.setDownsample("0all-sum"); + + ts_query.setQueries(Lists.newArrayList(sub)); + ts_query.validateAndSetQuery(); + query.configureFromQuery(ts_query, 0); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); - private void storeLongTimeSeriesMs() throws Exception { - setQueryStorage(); - // dump a bunch of rows of two metrics so that we can test filtering out - // on the metric - HashMap<String, String> tags = new HashMap<String, String>(1); + for (DataPoint dp : dps[0]) { + // Downsampler outputs just doubles. + assertFalse(dp.isInteger()); + assertEquals(15, dp.doubleValue(), 0.00001); + assertEquals(1356998500000L, dp.timestamp()); + } + // Out of 300 values, the first and the last intervals have one value each, + // and the 149 intervals in the middle have two values for each. + assertEquals(1, dps[0].size()); + } + + @Test + public void runLongSingleTSDownsampleAllNoEnd() throws Exception { + storeLongTimeSeriesSeconds(true, false); + final TSQuery ts_query = new TSQuery(); + ts_query.setStart("1356998400"); + + final HashMap<String, String> tags = new HashMap<String, String>(1); tags.put("host", "web01"); - long timestamp = 1356998400000L; - for (int i = 1; i <= 300; i++) { - tsdb.addPoint("sys.cpu.user", timestamp += 500, i, tags).joinUninterruptibly(); - tsdb.addPoint("sys.cpu.nice", timestamp, i, tags).joinUninterruptibly(); + final TSSubQuery sub = new TSSubQuery(); + sub.setTags(tags); + sub.setMetric("sys.cpu.user"); + sub.setAggregator("sum"); + sub.setDownsample("0all-sum"); + + ts_query.setQueries(Lists.newArrayList(sub)); + ts_query.validateAndSetQuery(); + query.configureFromQuery(ts_query, 0); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + for (DataPoint dp : dps[0]) { + // Downsampler outputs just doubles. + assertFalse(dp.isInteger()); + assertEquals(45150, dp.doubleValue(), 0.00001); + assertEquals(1356998400000L, dp.timestamp()); } + // Out of 300 values, the first and the last intervals have one value each, + // and the 149 intervals in the middle have two values for each. + assertEquals(1, dps[0].size()); + } + + // this could happen. + @Test + public void runFloatSingleTSDownsampleAndRateAndCount() throws Exception { + storeFloatTimeSeriesSeconds(true, false); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.downsample(60000, Aggregators.COUNT); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, true); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); - // dump a parallel set but invert the values - tags.clear(); - tags.put("host", "web02"); - timestamp = 1356998400000L; - for (int i = 300; i > 0; i--) { - tsdb.addPoint("sys.cpu.user", timestamp += 500, i, tags).joinUninterruptibly(); - tsdb.addPoint("sys.cpu.nice", timestamp, i, tags).joinUninterruptibly(); + // Timeseries in intervals: (1.25), (1.5, 1.75), (2, 2.25), ... + // (75.5, 75.75), (76). + // After downsampling: 1.25, 1.625, 2.125, ... 75.625, 76 + long expected_timestamp = 1356998460000L; + int i = 0; + for (DataPoint dp : dps[0]) { + assertFalse(dp.isInteger()); + if (i == 0) { + // switching from 1 value to 2 on the first point + assertEquals(0.016666F, dp.doubleValue(), 0.00001); + } else if (i == 149) { + // switching from 2 values to 1 on the last point + assertEquals(-0.016666F, dp.doubleValue(), 0.00001); + } else { + // no difference between the value counts for most of these so zero + assertEquals(0.000F, dp.doubleValue(), 0.00001); + } + // Timestamp of an interval should be aligned by the interval. + assertEquals(0, dp.timestamp() % 60000); + assertEquals(expected_timestamp, dp.timestamp()); + expected_timestamp += 60000; + ++i; } + assertEquals(150, dps[0].size()); } - private void storeFloatTimeSeriesSeconds(final boolean two_metrics, - final boolean offset) throws Exception { - setQueryStorage(); - // dump a bunch of rows of two metrics so that we can test filtering out - // on the metric + @Test (expected = IllegalArgumentException.class) + public void runLongSingleTSDownsampleNone() throws Exception { + storeLongTimeSeriesSeconds(true, false); HashMap<String, String> tags = new HashMap<String, String>(1); tags.put("host", "web01"); - long timestamp = 1356998400; - for (float i = 1.25F; i <= 76; i += 0.25F) { - tsdb.addPoint("sys.cpu.user", timestamp += 30, i, tags).joinUninterruptibly(); - if (two_metrics) { - tsdb.addPoint("sys.cpu.nice", timestamp, i, tags).joinUninterruptibly(); - } + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.downsample(60000, Aggregators.NONE); + query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); + } + + @Test (expected = RuntimeException.class) + public void runLongSingleTSDownsampleNoneSnuckIn() throws Exception { + storeLongTimeSeriesSeconds(true, false); + final TSQuery ts_query = new TSQuery(); + ts_query.setStart("1356998400"); + ts_query.setEnd("1357041600"); + + final HashMap<String, String> tags = new HashMap<String, String>(1); + tags.put("host", "web01"); + final TSSubQuery sub = new TSSubQuery(); + sub.setTags(tags); + sub.setMetric("sys.cpu.user"); + sub.setAggregator("sum"); + sub.setDownsample("1m-sum"); + + ts_query.setQueries(Lists.newArrayList(sub)); + ts_query.validateAndSetQuery(); + query.configureFromQuery(ts_query, 0); + Whitebox.setInternalState(query, "downsampler", Aggregators.NONE); + + final DataPoints[] dps = query.run(); + for (DataPoint dp : dps[0]) { + dp.timestamp(); } + } + + /** + * A helper interface to be used by the filling-test code. + */ + interface Validator { + /** @return true if the argument is valid. */ + boolean isValidValue(double value); + + /** @return the fill policy to be used while downsampling. */ + FillPolicy getFillPolicy(); + + /** @return true if the argument is the sentinel for empty intervals. */ + boolean isMissingValue(double value); + } - // dump a parallel set but invert the values - tags.clear(); - tags.put("host", "web02"); - timestamp = offset ? 1356998415 : 1356998400; - for (float i = 75F; i > 0; i -= 0.25F) { - tsdb.addPoint("sys.cpu.user", timestamp += 30, i, tags).joinUninterruptibly(); - if (two_metrics) { - tsdb.addPoint("sys.cpu.nice", timestamp, i, tags).joinUninterruptibly(); - } + // Fill missing intervals with NaNs. + abstract class NaNValidator implements Validator { + @Override + public FillPolicy getFillPolicy() { + return FillPolicy.NOT_A_NUMBER; + } + + @Override + public boolean isMissingValue(final double value) { + return Double.isNaN(value); } } - private void storeFloatTimeSeriesMs() throws Exception { - setQueryStorage(); - // dump a bunch of rows of two metrics so that we can test filtering out - // on the metric - HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - long timestamp = 1356998400000L; - for (float i = 1.25F; i <= 76; i += 0.25F) { - tsdb.addPoint("sys.cpu.user", timestamp += 500, i, tags).joinUninterruptibly(); - tsdb.addPoint("sys.cpu.nice", timestamp, i, tags).joinUninterruptibly(); + // Fill missing intervals with zeroes. + abstract class ZeroValidator implements Validator { + @Override + public FillPolicy getFillPolicy() { + return FillPolicy.ZERO; } - // dump a parallel set but invert the values - tags.clear(); - tags.put("host", "web02"); - timestamp = 1356998400000L; - for (float i = 75F; i > 0; i -= 0.25F) { - tsdb.addPoint("sys.cpu.user", timestamp += 500, i, tags).joinUninterruptibly(); - tsdb.addPoint("sys.cpu.nice", timestamp, i, tags).joinUninterruptibly(); + @Override + public boolean isMissingValue(final double value) { + return DoubleMath.fuzzyEquals(0.0, value, 0.0001); } } - @SuppressWarnings("unchecked") - private void setQueryStorage() throws Exception { - storage = new MockBase(tsdb, client, true, true, true, true); - storage.setFamily("t".getBytes(MockBase.ASCII())); - - PowerMockito.mockStatic(IncomingDataPoints.class); - PowerMockito.doAnswer( - new Answer<byte[]>() { - public byte[] answer(final InvocationOnMock args) - throws Exception { - final String metric = (String)args.getArguments()[1]; - final Map<String, String> tags = - (Map<String, String>)args.getArguments()[2]; - - if (metric.equals("sys.cpu.user")) { - if (tags.get("host").equals("web01")) { - return new byte[] { 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1}; - } else { - return new byte[] { 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2}; - } - } else { - if (tags.get("host").equals("web01")) { - return new byte[] { 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1}; - } else { - return new byte[] { 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2}; - } + @Test + public void runSumAvgLongSingleTSDownsampleWNulls() throws Exception { + storeLongTimeSeriesWithMissingData(); + + runTSDownsampleWithMissingData(Aggregators.SUM, Aggregators.AVG, + // 301.5, 301.5, 301.5, ... + new NaNValidator() { + @Override + public boolean isValidValue(final double value) { + return DoubleMath.fuzzyEquals(301.5, value, 0.0001); + } + }); + } + + @Test + public void runAvgSumLongSingleTSDownsampleWNulls() throws Exception { + storeLongTimeSeriesWithMissingData(); + + runTSDownsampleWithMissingData(Aggregators.AVG, Aggregators.SUM, + // 152, 301.5, 155, 301.5, 158, 301.5, ... + new NaNValidator() { + private boolean even = false; + private double even_expected = 149.0; + + @Override + public boolean isValidValue(final double value) { + even = !even; + if (even) { + even_expected += 3.0; + return DoubleMath.fuzzyEquals(even_expected, value, 0.0001); + } else { + return DoubleMath.fuzzyEquals(301.5, value, 0.0001); + } + } + }); + } + + @Test + public void runAvgAvgLongSingleTSDownsampleWNulls() throws Exception { + storeLongTimeSeriesWithMissingData(); + + runTSDownsampleWithMissingData(Aggregators.AVG, Aggregators.AVG, + // 150.75, 150.75, 150.75, ... + new ZeroValidator() { + @Override + public boolean isValidValue(final double value) { + return DoubleMath.fuzzyEquals(150.75, value, 0.0001); + } + }); + } + + @Test + public void runSumSumLongSingleTSDownsampleWNulls() throws Exception { + storeLongTimeSeriesWithMissingData(); + + runTSDownsampleWithMissingData(Aggregators.SUM, Aggregators.SUM, + // 304, 603, 310, 603, 316, 603, ... + new NaNValidator() { + private double even_expected = 298.0; + private final double odd_expected = 603.0; + private boolean even = false; + + @Override + public boolean isValidValue(final double value) { + even = !even; + if (even) { + even_expected += 6.0; + return DoubleMath.fuzzyEquals(even_expected, value, 0.0001); + } else { + return DoubleMath.fuzzyEquals(odd_expected, value, 0.0001); + } + } + }); + } + + @Test + public void runMinMinLongSingleTSDownsampleWNulls() throws Exception { + storeLongTimeSeriesWithMissingData(); + + runTSDownsampleWithMissingData(Aggregators.MIN, Aggregators.MIN, + // 2, 5, 8, ..., 143, 146, 149, 149, 145, 143, 139, 133, 131, ... + new ZeroValidator() { + private double even_expected = -4.0; + private double even_change = 6.0; + private double odd_expected = -1.0; + private double odd_change = 6.0; + private boolean even = false; + + @Override + public boolean isValidValue(final double value) { + even = !even; + if (even) { + even_expected += even_change; + + // Check for the point at which even terms change. + if (DoubleMath.fuzzyEquals(even_expected, 152.0, 0.0001)) { + // After this point, even terms begin decreasing by six. + even_expected = 149.0; + even_change = -6.0; + } + + return DoubleMath.fuzzyEquals(even_expected, value, 0.0001); + } else { + odd_expected += odd_change; + + // Check for the point at which odd terms change. + if (DoubleMath.fuzzyEquals(odd_expected, 155.0, 0.0001)) { + // After this point, odd terms begin decreasing by six. + odd_expected = 145.0; + odd_change = -6.0; + } + + return DoubleMath.fuzzyEquals(odd_expected, value, 0.0001); + } + } + }); + } + + @Test + public void runMinSumLongSingleTSDownsampleWNulls() throws Exception { + storeLongTimeSeriesWithMissingData(); + + runTSDownsampleWithMissingData(Aggregators.MIN, Aggregators.SUM, + // 5, 11, 17, 23, ..., 197, 203, 197, 215, 191, 227, 185, 239, ..., + // 287, 155, 299, 149, 292, 143, 280, ... + new NaNValidator() { + private double even_expected = -7.0; + private double even_change = 12.0; + private double odd_expected = -1.0; + private double odd_change = 12.0; + private boolean even = false; + + @Override + public boolean isValidValue(final double value) { + even = !even; + if (even) { + even_expected += even_change; + + // Check for the point at which even terms change. + if (DoubleMath.fuzzyEquals(even_expected, 209.0, 0.0001)) { + // After this point, even terms begin decreasing by six. + even_expected = 197.0; + even_change = -6.0; + } + + return DoubleMath.fuzzyEquals(even_expected, value, 0.0001); + } else { + odd_expected += odd_change; + + // Check for the point at which odd terms change. + if (DoubleMath.fuzzyEquals(odd_expected, 311.0, 0.0001)) { + // After this point, odd terms begin decreasing by twelve. + odd_expected = 292.0; + odd_change = -12.0; } + + return DoubleMath.fuzzyEquals(odd_expected, value, 0.0001); } } - ).when(IncomingDataPoints.class, "rowKeyTemplate", (TSDB)any(), anyString(), - (Map<String, String>)any()); + }); + } + + @Test + public void runSumMinLongSingleTSDownsampleWNulls() throws Exception { + storeLongTimeSeriesWithMissingData(); + + runTSDownsampleWithMissingData(Aggregators.SUM, Aggregators.MIN, + // 301, 300, 301, 300, ... + new NaNValidator() { + private boolean even = false; + + @Override + public boolean isValidValue(final double value) { + even = !even; + return DoubleMath.fuzzyEquals( + even ? 301.0 : 300.0, value, 0.0001); + } + }); + } + + /** + * Precondition: the time series have been stored. + */ + public void runTSDownsampleWithMissingData(final Aggregator queryAggregator, + final Aggregator downsampleAggregator, final Validator validator) + throws Exception { + final long start_time = 1356998400L; + final long end_time = 1357041600L; + final int ds_interval = 30; + final long ds_interval_ms = 1000L * ds_interval; + final String metric = METRIC_STRING; + + final HashMap<String, String> tags = new HashMap<String, String>(0); + query.setStartTime(start_time); + query.setEndTime(end_time); + query.downsample(ds_interval_ms, downsampleAggregator, + validator.getFillPolicy()); + query.setTimeSeries(metric, tags, queryAggregator, false); + final DataPoints[] dps = query.run(); + + assertNotNull(dps); + assertEquals(metric, dps[0].metricName()); + assertFalse(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + + // For the reasoning behind this calculation, see the following methods: + // TsdbQuery#getScanStartTimeSeconds() + // TsdbQuery#getScanEndTimeSeconds() + + int i = 0; + long expected_timestamp_ms = 1000L * start_time; + for (final DataPoint dp : dps[0]) { + // Downsampler outputs just doubles. + assertFalse(dp.isInteger()); + + // There should be only one hundred valid values. + if (i++ < 100) { + // Check the value. + assertTrue(validator.isValidValue(dp.doubleValue())); + } else { + // Otherwise, the value should be the special missing value. + assertTrue(validator.isMissingValue(dp.doubleValue())); + } + + // The timestamp should match our expectation based on the interval. + assertEquals(expected_timestamp_ms, dp.timestamp()); + + // Move to the next expected interval. + expected_timestamp_ms += ds_interval_ms; + } + + // Ensure we got the number of points we expected. + assertEquals((end_time - start_time + 3600L) / ds_interval, dps[0].size()); + } + + /** + * Helper to test the start and stop times in a query for downsampling + * @param downsample_interval The downsample interval + * @param start_time The start of the query + * @param end_time The end of the query + * @param expected_start_time What we expect the TSDBQuery class to give us + * @param expected_end_time What we expect the TSDBQuery class to give us + */ + private void testDownsampleScanBounds(final long downsample_interval, + final long start_time, final long end_time, + final long expected_start_time, final long expected_end_time) { + query.downsample(downsample_interval, Aggregators.SUM); + query.setStartTime(start_time); + query.setEndTime(end_time); + + assertEquals(expected_start_time, + TsdbQuery.ForTesting.getScanStartTimeSeconds(query)); + + assertEquals(expected_end_time, + TsdbQuery.ForTesting.getScanEndTimeSeconds(query)); } } diff --git a/test/core/TestTsdbQueryDownsampleSalted.java b/test/core/TestTsdbQueryDownsampleSalted.java new file mode 100644 index 0000000000..d4a679c8cf --- /dev/null +++ b/test/core/TestTsdbQueryDownsampleSalted.java @@ -0,0 +1,38 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import org.junit.Before; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.modules.junit4.PowerMockRunner; + +/** + * Integration test that runs all of the tests in {@see TestTsdbQueryDownsample} + * but with salting enabled just to verify nothing goes wrong with the extra + * salt bytes. + */ +@RunWith(PowerMockRunner.class) +public class TestTsdbQueryDownsampleSalted extends TestTsdbQueryDownsample { + + @Before + public void beforeLocal() throws Exception { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(2); + PowerMockito.when(Const.MAX_NUM_TAGS()).thenReturn((short) 8); + + query = new TsdbQuery(tsdb); + } + +} diff --git a/test/core/TestTsdbQueryHistogramQueries.java b/test/core/TestTsdbQueryHistogramQueries.java new file mode 100644 index 0000000000..118cfd5c72 --- /dev/null +++ b/test/core/TestTsdbQueryHistogramQueries.java @@ -0,0 +1,481 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. + +package net.opentsdb.core; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.hbase.async.HBaseClient; +import org.jboss.netty.util.HashedWheelTimer; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +import com.google.common.collect.Maps; + +import net.opentsdb.meta.Annotation; +import net.opentsdb.uid.UniqueId.UniqueIdType; +import net.opentsdb.utils.Config; +import net.opentsdb.utils.Threads; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({TSDB.class, TsdbQuery.class }) +public class TestTsdbQueryHistogramQueries extends BaseTsdbTest { + protected TsdbQuery query = null; + + @Before + public void before() throws Exception { + // Copying the whole thing as the SPY in the base mucks up the references. + uid_map = Maps.newHashMap(); + PowerMockito.mockStatic(Threads.class); + timer = new FakeTaskTimer(); + PowerMockito.when(Threads.newTimer(anyString())).thenReturn(timer); + PowerMockito.when(Threads.newTimer(anyInt(), anyString())).thenReturn(timer); + + PowerMockito.whenNew(HashedWheelTimer.class).withNoArguments() + .thenReturn(timer); + PowerMockito.whenNew(HBaseClient.class).withAnyArguments() + .thenReturn(client); + + config = new Config(false); + config.overrideConfig("tsd.storage.enable_compaction", "false"); + tsdb = new TSDB(config); + + config.setAutoMetric(true); + + Whitebox.setInternalState(tsdb, "metrics", metrics); + Whitebox.setInternalState(tsdb, "tag_names", tag_names); + Whitebox.setInternalState(tsdb, "tag_values", tag_values); + + setupMetricMaps(); + setupTagkMaps(); + setupTagvMaps(); + + mockUID(UniqueIdType.METRIC, HISTOGRAM_METRIC_STRING, HISTOGRAM_METRIC_BYTES); + + // add metrics and tags to the UIDs list for other functions to share + uid_map.put(METRIC_STRING, METRIC_BYTES); + uid_map.put(METRIC_B_STRING, METRIC_B_BYTES); + uid_map.put(NSUN_METRIC, NSUI_METRIC); + uid_map.put(HISTOGRAM_METRIC_STRING, HISTOGRAM_METRIC_BYTES); + + uid_map.put(TAGK_STRING, TAGK_BYTES); + uid_map.put(TAGK_B_STRING, TAGK_B_BYTES); + uid_map.put(NSUN_TAGK, NSUI_TAGK); + + uid_map.put(TAGV_STRING, TAGV_BYTES); + uid_map.put(TAGV_B_STRING, TAGV_B_BYTES); + uid_map.put(NSUN_TAGV, NSUI_TAGV); + + uid_map.putAll(UIDS); + + when(metrics.width()).thenReturn((short)3); + when(tag_names.width()).thenReturn((short)3); + when(tag_values.width()).thenReturn((short)3); + + tags = new HashMap<String, String>(1); + tags.put(TAGK_STRING, TAGV_STRING); + config.overrideConfig("tsd.core.histograms.config", + "{\"net.opentsdb.core.LongHistogramDataPointForTestDecoder\": 0}"); + HistogramCodecManager manager = + new HistogramCodecManager(tsdb); + Whitebox.setInternalState(tsdb, "histogram_manager", manager); + + query = new TsdbQuery(tsdb); + } + + @Test + public void runSingleTsMsSinglePercentile() throws Exception { + this.storeTestHistogramTimeSeriesMs(); + HashMap<String, String> tags = new HashMap<String, String>(1); + tags.put("host", "web01"); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries("msg.end2end.latency", tags, Aggregators.SUM, false); + + List<Float> percentiles = new ArrayList<Float>(); + float per_98 = 0.98F; + percentiles.add(per_98); + + query.setPercentiles(percentiles); + final DataPoints[] dps = query.runHistogram(); + + assertNotNull(dps); + assertTrue(dps[0].isPercentile()); + assertEquals("msg.end2end.latency_pct_0.98", dps[0].metricName()); + + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertEquals("web01", dps[0].getTags().get("host")); + + int value = 1; + for (DataPoint dp : dps[0]) { + assertEquals(value * 0.98, dp.doubleValue(), 0.0001); + value++; + } + assertEquals(300, dps[0].aggregatedSize()); + } // end runSingleTsMsSinglePercentile() + + @Test + public void runSingleTsMsDoulePercentile() throws Exception { + this.storeTestHistogramTimeSeriesMs(); + HashMap<String, String> tags = new HashMap<String, String>(1); + tags.put("host", "web01"); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries("msg.end2end.latency", tags, Aggregators.SUM, false); + + List<Float> percentiles = new ArrayList<Float>(); + float per_98 = 0.98F; + percentiles.add(per_98); + float per_95 = 0.95F; + percentiles.add(per_95); + + query.setPercentiles(percentiles); + + final DataPoints[] dps = query.runHistogram(); + + assertNotNull(dps); + assertTrue(dps[0].isPercentile()); + assertTrue(dps[1].isPercentile()); + + assertEquals("msg.end2end.latency_pct_0.98", dps[0].metricName()); + assertEquals("msg.end2end.latency_pct_0.95", dps[1].metricName()); + + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertEquals("web01", dps[0].getTags().get("host")); + + assertTrue(dps[1].getAggregatedTags().isEmpty()); + assertNull(dps[1].getAnnotations()); + assertEquals("web01", dps[1].getTags().get("host")); + + int value = 1; + for (DataPoint dp : dps[0]) { + assertEquals(value * 0.98, dp.doubleValue(), 0.0001); + value++; + } + assertEquals(300, dps[0].aggregatedSize()); + + int value_95 = 1; + for (DataPoint dp : dps[1]) { + assertEquals(value_95 * 0.95, dp.doubleValue(), 0.0001); + value_95++; + } + assertEquals(300, dps[1].aggregatedSize()); + } // end runSingleTsMsSinglePercentile() + + @Test + public void runSingleTsMsTwoAggSum() throws Exception { + this.storeTestHistogramTimeSeriesMs(); + + HashMap<String, String> tags = new HashMap<String, String>(); + + query.setStartTime(1356998400L); + query.setEndTime(1357041600L); + query.setTimeSeries("msg.end2end.latency", tags, Aggregators.SUM, false); + + List<Float> percentiles = new ArrayList<Float>(); + float per_98 = 0.98F; + percentiles.add(per_98); + query.setPercentiles(percentiles); + + final DataPoints[] dps = query.runHistogram(); + + assertNotNull(dps); + assertTrue(dps[0].isPercentile()); + + assertEquals("msg.end2end.latency_pct_0.98", dps[0].metricName()); + assertEquals("host", dps[0].getAggregatedTags().get(0)); + assertNull(dps[0].getAnnotations()); + assertTrue(dps[0].getTags().isEmpty()); + + for (DataPoint dp : dps[0]) { + assertEquals(301 * 0.98, dp.doubleValue(), 0.0001); + } + assertEquals(300, dps[0].size()); + } // end runSingleTsMsTwoAggSum() + + @Test + public void runSingleTsMsAggNone() throws Exception { + this.storeTestHistogramTimeSeriesMs(); + + HashMap<String, String> tags = new HashMap<String, String>(); + + query.setStartTime(1356998400L); + query.setEndTime(1357041600L); + query.setTimeSeries("msg.end2end.latency", tags, Aggregators.NONE, false); + + List<Float> percentiles = new ArrayList<Float>(); + float per_98 = 0.98F; + percentiles.add(per_98); + query.setPercentiles(percentiles); + + final DataPoints[] dps = query.runHistogram(); + + assertNotNull(dps); + assertEquals(2, dps.length); + + assertTrue(dps[0].isPercentile()); + assertTrue(dps[1].isPercentile()); + + assertEquals("msg.end2end.latency_pct_0.98", dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertEquals(1, dps[0].getTags().size()); + assertEquals("web01", dps[0].getTags().get("host")); + + assertEquals("msg.end2end.latency_pct_0.98", dps[1].metricName()); + assertTrue(dps[1].getAggregatedTags().isEmpty()); + assertNull(dps[1].getAnnotations()); + assertEquals(1, dps[1].getTags().size()); + assertEquals("web02", dps[1].getTags().get("host")); + + int value = 1; + for (DataPoint dp : dps[0]) { + assertEquals(value * 0.98, dp.doubleValue(), 0.0001); + ++value; + } + assertEquals(300, dps[0].size()); + + int value_other = 300; + for (DataPoint dp : dps[1]) { + assertEquals(value_other * 0.98, dp.doubleValue(), 0.0001); + --value_other; + } + assertEquals(300, dps[1].size()); + } // end runSingleTsMsTwoAggSum() + + @Test + public void runSingleTsMsAggSumTwoGroups() throws Exception { + this.storeTestHistogramTimeSeriesMs(); + + HashMap<String, String> tags = new HashMap<String, String>(); + tags.put("host", "*"); + + query.setStartTime(1356998400L); + query.setEndTime(1357041600L); + query.setTimeSeries("msg.end2end.latency", tags, Aggregators.SUM, false); + + List<Float> percentiles = new ArrayList<Float>(); + float per_98 = 0.98F; + percentiles.add(per_98); + query.setPercentiles(percentiles); + + final DataPoints[] dps = query.runHistogram(); + + assertNotNull(dps); + assertEquals(2, dps.length); + + assertTrue(dps[0].isPercentile()); + assertTrue(dps[1].isPercentile()); + + assertEquals("msg.end2end.latency_pct_0.98", dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertEquals(1, dps[0].getTags().size()); + assertEquals("web01", dps[0].getTags().get("host")); + + assertEquals("msg.end2end.latency_pct_0.98", dps[1].metricName()); + assertTrue(dps[1].getAggregatedTags().isEmpty()); + assertNull(dps[1].getAnnotations()); + assertEquals(1, dps[1].getTags().size()); + assertEquals("web02", dps[1].getTags().get("host")); + + int value = 1; + for (DataPoint dp : dps[0]) { + assertEquals(value * 0.98, dp.doubleValue(), 0.0001); + ++value; + } + assertEquals(300, dps[0].size()); + + int value_other = 300; + for (DataPoint dp : dps[1]) { + assertEquals(value_other * 0.98, dp.doubleValue(), 0.0001); + --value_other; + } + assertEquals(300, dps[1].size()); + } // end runSingleTsMsTwoAggSum() + + @Test + public void runWithAnnotation() throws Exception { + this.storeTestHistogramTimeSeriesSeconds(false); + + final Annotation note = new Annotation(); + note.setTSUID(getTSUIDString(HISTOGRAM_METRIC_STRING, TAGK_STRING, TAGV_STRING)); + note.setStartTime(1356998490); + note.setDescription("Hello World!"); + note.syncToStorage(tsdb, false).joinUninterruptibly(); + + HashMap<String, String> tags = new HashMap<String, String>(1); + tags.put("host", "web01"); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries("msg.end2end.latency", tags, Aggregators.SUM, false); + + List<Float> percentiles = new ArrayList<Float>(); + float per_98 = 0.98F; + percentiles.add(per_98); + query.setPercentiles(percentiles); + + final DataPoints[] dps = query.runHistogram(); + + assertNotNull(dps); + assertEquals(1, dps[0].getAnnotations().size()); + assertEquals("Hello World!", dps[0].getAnnotations().get(0).getDescription()); + + int value = 1; + for (DataPoint dp : dps[0]) { + assertEquals(value * 0.98, dp.doubleValue(), 0.0001); + value++; + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runWithOnlyAnnotation() throws Exception { + this.storeTestHistogramTimeSeriesSeconds(false); + + byte[] key = getRowKey(HISTOGRAM_METRIC_STRING, 1357002000, TAGK_STRING, TAGV_STRING); + storage.flushRow(key); + final Annotation note = new Annotation(); + note.setTSUID(getTSUIDString(HISTOGRAM_METRIC_STRING, TAGK_STRING, TAGV_STRING)); + note.setStartTime(1357002090); + note.setDescription("Hello World!"); + note.syncToStorage(tsdb, false).joinUninterruptibly(); + + HashMap<String, String> tags = new HashMap<String, String>(1); + tags.put("host", "web01"); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries("msg.end2end.latency", tags, Aggregators.SUM, false); + + List<Float> percentiles = new ArrayList<Float>(); + float per_98 = 0.98F; + percentiles.add(per_98); + query.setPercentiles(percentiles); + + final DataPoints[] dps = query.runHistogram(); + + assertNotNull(dps); + assertEquals(1, dps[0].getAnnotations().size()); + assertEquals("Hello World!", dps[0].getAnnotations().get(0).getDescription()); + + int value = 1; + for (DataPoint dp : dps[0]) { + assertEquals(value * 0.98, dp.doubleValue(), 0.0001); + value++; + // account for the jump + if (value == 120) { + value = 240; + } + } + assertEquals(180, dps[0].size()); + } + + @Test + public void runTSUIDQuery() throws Exception { + this.storeTestHistogramTimeSeriesSeconds(false); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + final List<String> tsuids = new ArrayList<String>(1); + tsuids.add(getTSUIDString(HISTOGRAM_METRIC_STRING, TAGK_STRING, TAGV_STRING)); + + query.setTimeSeries(tsuids, Aggregators.SUM, false); + List<Float> percentiles = new ArrayList<Float>(); + float per_98 = 0.98F; + percentiles.add(per_98); + query.setPercentiles(percentiles); + + final DataPoints[] dps = query.runHistogram(); + + assertNotNull(dps); + assertEquals("msg.end2end.latency_pct_0.98", dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertEquals("web01", dps[0].getTags().get("host")); + + int value = 1; + for (DataPoint dp : dps[0]) { + assertEquals(value * 0.98, dp.doubleValue(), 0.0001); + value++; + } + assertEquals(300, dps[0].aggregatedSize()); + } + + @Test + public void runTSUIDsAggSum() throws Exception { + this.storeTestHistogramTimeSeriesSeconds(false); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + + final List<String> tsuids = new ArrayList<String>(1); + tsuids.add(getTSUIDString(HISTOGRAM_METRIC_STRING, TAGK_STRING, TAGV_STRING)); + tsuids.add(getTSUIDString(HISTOGRAM_METRIC_STRING, TAGK_STRING, TAGV_B_STRING)); + query.setTimeSeries(tsuids, Aggregators.SUM, false); + + List<Float> percentiles = new ArrayList<Float>(); + float per_98 = 0.98F; + percentiles.add(per_98); + query.setPercentiles(percentiles); + + final DataPoints[] dps = query.runHistogram(); + + assertNotNull(dps); + assertEquals("msg.end2end.latency_pct_0.98", dps[0].metricName()); + assertEquals("host", dps[0].getAggregatedTags().get(0)); + assertNull(dps[0].getAnnotations()); + assertTrue(dps[0].getTags().isEmpty()); + + for (DataPoint dp : dps[0]) { + assertEquals(301 * 0.98, dp.doubleValue(), 0.0001); + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runTSUIDQueryNoData() throws Exception { + setDataPointStorage(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + final List<String> tsuids = new ArrayList<String>(1); + tsuids.add(getTSUIDString(HISTOGRAM_METRIC_STRING, TAGK_STRING, TAGV_STRING)); + query.setTimeSeries(tsuids, Aggregators.SUM, false); + + List<Float> percentiles = new ArrayList<Float>(); + float per_98 = 0.98F; + percentiles.add(per_98); + query.setPercentiles(percentiles); + + final DataPoints[] dps = query.runHistogram(); + + assertNotNull(dps); + assertEquals(0, dps.length); + } +} diff --git a/test/core/TestTsdbQueryHistogramQueriesSalted.java b/test/core/TestTsdbQueryHistogramQueriesSalted.java new file mode 100644 index 0000000000..89613cc7ba --- /dev/null +++ b/test/core/TestTsdbQueryHistogramQueriesSalted.java @@ -0,0 +1,29 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. + +package net.opentsdb.core; + +import org.junit.Before; +import org.powermock.api.mockito.PowerMockito; + +public class TestTsdbQueryHistogramQueriesSalted extends TestTsdbQueryHistogramQueries { + + @Before + public void beforeLocal() throws Exception { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(2); + PowerMockito.when(Const.MAX_NUM_TAGS()).thenReturn((short) 8); + } + +} diff --git a/test/core/TestTsdbQueryQueries.java b/test/core/TestTsdbQueryQueries.java new file mode 100644 index 0000000000..977e72e150 --- /dev/null +++ b/test/core/TestTsdbQueryQueries.java @@ -0,0 +1,1782 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import net.opentsdb.query.filter.TagVFilter; +import net.opentsdb.query.filter.TagVLiteralOrFilter; +import net.opentsdb.rollup.RollupConfig; +import net.opentsdb.rollup.RollupInterval; +import org.hbase.async.Bytes; +import org.hbase.async.FilterList; +import org.hbase.async.FuzzyRowFilter; +import org.hbase.async.KeyRegexpFilter; +import org.hbase.async.Scanner; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +import com.google.common.collect.Lists; +import com.stumbleupon.async.Deferred; + +import net.opentsdb.storage.MockBase; +import net.opentsdb.storage.MockBase.MockScanner; +import net.opentsdb.uid.NoSuchUniqueId; +import net.opentsdb.utils.Config; + +/** + * An integration test class that makes sure our query path is up to snuff. + * This class should have tests for different data point types, rates, + * compactions, etc. Other files can cover salting, aggregation and downsampling. + */ +@RunWith(PowerMockRunner.class) +@PrepareForTest({ Scanner.class }) +public class TestTsdbQueryQueries extends BaseTsdbTest { + protected TsdbQuery query = null; + + @Before + public void beforeLocal() throws Exception { + query = new TsdbQuery(tsdb); + } + + @Test + public void runLongSingleTS() throws Exception { + storeLongTimeSeriesSeconds(true, false); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + int value = 1; + long timestamp = 1356998430000L; + verify(tag_values, times(1)).getNameAsync(TAGV_BYTES); + verify(tag_values, never()).getNameAsync(TAGV_B_BYTES); + for (DataPoint dp : dps[0]) { + assertEquals(value, dp.longValue()); + assertEquals(timestamp, dp.timestamp()); + value++; + timestamp += 30000; + } + assertEquals(300, dps[0].aggregatedSize()); + } + + @Test + public void runLongSingleTSMs() throws Exception { + storeLongTimeSeriesMs(); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + int value = 1; + long timestamp = 1356998400500L; + for (DataPoint dp : dps[0]) { + assertEquals(value, dp.longValue()); + assertEquals(timestamp, dp.timestamp()); + value++; + timestamp += 500; + } + assertEquals(300, dps[0].aggregatedSize()); + } + + @Test + public void runLongSingleTSNoData() throws Exception { + setDataPointStorage(); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + assertNotNull(dps); + assertEquals(0, dps.length); + } + + @Test + public void runLongTwoAggSum() throws Exception { + storeLongTimeSeriesSeconds(true, false); + + tags.clear(); + query.setStartTime(1356998400L); + query.setEndTime(1357041600L); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long timestamp = 1356998430000L; + for (DataPoint dp : dps[0]) { + assertEquals(301, dp.longValue()); + assertEquals(timestamp, dp.timestamp()); + timestamp += 30000; + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runLongTwoAggSumMs() throws Exception { + storeLongTimeSeriesMs(); + + tags.clear(); + query.setStartTime(1356998400L); + query.setEndTime(1357041600L); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long timestamp = 1356998400500L; + for (DataPoint dp : dps[0]) { + assertEquals(301, dp.longValue()); + assertEquals(timestamp, dp.timestamp()); + timestamp += 500; + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runLongTwoGroup() throws Exception { + storeLongTimeSeriesSeconds(true, false); + + tags.clear(); + tags.put(TAGK_STRING , "*"); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + assertMeta(dps, 1, false); + assertEquals(2, dps.length); + + int value = 1; + long timestamp = 1356998430000L; + for (DataPoint dp : dps[0]) { + assertEquals(value, dp.longValue()); + assertEquals(timestamp, dp.timestamp()); + value++; + timestamp += 30000; + } + assertEquals(300, dps[0].size()); + + value = 300; + timestamp = 1356998430000L; + for (DataPoint dp : dps[1]) { + assertEquals(value, dp.longValue()); + assertEquals(timestamp, dp.timestamp()); + value--; + timestamp += 30000; + } + assertEquals(300, dps[1].size()); + } + + @Test + public void runLongSingleTSRate() throws Exception { + storeLongTimeSeriesSeconds(true, false); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, true); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + long timestamp = 1356998460000L; + for (DataPoint dp : dps[0]) { + assertEquals(0.033F, dp.doubleValue(), 0.001); + assertEquals(timestamp, dp.timestamp()); + timestamp += 30000; + } + assertEquals(299, dps[0].size()); + } + + @Test + public void runLongSingleTSRateMs() throws Exception { + storeLongTimeSeriesMs(); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, true); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + long timestamp = 1356998401000L; + for (DataPoint dp : dps[0]) { + assertEquals(2.0F, dp.doubleValue(), 0.001); + assertEquals(timestamp, dp.timestamp()); + timestamp += 500; + } + assertEquals(299, dps[0].size()); + } + + @Test + public void runFloatSingleTS() throws Exception { + storeFloatTimeSeriesSeconds(true, false); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + double value = 1.25D; + long timestamp = 1356998430000L; + for (DataPoint dp : dps[0]) { + assertEquals(value, dp.doubleValue(), 0.001); + assertEquals(timestamp, dp.timestamp()); + value += 0.25D; + timestamp += 30000; + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runFloatSingleTSMs() throws Exception { + storeFloatTimeSeriesMs(); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + double value = 1.25D; + long timestamp = 1356998400500L; + for (DataPoint dp : dps[0]) { + assertEquals(value, dp.doubleValue(), 0.001); + assertEquals(timestamp, dp.timestamp()); + value += 0.25D; + timestamp += 500; + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runFloatTwoAggSum() throws Exception { + storeFloatTimeSeriesSeconds(true, false); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long timestamp = 1356998430000L; + for (DataPoint dp : dps[0]) { + assertEquals(76.25, dp.doubleValue(), 0.00001); + assertEquals(timestamp, dp.timestamp()); + timestamp += 30000; + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runFloatTwoAggNoneAgg() throws Exception { + storeFloatTimeSeriesSeconds(true, false); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.NONE, false); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + assertMeta(dps, 1, false); + assertEquals(2, dps.length); + + double value = 1.25D; + long timestamp = 1356998430000L; + for (DataPoint dp : dps[0]) { + assertEquals(value, dp.doubleValue(), 0.0001); + assertEquals(timestamp, dp.timestamp()); + value += 0.25D; + timestamp += 30000; + } + assertEquals(300, dps[0].size()); + + value = 75D; + timestamp = 1356998430000L; + for (DataPoint dp : dps[1]) { + assertEquals(value, dp.doubleValue(), 0.0001); + assertEquals(timestamp, dp.timestamp()); + value -= 0.25d; + timestamp += 30000; + } + assertEquals(300, dps[1].size()); + } + + @Test + public void runFloatTwoAggSumMs() throws Exception { + storeFloatTimeSeriesMs(); + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long timestamp = 1356998400500L; + for (DataPoint dp : dps[0]) { + assertEquals(76.25, dp.doubleValue(), 0.00001); + assertEquals(timestamp, dp.timestamp()); + timestamp += 500; + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runFloatTwoGroup() throws Exception { + storeFloatTimeSeriesSeconds(true, false); + final HashMap<String, String> tags = new HashMap<String, String>(1); + tags.put(TAGK_STRING , "*"); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + assertMeta(dps, 1, false); + assertEquals(2, dps.length); + + double value = 1.25D; + long timestamp = 1356998430000L; + for (DataPoint dp : dps[0]) { + assertEquals(value, dp.doubleValue(), 0.0001); + assertEquals(timestamp, dp.timestamp()); + value += 0.25D; + timestamp += 30000; + } + assertEquals(300, dps[0].size()); + + value = 75D; + timestamp = 1356998430000L; + for (DataPoint dp : dps[1]) { + assertEquals(value, dp.doubleValue(), 0.0001); + assertEquals(timestamp, dp.timestamp()); + value -= 0.25d; + timestamp += 30000; + } + assertEquals(300, dps[1].size()); + } + + @Test + public void runFloatSingleTSRate() throws Exception { + storeFloatTimeSeriesSeconds(true, false); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, true); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + long timestamp = 1356998460000L; + for (DataPoint dp : dps[0]) { + assertEquals(0.00833F, dp.doubleValue(), 0.00001); + assertEquals(timestamp, dp.timestamp()); + timestamp += 30000; + } + assertEquals(299, dps[0].size()); + } + + @Test + public void runFloatSingleTSRateMs() throws Exception { + storeFloatTimeSeriesMs(); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, true); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + long timestamp = 1356998401000L; + for (DataPoint dp : dps[0]) { + assertEquals(0.5F, dp.doubleValue(), 0.00001); + assertEquals(timestamp, dp.timestamp()); + timestamp += 500; + } + assertEquals(299, dps[0].size()); + } + + @Test + public void runFloatSingleTSCompacted() throws Exception { + storeFloatTimeSeriesSeconds(true, false); + storage.tsdbCompactAllRows(); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + long timestamp = 1356998430000L; + double value = 1.25D; + for (DataPoint dp : dps[0]) { + assertEquals(value, dp.doubleValue(), 0.001); + assertEquals(timestamp, dp.timestamp()); + value += 0.25D; + timestamp += 30000; + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runMixedSingleTS() throws Exception { + storeMixedTimeSeriesSeconds(); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.AVG, false); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + long timestamp = 1356998430000L; + double float_value = 1.25D; + int int_value = 76; + // due to aggregation, the only int that will be returned will be the very + // last value of 76 since the agg will convert every point in between to a + // double + for (DataPoint dp : dps[0]) { + if (dp.isInteger()) { + assertEquals(int_value, dp.longValue()); + int_value++; + float_value = int_value; + } else { + assertEquals(float_value, dp.doubleValue(), 0.001); + float_value += 0.25D; + } + assertEquals(timestamp, dp.timestamp()); + timestamp += 30000; + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runMixedSingleTSMsAndS() throws Exception { + storeMixedTimeSeriesMsAndS(); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.AVG, false); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + long timestamp = 1356998400500L; + double float_value = 1.25D; + int int_value = 76; + // due to aggregation, the only int that will be returned will be the very + // last value of 76 since the agg will convert every point in between to a + // double + for (DataPoint dp : dps[0]) { + if (dp.isInteger()) { + assertEquals(int_value, dp.longValue()); + int_value++; + float_value = int_value; + } else { + assertEquals(float_value, dp.doubleValue(), 0.001); + float_value += 0.25D; + } + assertEquals(timestamp, dp.timestamp()); + timestamp += 500; + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runMixedSingleTSPostCompaction() throws Exception { + storeMixedTimeSeriesSeconds(); + + final Field compact = Config.class.getDeclaredField("enable_compactions"); + compact.setAccessible(true); + compact.set(config, true); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.AVG, false); + assertNotNull(query.run()); + + // this should only compact the rows for the time series that we fetched and + // leave the others alone + + final byte[] key = + IncomingDataPoints.rowKeyTemplate(tsdb, METRIC_STRING, tags); + RowKey.prefixKeyWithSalt(key); + System.arraycopy(Bytes.fromInt(1356998400), 0, key, + Const.SALT_WIDTH() + TSDB.metrics_width(), Const.TIMESTAMP_BYTES); + assertEquals(1, storage.numColumns(key)); + System.arraycopy(Bytes.fromInt(1357002000), 0, key, + Const.SALT_WIDTH() + TSDB.metrics_width(), Const.TIMESTAMP_BYTES); + assertEquals(1, storage.numColumns(key)); + System.arraycopy(Bytes.fromInt(1357005600), 0, key, + Const.SALT_WIDTH() + TSDB.metrics_width(), Const.TIMESTAMP_BYTES); + assertEquals(1, storage.numColumns(key)); + + // run it again to verify the compacted data uncompacts properly + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + long timestamp = 1356998430000L; + double float_value = 1.25D; + int int_value = 76; + // due to aggregation, the only int that will be returned will be the very + // last value of 76 since the agg will convert every point in between to a + // double + for (DataPoint dp : dps[0]) { + if (dp.isInteger()) { + assertEquals(int_value, dp.longValue()); + int_value++; + float_value = int_value; + } else { + assertEquals(float_value, dp.doubleValue(), 0.001); + float_value += 0.25D; + } + assertEquals(timestamp, dp.timestamp()); + timestamp += 30000; + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runEndTime() throws Exception { + storeLongTimeSeriesSeconds(true, false); + + query.setStartTime(1356998400); + query.setEndTime(1357001900); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + int value = 1; + long timestamp = 1356998430000L; + for (DataPoint dp : dps[0]) { + assertEquals(value, dp.longValue()); + assertEquals(timestamp, dp.timestamp()); + value++; + timestamp += 30000; + } + assertEquals(119, dps[0].size()); + } + + @Test + public void runCompactPostQuery() throws Exception { + storeLongTimeSeriesSeconds(true, false); + + final Field compact = Config.class.getDeclaredField("enable_compactions"); + compact.setAccessible(true); + compact.set(config, true); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + // this should only compact the rows for the time series that we fetched and + // leave the others alone + final byte[] key_a = + IncomingDataPoints.rowKeyTemplate(tsdb, METRIC_STRING, tags); + RowKey.prefixKeyWithSalt(key_a); + final Map<String, String> tags_copy = new HashMap<String, String>(tags); + tags_copy.put(TAGK_STRING, TAGV_B_STRING); + final byte[] key_b = + IncomingDataPoints.rowKeyTemplate(tsdb, METRIC_STRING, tags_copy); + RowKey.prefixKeyWithSalt(key_b); + + System.arraycopy(Bytes.fromInt(1356998400), 0, key_a, + Const.SALT_WIDTH() + TSDB.metrics_width(), Const.TIMESTAMP_BYTES); + assertEquals(1, storage.numColumns(key_a)); + + System.arraycopy(Bytes.fromInt(1356998400), 0, key_b, + Const.SALT_WIDTH() + TSDB.metrics_width(), Const.TIMESTAMP_BYTES); + if (config.enable_appends()) { + assertEquals(1, storage.numColumns(key_b)); + } else { + assertEquals(119, storage.numColumns(key_b)); + } + + System.arraycopy(Bytes.fromInt(1357002000), 0, key_a, + Const.SALT_WIDTH() + TSDB.metrics_width(), Const.TIMESTAMP_BYTES); + assertEquals(1, storage.numColumns(key_a)); + + System.arraycopy(Bytes.fromInt(1357002000), 0, key_b, + Const.SALT_WIDTH() + TSDB.metrics_width(), Const.TIMESTAMP_BYTES); + if (config.enable_appends()) { + assertEquals(1, storage.numColumns(key_b)); + } else { + assertEquals(120, storage.numColumns(key_b)); + } + + System.arraycopy(Bytes.fromInt(1357005600), 0, key_a, + Const.SALT_WIDTH() + TSDB.metrics_width(), Const.TIMESTAMP_BYTES); + assertEquals(1, storage.numColumns(key_a)); + + System.arraycopy(Bytes.fromInt(1357005600), 0, key_b, + Const.SALT_WIDTH() + TSDB.metrics_width(), Const.TIMESTAMP_BYTES); + if (config.enable_appends()) { + assertEquals(1, storage.numColumns(key_b)); + } else { + assertEquals(61, storage.numColumns(key_b)); + } + + // run it again to verify the compacted data uncompacts properly + dps = query.run(); + assertMeta(dps, 0, false); + + int value = 1; + long timestamp = 1356998430000L; + for (DataPoint dp : dps[0]) { + assertEquals(value, dp.longValue()); + assertEquals(timestamp, dp.timestamp()); + value++; + timestamp += 30000; + } + assertEquals(300, dps[0].size()); + } + + @Test (expected = IllegalStateException.class) + public void runStartNotSet() throws Exception { + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + query.run(); + } + + @Test + public void runFloatAndIntSameTSNoFix() throws Exception { + tsdb.config.overrideConfig("tsd.storage.use_otsdb_timestamp", "true"); + // if a row has an integer and a float for the same timestamp, there will be + // two different qualifiers that will resolve to the same offset. With DTCS enabled + // the conflicts are auto resolved i.e. either the maximum or the minimum value of + // data is returned depending on the configuration tsd.storage.use_max_value + // If DTCS is disabled, it will fall throw the exception. + // DTCS -> Date Tiered Compaction Strategy (tsd.storage.use_otsdb_timestamp config parameter) + + storeLongTimeSeriesSeconds(true, false); + + tsdb.addPoint(METRIC_STRING, 1356998430, 42.5F, tags).joinUninterruptibly(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + + if (config.enable_appends() || config.use_otsdb_timestamp()) { + DataPoints[] dps = query.run(); + assertMeta(dps, 0, false, false); + + int value = 1; + long timestamp = 1356998430000L; + for (DataPoint dp : dps[0]) { + if (value == 1) { + // first value was replaced in the append + assertEquals(42.5, dp.doubleValue(), 0.0001); + } else { + assertEquals(value, dp.longValue()); + } + assertEquals(timestamp, dp.timestamp()); + value++; + timestamp += 30000; + } + assertEquals(300, dps[0].size()); + } + + tsdb.config.overrideConfig("tsd.storage.use_otsdb_timestamp", "false"); + try { + query.run(); + fail("Expected an IllegalDataException"); + } catch (IllegalDataException ide) { } + + } + + @Test + public void runFloatAndIntSameTSFix() throws Exception { + config.setFixDuplicates(true); + // if a row has an integer and a float for the same timestamp, there will be + // two different qualifiers that owill resolve to the same offset. This no + // longer tosses an exception, and keeps the maximum of all values + // This can also be configured via tsdb.storage.use_max_value config + storeLongTimeSeriesSeconds(true, false); + + tsdb.addPoint(METRIC_STRING, 1356998430, 42.5F, tags).joinUninterruptibly(); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + int value = 1; + long timestamp = 1356998430000L; + for (DataPoint dp : dps[0]) { + if (value == 1) { + assertEquals(42.5, dp.doubleValue(), 0.001); + } else { + assertEquals(value, dp.longValue()); + } + assertEquals(timestamp, dp.timestamp()); + value++; + timestamp += 30000; + } + assertEquals(300, dps[0].aggregatedSize()); + } + + @Test + public void multipleValuesAtSameTimestampShouldReturnMaxValueDefault() throws Exception { + tsdb.config.overrideConfig("tsd.storage.use_otsdb_timestamp", "true"); + config.setFixDuplicates(true); + // if a row has an integer and a float for the same timestamp, there will be + // two different qualifiers that will resolve to the same offset. This no + // longer tosses an exception, and keeps the maximum of all values + // This can also be configured via tsdb.storage.use_max_value config + setDataPointStorage(); + + tsdb.addPoint(METRIC_STRING, 1356998430, 69755263, tags).joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, 1356998430, 62500.52F, tags).joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, 1356998430, 2533, tags).joinUninterruptibly(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + long timestamp = 1356998430000L; + for (DataPoint dp : dps[0]) { + assertEquals(69755263, dp.longValue()); + assertEquals(timestamp, dp.timestamp()); + } + assertEquals(1, dps[0].aggregatedSize()); + } + + @Test + public void multipleValuesAtSameTimestampShouldReturnMinValueIfConfigured() throws Exception { + tsdb.config.overrideConfig("tsd.storage.use_otsdb_timestamp", "true"); + // This test explicitly configures the use_max_value as false thus when different data type values + // are written at same timestamp, the minimum of all will be provided in output + tsdb.getConfig().overrideConfig("tsd.storage.use_max_value", "false"); + config.setFixDuplicates(true); + setDataPointStorage(); + + tsdb.addPoint(METRIC_STRING, 1356998430, 69755263, tags).joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, 1356998430, 62500.52F, tags).joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, 1356998430, 2533, tags).joinUninterruptibly(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + long timestamp = 1356998430000L; + for (DataPoint dp : dps[0]) { + assertEquals(2533, dp.longValue()); + assertEquals(timestamp, dp.timestamp()); + } + assertEquals(1, dps[0].aggregatedSize()); + } + + + @Test + public void runWithAnnotation() throws Exception { + storeLongTimeSeriesSeconds(true, false); + storeAnnotation(1356998490); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false, true); + + int value = 1; + long timestamp = 1356998430000L; + for (DataPoint dp : dps[0]) { + assertEquals(value, dp.longValue()); + assertEquals(timestamp, dp.timestamp()); + value++; + timestamp += 30000; + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runWithAnnotationPostCompact() throws Exception { + storeLongTimeSeriesSeconds(true, false); + storeAnnotation(1356998490); + + final Field compact = Config.class.getDeclaredField("enable_compactions"); + compact.setAccessible(true); + compact.set(config, true); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + DataPoints[] dps = query.run(); + assertMeta(dps, 0, false, true); + + // this should only compact the rows for the time series that we fetched and + // leave the others alone + final byte[] key_a = + IncomingDataPoints.rowKeyTemplate(tsdb, METRIC_STRING, tags); + RowKey.prefixKeyWithSalt(key_a); + final Map<String, String> tags_copy = new HashMap<String, String>(tags); + tags_copy.put(TAGK_STRING, TAGV_B_STRING); + final byte[] key_b = + IncomingDataPoints.rowKeyTemplate(tsdb, METRIC_STRING, tags_copy); + RowKey.prefixKeyWithSalt(key_b); + + System.arraycopy(Bytes.fromInt(1356998400), 0, key_a, + Const.SALT_WIDTH() + TSDB.metrics_width(), Const.TIMESTAMP_BYTES); + assertEquals(2, storage.numColumns(key_a)); + + System.arraycopy(Bytes.fromInt(1356998400), 0, key_b, + Const.SALT_WIDTH() + TSDB.metrics_width(), Const.TIMESTAMP_BYTES); + if (config.enable_appends()) { + assertEquals(1, storage.numColumns(key_b)); + } else { + assertEquals(119, storage.numColumns(key_b)); + } + + System.arraycopy(Bytes.fromInt(1357002000), 0, key_a, + Const.SALT_WIDTH() + TSDB.metrics_width(), Const.TIMESTAMP_BYTES); + assertEquals(1, storage.numColumns(key_a)); + + System.arraycopy(Bytes.fromInt(1357002000), 0, key_b, + Const.SALT_WIDTH() + TSDB.metrics_width(), Const.TIMESTAMP_BYTES); + if (config.enable_appends()) { + assertEquals(1, storage.numColumns(key_b)); + } else { + assertEquals(120, storage.numColumns(key_b)); + } + + System.arraycopy(Bytes.fromInt(1357005600), 0, key_a, + Const.SALT_WIDTH() + TSDB.metrics_width(), Const.TIMESTAMP_BYTES); + assertEquals(1, storage.numColumns(key_a)); + + System.arraycopy(Bytes.fromInt(1357005600), 0, key_b, + Const.SALT_WIDTH() + TSDB.metrics_width(), Const.TIMESTAMP_BYTES); + if (config.enable_appends()) { + assertEquals(1, storage.numColumns(key_b)); + } else { + assertEquals(61, storage.numColumns(key_b)); + } + + dps = query.run(); + assertMeta(dps, 0, false, true); + + int value = 1; + long timestamp = 1356998430000L; + for (DataPoint dp : dps[0]) { + assertEquals(value, dp.longValue()); + assertEquals(timestamp, dp.timestamp()); + value++; + timestamp += 30000; + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runWithOnlyAnnotation() throws Exception { + storeLongTimeSeriesSeconds(true, false); + + // verifies that we can pickup an annotation stored all by it's lonesome + // in a row without any data + final byte[] key = + IncomingDataPoints.rowKeyTemplate(tsdb, METRIC_STRING, tags); + RowKey.prefixKeyWithSalt(key); + System.arraycopy(Bytes.fromInt(1357002000), 0, key, + Const.SALT_WIDTH() + TSDB.metrics_width(), Const.TIMESTAMP_BYTES); + storage.flushRow(key); + + storeAnnotation(1357002090); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false, true); + + int value = 1; + long timestamp = 1356998430000L; + for (DataPoint dp : dps[0]) { + assertEquals(value, dp.longValue()); + assertEquals(timestamp, dp.timestamp()); + if (timestamp == 1357001970000L) { + timestamp = 1357005600000L; + } else { + timestamp += 30000; + } + value++; + // account for the jump + if (value == 120) { + value = 240; + } + } + assertEquals(180, dps[0].size()); + } + + @Test + public void runWithSingleAnnotation() throws Exception { + setDataPointStorage(); + + // verifies that we can pickup an annotation stored all by it's lonesome + // in a row without any data + final byte[] key = + IncomingDataPoints.rowKeyTemplate(tsdb, METRIC_STRING, tags); + RowKey.prefixKeyWithSalt(key); + System.arraycopy(Bytes.fromInt(1357002000), 0, key, + Const.SALT_WIDTH() + TSDB.metrics_width(), Const.TIMESTAMP_BYTES); + storage.flushRow(key); + + storeAnnotation(1357002090); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + // TODO - apparently if you only fetch annotations, the metric and tags + // may not be set. Check this + //assertMeta(dps, 0, false, true); + assertEquals(1, dps[0].getAnnotations().size()); + assertEquals(NOTE_DESCRIPTION, dps[0].getAnnotations().get(0) + .getDescription()); + assertEquals(NOTE_NOTES, dps[0].getAnnotations().get(0).getNotes()); + assertEquals(0, dps[0].size()); + } + + @Test + public void runSingleDataPoint() throws Exception { + setDataPointStorage(); + long timestamp = 1356998410; + tsdb.addPoint(METRIC_STRING, timestamp, 42, tags).joinUninterruptibly(); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + storage.dumpToSystemOut(); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + assertEquals(1, dps[0].size()); + assertEquals(42, dps[0].longValue(0)); + assertEquals(1356998410000L, dps[0].timestamp(0)); + } + + @Test + public void runSingleDataPointWithAnnotation() throws Exception { + setDataPointStorage(); + long timestamp = 1356998410; + tsdb.addPoint(METRIC_STRING, timestamp, 42, tags).joinUninterruptibly(); + + final byte[] key = + IncomingDataPoints.rowKeyTemplate(tsdb, METRIC_STRING, tags); + RowKey.prefixKeyWithSalt(key); + System.arraycopy(Bytes.fromInt(1357002000), 0, key, + Const.SALT_WIDTH() + TSDB.metrics_width(), Const.TIMESTAMP_BYTES); + storage.flushRow(key); + + storeAnnotation(1357002090); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false, true); + + assertEquals(1, dps[0].size()); + assertEquals(42, dps[0].longValue(0)); + assertEquals(1356998410000L, dps[0].timestamp(0)); + } + + @Test + public void runTSUIDQuery() throws Exception { + storeLongTimeSeriesSeconds(true, false); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + final List<String> tsuids = new ArrayList<String>(1); + tsuids.add("000001000001000001"); + query.setTimeSeries(tsuids, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + int value = 1; + long timestamp = 1356998430000L; + for (DataPoint dp : dps[0]) { + assertEquals(value, dp.longValue()); + assertEquals(timestamp, dp.timestamp()); + value++; + timestamp += 30000; + } + assertEquals(300, dps[0].aggregatedSize()); + } + + @Test + public void runTSUIDsAggSum() throws Exception { + storeLongTimeSeriesSeconds(true, false); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + final List<String> tsuids = new ArrayList<String>(1); + tsuids.add("000001000001000001"); + tsuids.add("000001000001000002"); + query.setTimeSeries(tsuids, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long timestamp = 1356998430000L; + for (DataPoint dp : dps[0]) { + assertEquals(301, dp.longValue()); + assertEquals(timestamp, dp.timestamp()); + timestamp += 30000; + } + assertEquals(300, dps[0].size()); + } + + @Test + public void runTSUIDQueryNoData() throws Exception { + setDataPointStorage(); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + + final List<String> tsuids = new ArrayList<String>(1); + tsuids.add("000001000001000001"); + query.setTimeSeries(tsuids, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + assertNotNull(dps); + assertEquals(0, dps.length); + } + + @Test + public void runTSUIDQueryNoDataForTSUID() throws Exception { + // this doesn't throw an exception since the UIDs are only looked for when + // the query completes. + setDataPointStorage(); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + final List<String> tsuids = new ArrayList<String>(1); + tsuids.add("000001000001000005"); + query.setTimeSeries(tsuids, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + assertNotNull(dps); + assertEquals(0, dps.length); + } + + @Test (expected = NoSuchUniqueId.class) + public void runTSUIDQueryNSU() throws Exception { + when(metrics.getNameAsync(new byte[] { 0, 0, 1 })) + .thenThrow(new NoSuchUniqueId("metrics", new byte[] { 0, 0, 1 })); + storeLongTimeSeriesSeconds(true, false); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + final List<String> tsuids = new ArrayList<String>(1); + tsuids.add("000001000001000001"); + query.setTimeSeries(tsuids, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + assertNotNull(dps); + dps[0].metricName(); + } + + @Test + public void runRateCounterDefault() throws Exception { + setDataPointStorage(); + long timestamp = 1356998400; + tsdb.addPoint(METRIC_STRING, timestamp += 30, Long.MAX_VALUE - 55, tags) + .joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, timestamp += 30, Long.MAX_VALUE - 25, tags) + .joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, timestamp += 30, 5, tags).joinUninterruptibly(); + + final RateOptions ro = new RateOptions(true, Long.MAX_VALUE, 0); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, true, ro); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + timestamp = 1356998460000L; + for (DataPoint dp : dps[0]) { + assertEquals(1.0, dp.doubleValue(), 0.001); + assertEquals(timestamp, dp.timestamp()); + timestamp += 30000; + } + assertEquals(2, dps[0].size()); + } + + @Test + public void runRateCounterDefaultNoOp() throws Exception { + setDataPointStorage(); + long timestamp = 1356998400; + tsdb.addPoint(METRIC_STRING, timestamp += 30, 30, tags).joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, timestamp += 30, 60, tags).joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, timestamp += 30, 90, tags).joinUninterruptibly(); + + final RateOptions ro = new RateOptions(true, Long.MAX_VALUE, 0); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, true, ro); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + timestamp = 1356998460000L; + for (DataPoint dp : dps[0]) { + assertEquals(1.0, dp.doubleValue(), 0.001); + assertEquals(timestamp, dp.timestamp()); + timestamp += 30000; + } + assertEquals(2, dps[0].size()); + } + + @Test + public void runRateCounterMaxSet() throws Exception { + setDataPointStorage(); + long timestamp = 1356998400; + tsdb.addPoint(METRIC_STRING, timestamp += 30, 45, tags).joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, timestamp += 30, 75, tags).joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, timestamp += 30, 5, tags).joinUninterruptibly(); + + final RateOptions ro = new RateOptions(true, 100, 0); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, true, ro); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + timestamp = 1356998460000L; + for (DataPoint dp : dps[0]) { + assertEquals(1.0, dp.doubleValue(), 0.001); + assertEquals(timestamp, dp.timestamp()); + timestamp += 30000; + } + assertEquals(2, dps[0].size()); + } + + @Test + public void runRateCounterAnomally() throws Exception { + setDataPointStorage(); + long timestamp = 1356998400; + tsdb.addPoint(METRIC_STRING, timestamp += 30, 45, tags).joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, timestamp += 30, 75, tags).joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, timestamp += 30, 25, tags).joinUninterruptibly(); + + final RateOptions ro = new RateOptions(true, 10000, 35); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, true, ro); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + assertEquals(1.0, dps[0].doubleValue(0), 0.001); + assertEquals(1356998460000L, dps[0].timestamp(0)); + assertEquals(0, dps[0].doubleValue(1), 0.001); + assertEquals(1356998490000L, dps[0].timestamp(1)); + assertEquals(2, dps[0].size()); + } + + @Test + public void runRateCounterAnomallyDrop() throws Exception { + setDataPointStorage(); + long timestamp = 1356998400; + tsdb.addPoint(METRIC_STRING, timestamp += 30, 45, tags).joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, timestamp += 30, 75, tags).joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, timestamp += 30, 25, tags).joinUninterruptibly(); + tsdb.addPoint(METRIC_STRING, timestamp += 30, 55, tags).joinUninterruptibly(); + + final RateOptions ro = new RateOptions(true, 10000, 35, true); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, true, ro); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + assertEquals(1.0, dps[0].doubleValue(0), 0.001); + assertEquals(1356998460000L, dps[0].timestamp(0)); + assertEquals(1, dps[0].doubleValue(1), 0.001); + assertEquals(1356998520000L, dps[0].timestamp(1)); + assertEquals(2, dps[0].size()); + } + + @Test + public void runMultiCompact() throws Exception { + final byte[] qual1 = { 0x00, 0x17 }; + final byte[] val1 = Bytes.fromLong(1L); + final byte[] qual2 = { 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(2L); + + // 2nd compaction + final byte[] qual3 = { 0x00, 0x37 }; + final byte[] val3 = Bytes.fromLong(3L); + final byte[] qual4 = { 0x00, 0x47 }; + final byte[] val4 = Bytes.fromLong(4L); + + // 3rd compaction + final byte[] qual5 = { 0x00, 0x57 }; + final byte[] val5 = Bytes.fromLong(5L); + final byte[] qual6 = { 0x00, 0x67 }; + final byte[] val6 = Bytes.fromLong(6L); + + final byte[] key = IncomingDataPoints.rowKeyTemplate(tsdb, METRIC_STRING, tags); + RowKey.prefixKeyWithSalt(key); + System.arraycopy(Bytes.fromInt(1356998400), 0, key, + Const.SALT_WIDTH() + TSDB.metrics_width(), Const.TIMESTAMP_BYTES); + + setDataPointStorage(); + storage.addColumn(key, + MockBase.concatByteArrays(qual1, qual2), + MockBase.concatByteArrays(val1, val2, new byte[] { 0 })); + storage.addColumn(key, + MockBase.concatByteArrays(qual3, qual4), + MockBase.concatByteArrays(val3, val4, new byte[] { 0 })); + storage.addColumn(key, + MockBase.concatByteArrays(qual5, qual6), + MockBase.concatByteArrays(val5, val6, new byte[] { 0 })); + + HashMap<String, String> tags = new HashMap<String, String>(1); + tags.put(TAGK_STRING , TAGV_STRING ); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + int value = 1; + long timestamp = 1356998401000L; + for (DataPoint dp : dps[0]) { + assertEquals(value, dp.longValue()); + assertEquals(timestamp, dp.timestamp()); + value++; + timestamp += 1000; + } + assertEquals(6, dps[0].aggregatedSize()); + } + + @Test + public void runMultiCompactAndSingles() throws Exception { + final byte[] qual1 = { 0x00, 0x17 }; + final byte[] val1 = Bytes.fromLong(1L); + final byte[] qual2 = { 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(2L); + + // 2nd compaction + final byte[] qual3 = { 0x00, 0x37 }; + final byte[] val3 = Bytes.fromLong(3L); + final byte[] qual4 = { 0x00, 0x47 }; + final byte[] val4 = Bytes.fromLong(4L); + + // 3rd compaction + final byte[] qual5 = { 0x00, 0x57 }; + final byte[] val5 = Bytes.fromLong(5L); + final byte[] qual6 = { 0x00, 0x67 }; + final byte[] val6 = Bytes.fromLong(6L); + + final byte[] key = IncomingDataPoints.rowKeyTemplate(tsdb, METRIC_STRING, tags); + RowKey.prefixKeyWithSalt(key); + System.arraycopy(Bytes.fromInt(1356998400), 0, key, + Const.SALT_WIDTH() + TSDB.metrics_width(), Const.TIMESTAMP_BYTES); + + setDataPointStorage(); + storage.addColumn(key, + MockBase.concatByteArrays(qual1, qual2), + MockBase.concatByteArrays(val1, val2, new byte[] { 0 })); + storage.addColumn(key, qual3, val3); + storage.addColumn(key, qual4, val4); + storage.addColumn(key, + MockBase.concatByteArrays(qual5, qual6), + MockBase.concatByteArrays(val5, val6, new byte[] { 0 })); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + + int value = 1; + long timestamp = 1356998401000L; + for (DataPoint dp : dps[0]) { + assertEquals(value, dp.longValue()); + assertEquals(timestamp, dp.timestamp()); + value++; + timestamp += 1000; + } + assertEquals(6, dps[0].aggregatedSize()); + } + + @Test + public void runInterpolationSeconds() throws Exception { + setDataPointStorage(); + long timestamp = 1356998400; + for (int i = 1; i <= 300; i++) { + tsdb.addPoint(METRIC_STRING, timestamp += 30, i, tags) + .joinUninterruptibly(); + } + tags.clear(); + tags.put(TAGK_STRING , TAGV_B_STRING); + timestamp = 1356998415; + for (int i = 300; i > 0; i--) { + tsdb.addPoint(METRIC_STRING, timestamp += 30, i, tags) + .joinUninterruptibly(); + } + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long v = 1; + long ts = 1356998430000L; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 15000; + assertEquals(v, dp.longValue()); + + if (dp.timestamp() == 1357007400000L) { + v = 1; + } else if (v == 1 || v == 302) { + v = 301; + } else { + v = 302; + } + } + assertEquals(600, dps[0].size()); + } + + @Test + public void runInterpolationMs() throws Exception { + setDataPointStorage(); + long timestamp = 1356998400000L; + for (int i = 1; i <= 300; i++) { + tsdb.addPoint(METRIC_STRING, timestamp += 500, i, tags) + .joinUninterruptibly(); + } + tags.clear(); + tags.put(TAGK_STRING , TAGV_B_STRING ); + timestamp = 1356998400250L; + for (int i = 300; i > 0; i--) { + tsdb.addPoint(METRIC_STRING, timestamp += 500, i, tags) + .joinUninterruptibly(); + } + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + long v = 1; + long ts = 1356998400500L; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 250; + assertEquals(v, dp.longValue()); + + if (dp.timestamp() == 1356998550000L) { + v = 1; + } else if (v == 1 || v == 302) { + v = 301; + } else { + v = 302; + } + } + assertEquals(600, dps[0].size()); + } + + @Test + public void runInterpolationMsDownsampled() throws Exception { + setDataPointStorage(); + // ts = 1356998400500, v = 1 + // ts = 1356998401000, v = 2 + // ts = 1356998401500, v = 3 + // ts = 1356998402000, v = 4 + // ts = 1356998402500, v = 5 + // ... + // ts = 1356998449000, v = 98 + // ts = 1356998449500, v = 99 + // ts = 1356998450000, v = 100 + // ts = 1356998455000, v = 101 + // ts = 1356998460000, v = 102 + // ... + // ts = 1356998550000, v = 120 + long timestamp = 1356998400000L; + for (int i = 1; i <= 120; i++) { + timestamp += i <= 100 ? 500 : 5000; + tsdb.addPoint(METRIC_STRING, timestamp, i, tags) + .joinUninterruptibly(); + } + + // ts = 1356998400750, v = 300 + // ts = 1356998401250, v = 299 + // ts = 1356998401750, v = 298 + // ts = 1356998402250, v = 297 + // ts = 1356998402750, v = 296 + // ... + // ts = 1356998549250, v = 3 + // ts = 1356998549750, v = 2 + // ts = 1356998550250, v = 1 + tags.clear(); + tags.put(TAGK_STRING , TAGV_B_STRING); + timestamp = 1356998400250L; + for (int i = 300; i > 0; i--) { + tsdb.addPoint(METRIC_STRING, timestamp += 500, i, tags) + .joinUninterruptibly(); + } + + tags.clear(); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + query.downsample(1000, Aggregators.SUM); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, true); + + // TS1 in intervals = (1), (2,3), (4,5) ... (98,99), 100, (), (), (), (), + // (101), ... (120) + // TS2 in intervals = (300), (299,298), (297,296), ... (203, 202) ... + // (3,2), (1) + // TS1 downsample = 1, 5, 9, ... 197, 100, _, _, _, _, 101, ... 120 + // TS1 interpolation = 1, 5, ... 197, 100, 100.2, 100.4, 100.6, 100.8, 101, + // ... 119.6, 119.8, 120 + // TS2 downsample = 300, 597, 593, ... 405, 401, ... 5, 1 + // TS1 + TS2 = 301, 602, 602, ... 501, 497.2, ... 124.8, 121 + int i = 0; + long ts = 1356998400000L; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + ts += 1000; + if (i == 0) { + assertEquals(301, dp.doubleValue(), 0.0000001); + } else if (i < 50) { + // TS1 = i * 2 + i * 2 + 1 + // TS2 = (300 - i * 2 + 1) + (300 - i * 2) + // TS1 + TS2 = 602 + assertEquals(602, dp.doubleValue(), 0.0000001); + } else { + // TS1 = 100 + (i - 50) * 0.2 + // TS2 = (300 - i * 2 + 1) + (300 - i * 2) + // TS1 + TS2 = 701 + (i - 50) * 0.2 - i * 4 + double value = 701 + (i - 50) * 0.2 - i * 4; + assertEquals(value, dp.doubleValue(), 0.0000001); + } + ++i; + } + assertEquals(151, dps[0].size()); + } + + @Test + public void runRegexp() throws Exception { + storeLongTimeSeriesSeconds(true, false); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + tags.clear(); + tags.put("host", "regexp(web01)"); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + assertMeta(dps, 0, false); + verify(tag_values, atLeast(1)).getNameAsync(TAGV_BYTES); + verify(tag_values, atLeast(1)).getNameAsync(TAGV_B_BYTES); + int value = 1; + long timestamp = 1356998430000L; + for (DataPoint dp : dps[0]) { + assertEquals(value, dp.longValue()); + assertEquals(timestamp, dp.timestamp()); + value++; + timestamp += 30000; + } + assertEquals(300, dps[0].aggregatedSize()); + } + + @Test + public void runRegexpNoMatch() throws Exception { + storeLongTimeSeriesSeconds(true, false); + + query.setStartTime(1356998400); + query.setEndTime(1357041600); + tags.clear(); + tags.put("host", "regexp(dbsvr.*)"); + query.setTimeSeries(METRIC_STRING, tags, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + verify(tag_values, atLeast(1)).getNameAsync(TAGV_BYTES); + verify(tag_values, atLeast(1)).getNameAsync(TAGV_B_BYTES); + assertEquals(0, dps.length); + } + @Test + public void runRollupFiltering() throws Exception { + storeLongTimeSeriesSeconds(false, false); + final List<byte[]> families = new ArrayList<byte[]>(); + families.add("t".getBytes(MockBase.ASCII())); + storage.addTable("tsdb-agg".getBytes(), families); + setupGroupByTagValues(); + long start_timestamp = 1559347200L; + + + RollupInterval defaultInterval = RollupInterval.builder() + .setTable("tsdb") + .setPreAggregationTable("tsdb-agg") + .setInterval("1m") + .setRowSpan("1h") + .build(); + + RollupInterval rollupInterval = RollupInterval.builder() + .setTable("tsdb-agg") + .setPreAggregationTable("tsdb-agg") + .setInterval("1h") + .setRowSpan("1d") + .build(); + Whitebox.setInternalState(tsdb, "default_interval", defaultInterval); + + RollupConfig rollupConfig = RollupConfig.builder().setAggregationIds(new HashMap<String, Integer>() {{ + put("sum", 0); + put("count", 1); + put("min", 2); + put("max", 3); + put("avg", 4); + }}).setIntervals(Arrays.asList(defaultInterval, rollupInterval)).build(); + + Whitebox.setInternalState(tsdb, "default_interval", defaultInterval); + Whitebox.setInternalState(tsdb,"rollup_config", rollupConfig); + + this.tsdb.addAggregatePoint(METRIC_STRING, start_timestamp, 42L, new HashMap<String, String>() {{ put("host", "web01");}}, false, "1h", "sum", null); + this.tsdb.addAggregatePoint(METRIC_STRING, start_timestamp, 42L, new HashMap<String, String>() {{ put("host", "web02");}}, false, "1h", "sum", null); + this.tsdb.addAggregatePoint(METRIC_STRING, start_timestamp, 42L, new HashMap<String, String>() {{ put("host", "web01");}}, false, "1h", "count", null); + this.tsdb.addAggregatePoint(METRIC_STRING, start_timestamp, 42L, new HashMap<String, String>() {{ put("host", "web02");}}, false, "1h", "count", null); + + TSQuery ts_query = new TSQuery(); + ts_query.setStart("1559343600"); + ts_query.setEnd("1559350800"); + + final TSSubQuery sub = new TSSubQuery(); + sub.setMetric(METRIC_STRING); + sub.setAggregator("sum"); + sub.setDownsample("1h-sum"); + sub.setFilters(Lists.<TagVFilter>newArrayList(new TagVLiteralOrFilter("host", TAGV_STRING))); + + ts_query.setQueries(Arrays.asList(sub)); + ts_query.validateAndSetQuery(); + query.configureFromQuery(ts_query, 0); + + final DataPoints[] dps = query.run(); + assertEquals(1, dps.length); + assertEquals(1, dps[0].aggregatedSize()); + assertEquals(METRIC_STRING, dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertEquals(TAGV_STRING, dps[0].getTags().get(TAGK_STRING)); + + long ts = start_timestamp * 1000; + final DataPoint dp = dps[0].iterator().next(); + assertEquals(42, dp.doubleValue(), 0); + assertEquals(ts, dp.timestamp()); + assertEquals(1, dps[0].size()); + } + + @Test + public void runPreAggregate() throws Exception { + storeLongTimeSeriesSeconds(false, false); + final List<byte[]> families = new ArrayList<byte[]>(); + families.add("t".getBytes(MockBase.ASCII())); + storage.addTable("tsdb-agg".getBytes(), families); + setupGroupByTagValues(); + long start_timestamp = 1356998400L; + Whitebox.setInternalState(tsdb, "agg_tag_key", + config.getString("tsd.rollups.agg_tag_key")); + Whitebox.setInternalState(tsdb, "raw_agg_tag_value", + config.getString("tsd.rollups.raw_agg_tag_value")); + Whitebox.setInternalState(tsdb, "default_interval", + RollupInterval.builder() + .setTable("tsdb") + .setPreAggregationTable("tsdb-agg") + .setInterval("1m") + .setRowSpan("1h") + .build()); + + tsdb.addAggregatePoint(METRIC_STRING, start_timestamp, 42L, tags, true, null, + null, "SUM"); + + tags.put(config.getString("tsd.rollups.agg_tag_key"), "SUM"); + TSQuery ts_query = new TSQuery(); + ts_query.setStart("1356998400"); + ts_query.setEnd("1357041600"); + + final TSSubQuery sub = new TSSubQuery(); + sub.setMetric(METRIC_STRING); + sub.setTags(new HashMap<String, String>(tags)); + sub.setAggregator("sum"); + + ts_query.setQueries(Arrays.asList(sub)); + ts_query.validateAndSetQuery(); + query.configureFromQuery(ts_query, 0); + + final DataPoints[] dps = query.run(); + assertEquals(1, dps.length); + assertEquals(METRIC_STRING, dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertEquals(TAGV_STRING, dps[0].getTags().get(TAGK_STRING)); + + long ts = start_timestamp * 1000; + final DataPoint dp = dps[0].iterator().next(); + assertTrue(dp.isInteger()); + assertEquals(42, dp.longValue()); + assertEquals(ts, dp.timestamp()); + assertEquals(1, dps[0].size()); + } + + @Test + public void filterExplicitTagsOK() throws Exception { + tsdb.getConfig().overrideConfig("tsd.query.enable_fuzzy", "true"); + storeLongTimeSeriesSeconds(true, false); + HashMap<String, String> tags = new HashMap<String, String>(1); + tags.put("host", "web01"); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setExplicitTags(true); + query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + + assertNotNull(dps); + assertEquals("sys.cpu.user", dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertEquals("web01", dps[0].getTags().get("host")); + + int value = 1; + for (DataPoint dp : dps[0]) { + assertEquals(value, dp.longValue()); + value++; + } + assertEquals(300, dps[0].aggregatedSize()); + // assert fuzzy + for (final MockScanner scanner : storage.getScanners()) { + assertTrue(scanner.getFilter() instanceof FilterList); + FilterList filter_list = (FilterList) scanner.getFilter(); + assertEquals(2, filter_list.size()); + assertTrue(filter_list.filters().get(0) instanceof FuzzyRowFilter); + assertTrue(filter_list.filters().get(1) instanceof KeyRegexpFilter); + } + } + + @Test + public void filterExplicitTagsGroupByOK() throws Exception { + tsdb.getConfig().overrideConfig("tsd.query.enable_fuzzy", "true"); + storeLongTimeSeriesSeconds(true, false); + HashMap<String, String> tags = new HashMap<String, String>(1); + tags.put("host", "*"); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setExplicitTags(true); + query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + + assertNotNull(dps); + assertEquals("sys.cpu.user", dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertEquals("web01", dps[0].getTags().get("host")); + + int value = 1; + for (DataPoint dp : dps[0]) { + assertEquals(value, dp.longValue()); + value++; + } + assertEquals(300, dps[0].aggregatedSize()); + // assert fuzzy + for (final MockScanner scanner : storage.getScanners()) { + FilterList filter_list = (FilterList) scanner.getFilter(); + assertEquals(2, filter_list.size()); + assertTrue(filter_list.filters().get(0) instanceof FuzzyRowFilter); + assertTrue(filter_list.filters().get(1) instanceof KeyRegexpFilter); + } + } + + @Test + public void filterExplicitTagsMissing() throws Exception { + tsdb.getConfig().overrideConfig("tsd.query.enable_fuzzy", "true"); + when(tag_names.getIdAsync("colo")) + .thenReturn(Deferred.fromResult(new byte[] { 0, 0, 0, 4 })); + when(tag_values.getIdAsync("lga")) + .thenReturn(Deferred.fromResult(new byte[] { 0, 0, 0, 4 })); + storeLongTimeSeriesSeconds(true, false); + HashMap<String, String> tags = new HashMap<String, String>(1); + tags.put("host", "web01"); + tags.put("colo", "lga"); + query.setStartTime(1356998400); + query.setEndTime(1357041600); + query.setExplicitTags(true); + query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false); + + final DataPoints[] dps = query.run(); + + assertNotNull(dps); + assertEquals(0, dps.length); + // assert fuzzy + for (final MockScanner scanner : storage.getScanners()) { + assertTrue(scanner.getFilter() instanceof FilterList); + FilterList filter_list = (FilterList) scanner.getFilter(); + assertEquals(2, filter_list.size()); + assertTrue(filter_list.filters().get(0) instanceof FuzzyRowFilter); + assertTrue(filter_list.filters().get(1) instanceof KeyRegexpFilter); + } + } + +} diff --git a/test/core/TestTsdbQueryRollup.java b/test/core/TestTsdbQueryRollup.java new file mode 100644 index 0000000000..7e93c7ef2f --- /dev/null +++ b/test/core/TestTsdbQueryRollup.java @@ -0,0 +1,1012 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2013 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.ArrayList; +import java.util.List; + +import net.opentsdb.rollup.RollupConfig; +import net.opentsdb.rollup.RollupInterval; +import net.opentsdb.storage.MockBase; +import net.opentsdb.uid.UniqueId; +import net.opentsdb.utils.Config; + +import org.hbase.async.KeyValue; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +@RunWith(PowerMockRunner.class) +//"Classloader hell"... It's real. Tell PowerMock to ignore these classes +//because they fiddle with the class loader. We don't test them anyway. +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ RowSeq.class, TSDB.class, UniqueId.class, KeyValue.class, +Config.class, RowKey.class }) +public class TestTsdbQueryRollup extends BaseTsdbTest { + protected final static byte[] FAMILY = "t".getBytes(MockBase.ASCII()); + protected TsdbQuery query = null; + protected RollupConfig rollup_config; + protected Map<String, String> tags2; + protected TSQuery ts_query; + + @Before + public void beforeLocal() throws Exception { + storeLongTimeSeriesSeconds(false, false); + final List<byte[]> families = new ArrayList<byte[]>(); + families.add(FAMILY); + + storage.addTable("tsdb-rollup-10m".getBytes(), families); + storage.addTable("tsdb-rollup-agg-10m".getBytes(), families); + storage.addTable("tsdb-rollup-1h".getBytes(), families); + storage.addTable("tsdb-rollup-agg-1h".getBytes(), families); + storage.addTable("tsdb-rollup-1d".getBytes(), families); + storage.addTable("tsdb-rollup-agg-1d".getBytes(), families); + + query = new TsdbQuery(tsdb); + tags2 = new HashMap<String, String>(1); + tags2.put(TAGK_STRING, TAGV_B_STRING); + + rollup_config = RollupConfig.builder() + .addAggregationId("sum", 0) + .addAggregationId("count", 1) + .addAggregationId("max", 2) + .addAggregationId("min", 3) + .addInterval(RollupInterval.builder() + .setTable("tsdb-rollup-10m") + .setPreAggregationTable("tsdb-rollup-agg-10m") + .setInterval("10m") + .setRowSpan("6h")) + .addInterval(RollupInterval.builder() + .setTable("tsdb-rollup-1h") + .setPreAggregationTable("tsdb-rollup-agg-1h") + .setInterval("1h") + .setRowSpan("1d")) + .addInterval(RollupInterval.builder() + .setTable("tsdb-rollup-1d") + .setPreAggregationTable("tsdb-rollup-agg-1d") + .setInterval("1d") + .setRowSpan("1n")) + .build(); + Whitebox.setInternalState(tsdb, "rollup_config", rollup_config); + } + + // This test shows us falling back to raw data if the requested downsample + // interval doesn't match a configured rollup + @Test + public void run15mSumLongSingleTS() throws Exception { + final RollupInterval interval = rollup_config.getRollupInterval("10m"); + final Aggregator aggr = Aggregators.SUM; + long start_timestamp = 1356998400L; + long end_timestamp = 1357041600L; + storeLongRollup(start_timestamp, end_timestamp, false, false, + interval, aggr); + + // 15 minutes down sampling that falls back to raw data + final int time_interval = 15 * 60 * 1000; + setQuery("15m", aggr, tags, aggr); + query.configureFromQuery(ts_query, 0); + + final DataPoints[] dps = query.run(); + assertEquals(1, dps.length); + assertEquals(METRIC_STRING, dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertEquals(TAGV_STRING, dps[0].getTags().get(TAGK_STRING)); + + long ts = start_timestamp * 1000; + double value = 435; + for (DataPoint dp : dps[0]) { + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.doubleValue(), 0.00001); + if (value >= 8535.0) { + value = 300; // last dp all by it's lonesom + } else { + value += 900; + } + ts += time_interval; + } + assertEquals(11, dps[0].size()); + } + + // In this case we're downsampling rolled up values + @Test + public void run30mSumLongSingleTS() throws Exception { + final RollupInterval interval = rollup_config.getRollupInterval("10m"); + final Aggregator aggr = Aggregators.SUM; + long start_timestamp = 1356998400L; + long end_timestamp = 1357041599L; + storeLongRollup(start_timestamp, end_timestamp, false, false, + interval, aggr); + + setQuery("30m", aggr, tags, aggr); + query.configureFromQuery(ts_query, 0); + + DataPoints[] dps = query.run(); + assertEquals(1, dps.length); + assertEquals(METRIC_STRING, dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertEquals(TAGV_STRING, dps[0].getTags().get(TAGK_STRING)); + + double value = 3600; + long ts = start_timestamp * 1000; + + for (DataPoint dp : dps[0]) { + assertEquals(value, dp.doubleValue(), 0); + assertEquals(ts, dp.timestamp()); + value += 5400; + ts += (interval.getIntervalSeconds() * 3) * 1000; + } + assertEquals(24, dps[0].size()); + } + + // Zimsum == SUM when comparing qualifiers + @Test + public void run10mZimSumLongSingleTS() throws Exception { + final RollupInterval interval = rollup_config.getRollupInterval("10m"); + Aggregator aggr = Aggregators.SUM; // still have to write as sum + long start_timestamp = 1356998400L; + long end_timestamp = 1357041599L; + storeLongRollup(start_timestamp, end_timestamp, false, false, + interval, aggr); + + aggr = Aggregators.ZIMSUM; + setQuery(interval.getInterval(), aggr, tags, aggr); + query.configureFromQuery(ts_query, 0); + + final DataPoints[] dps = query.run(); + assertEquals(1, dps.length); + assertEquals(METRIC_STRING, dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertEquals(TAGV_STRING, dps[0].getTags().get(TAGK_STRING)); + + int i = 600; + long ts = start_timestamp * 1000; + + for (final DataPoint dp : dps[0]) { + assertFalse(dp.isInteger()); + assertEquals(i, dp.doubleValue(), 0.0001); + assertEquals(ts, dp.timestamp()); + ts += interval.getIntervalSeconds() * 1000; + i += interval.getIntervalSeconds(); + } + assertEquals(72, dps[0].size()); + } + + @Test + public void run10mMaxLongSingleTSNotFound() throws Exception { + final RollupInterval interval = rollup_config.getRollupInterval("10m"); + Aggregator aggr = Aggregators.SUM; + long start_timestamp = 1356998400L; + long end_timestamp = 1357041599L; + storeLongRollup(start_timestamp, end_timestamp, false, false, + interval, aggr); + + aggr = Aggregators.MAX; + setQuery(interval.getInterval(), aggr, tags, aggr); + query.configureFromQuery(ts_query, 0); + + final DataPoints[] dps = query.run(); + assertEquals(0, dps.length); + } + + @Test + public void run10mSumLongSingleTS() throws Exception { + final RollupInterval interval = rollup_config.getRollupInterval("10m"); + final Aggregator aggr = Aggregators.SUM; + long start_timestamp = 1356998400L; + long end_timestamp = 1357041600L; + storeLongRollup(1356998400L, end_timestamp, false, false, interval, aggr); + + final int time_interval = interval.getIntervalSeconds(); + setQuery(interval.getInterval(), aggr, tags, aggr); + query.configureFromQuery(ts_query, 0); + + final DataPoints[] dps = query.run(); + assertEquals(1, dps.length); + assertEquals(METRIC_STRING, dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertEquals(TAGV_STRING, dps[0].getTags().get(TAGK_STRING)); + + int i = 600; + long ts = start_timestamp * 1000; + + for (final DataPoint dp : dps[0]) { + assertFalse(dp.isInteger()); + assertEquals(i, dp.doubleValue(), 0.0001); + assertEquals(ts, dp.timestamp()); + ts += time_interval * 1000; + i += time_interval; + } + assertEquals(73, dps[0].size()); + } + + @Test (expected = IllegalArgumentException.class) + public void run10mSumLongSingleTSInMS() throws Exception { + RollupInterval ten_min_interval = rollup_config.getRollupInterval("10m"); + Aggregator aggr = Aggregators.SUM; + long start_timestamp = 1356998400000L; + + //rollup doesn't accept timestamps in milliseconds + tsdb.addAggregatePoint(METRIC_STRING, start_timestamp, + 0, tags, false, ten_min_interval.getInterval(), + aggr.toString(), null).joinUninterruptibly(); + } + + @Test + public void run10mSumLongSingleTSRate() throws Exception { + final RollupInterval interval = rollup_config.getRollupInterval("10m"); + final Aggregator aggr = Aggregators.SUM; + final long start_timestamp = 1356998400; + final long end_timestamp = 1357041600; + + storeLongRollup(start_timestamp, end_timestamp, false, false, + interval, aggr); + + setQuery(interval.getInterval(), aggr, tags, aggr); + ts_query.getQueries().get(0).setRate(true); + query.configureFromQuery(ts_query, 0); + final DataPoints[] dps = query.run(); + + assertNotNull(dps); + assertEquals(METRIC_STRING, dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertEquals(TAGV_STRING, dps[0].getTags().get(TAGK_STRING)); + + long expected_timestamp = (start_timestamp + interval.getIntervalSeconds()) * 1000; + for (DataPoint dp : dps[0]) { + assertEquals(1.0F, dp.doubleValue(), 0.00001); + assertEquals(expected_timestamp, dp.timestamp()); + expected_timestamp += interval.getIntervalSeconds() * 1000; + } + + assertEquals(72, dps[0].size()); + } + + @Test + public void run10mSumFloatSingleTS() throws Exception { + final RollupInterval interval = rollup_config.getRollupInterval("10m"); + final Aggregator aggr = Aggregators.SUM; + long start_timestamp = 1356998400L; + final long end_timestamp = 1357041600; + storeFloatRollup(start_timestamp, end_timestamp, true, false, interval, aggr); + + setQuery(interval.getInterval(), aggr, tags, aggr); + query.configureFromQuery(ts_query, 0); + final DataPoints[] dps = query.run(); + + assertEquals(1, dps.length); + assertEquals(METRIC_STRING, dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertEquals(TAGV_STRING, dps[0].getTags().get(TAGK_STRING)); + + double value = 600.5F; + long expected_timestamp = start_timestamp * 1000; + + for (DataPoint dp : dps[0]) { + assertEquals(value, dp.doubleValue(), 0.00001); + assertEquals(expected_timestamp, dp.timestamp()); + value += interval.getIntervalSeconds(); + expected_timestamp += interval.getIntervalSeconds() * 1000; + } + + assertEquals(73, dps[0].size()); + } + + @Test + public void run10mSumFloatSingleTSRate() throws Exception { + final RollupInterval interval = rollup_config.getRollupInterval("10m"); + final Aggregator aggr = Aggregators.SUM; + final long start_timestamp = 1356998400; + final long end_timestamp = 1357041600; + + storeFloatRollup(start_timestamp, end_timestamp, false, false, + interval, aggr); + + setQuery(interval.getInterval(), aggr, tags, aggr); + ts_query.getQueries().get(0).setRate(true); + query.configureFromQuery(ts_query, 0); + final DataPoints[] dps = query.run(); + + assertEquals(1, dps.length); + assertEquals(METRIC_STRING, dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertEquals(TAGV_STRING, dps[0].getTags().get(TAGK_STRING)); + + long expected_timestamp = (start_timestamp + + interval.getIntervalSeconds()) * 1000; + for (DataPoint dp : dps[0]) { + assertEquals(1.0F, dp.doubleValue(), 0.00001); + assertEquals(expected_timestamp, dp.timestamp()); + expected_timestamp += interval.getIntervalSeconds() * 1000; + } + assertEquals(72, dps[0].size()); + } + + // Make sure filtering on time series still operates + @Test + public void run10mSumLongDoubleTSFilter() throws Exception { + final RollupInterval interval = rollup_config.getRollupInterval("10m"); + final Aggregator aggr = Aggregators.SUM; + long start_timestamp = 1356998400L; + long end_timestamp = 1357041600L; + storeLongRollup(1356998400L, end_timestamp, true, false, interval, aggr); + + final int time_interval = interval.getIntervalSeconds(); + setQuery(interval.getInterval(), aggr, tags, aggr); + query.configureFromQuery(ts_query, 0); + + final DataPoints[] dps = query.run(); + assertEquals(1, dps.length); + assertEquals(METRIC_STRING, dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertEquals(TAGV_STRING, dps[0].getTags().get(TAGK_STRING)); + + int i = 600; + long ts = start_timestamp * 1000; + + for (final DataPoint dp : dps[0]) { + assertFalse(dp.isInteger()); + assertEquals(i, dp.doubleValue(), 0.0001); + assertEquals(ts, dp.timestamp()); + ts += time_interval * 1000; + i += time_interval; + } + assertEquals(73, dps[0].size()); + } + + @Test + public void run10mSumLongDoubleTS() throws Exception { + final RollupInterval interval = rollup_config.getRollupInterval("10m"); + final Aggregator aggr = Aggregators.SUM; + long start_timestamp = 1356998400L; + long end_timestamp = 1357041600L; + storeLongRollup(1356998400L, end_timestamp, true, false, interval, aggr); + + final int time_interval = interval.getIntervalSeconds(); + tags.clear(); + setQuery(interval.getInterval(), aggr, tags, aggr); + ts_query.getQueries().get(0).getFilters().clear(); + ts_query.validateAndSetQuery(); + query.configureFromQuery(ts_query, 0); + + final DataPoints[] dps = query.run(); + assertEquals(1, dps.length); + assertEquals(METRIC_STRING, dps[0].metricName()); + assertEquals(TAGK_STRING, dps[0].getAggregatedTags().get(0)); + assertNull(dps[0].getAnnotations()); + assertNull(dps[0].getTags().get(TAGK_STRING)); + + long ts = start_timestamp * 1000; + + for (final DataPoint dp : dps[0]) { + assertEquals(43800, dp.doubleValue(), 0.0001); + assertEquals(ts, dp.timestamp()); + ts += time_interval * 1000; + } + assertEquals(73, dps[0].size()); + } + + // Make sure other aggregates don't polute our results + @Test + public void run10mSumLongDoubleTSFilterOtherAggs() throws Exception { + final RollupInterval interval = rollup_config.getRollupInterval("10m"); + final Aggregator aggr = Aggregators.SUM; + long start_timestamp = 1356998400L; + long end_timestamp = 1357041600L; + storeLongRollup(1356998400L, end_timestamp, true, false, interval, aggr); + storeLongRollup(1356998400L, end_timestamp, true, false, interval, + Aggregators.MAX); + storeLongRollup(1356998400L, end_timestamp, true, false, interval, + Aggregators.MIN); + + final int time_interval = interval.getIntervalSeconds(); + setQuery(interval.getInterval(), aggr, tags, aggr); + query.configureFromQuery(ts_query, 0); + + final DataPoints[] dps = query.run(); + assertEquals(1, dps.length); + assertEquals(METRIC_STRING, dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertEquals(TAGV_STRING, dps[0].getTags().get(TAGK_STRING)); + + int i = 600; + long ts = start_timestamp * 1000; + + for (final DataPoint dp : dps[0]) { + assertFalse(dp.isInteger()); + assertEquals(i, dp.doubleValue(), 0.0001); + assertEquals(ts, dp.timestamp()); + ts += time_interval * 1000; + i += time_interval; + } + assertEquals(73, dps[0].size()); + } + + @Test + public void run10mMaxLongSingleTS() throws Exception { + final RollupInterval interval = rollup_config.getRollupInterval("10m"); + final Aggregator aggr = Aggregators.MAX; + long start_timestamp = 1356998400L; + long end_timestamp = 1357041600L; + storeLongRollup(1356998400L, end_timestamp, false, false, interval, aggr); + + final int time_interval = interval.getIntervalSeconds(); + setQuery(interval.getInterval(), aggr, tags, aggr); + query.configureFromQuery(ts_query, 0); + + final DataPoints[] dps = query.run(); + assertEquals(1, dps.length); + assertEquals(METRIC_STRING, dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertEquals(TAGV_STRING, dps[0].getTags().get(TAGK_STRING)); + + int i = 600; + long ts = start_timestamp * 1000; + + for (final DataPoint dp : dps[0]) { + assertFalse(dp.isInteger()); + assertEquals(i, dp.doubleValue(), 0.0001); + assertEquals(ts, dp.timestamp()); + ts += time_interval * 1000; + i += time_interval; + } + assertEquals(73, dps[0].size()); + } + + @Test + public void run10mMinLongSingleTS() throws Exception { + final RollupInterval interval = rollup_config.getRollupInterval("10m"); + final Aggregator aggr = Aggregators.MIN; + long start_timestamp = 1356998400L; + long end_timestamp = 1357041600L; + storeLongRollup(1356998400L, end_timestamp, false, false, interval, aggr); + + final int time_interval = interval.getIntervalSeconds(); + setQuery(interval.getInterval(), aggr, tags, aggr); + query.configureFromQuery(ts_query, 0); + + final DataPoints[] dps = query.run(); + assertEquals(1, dps.length); + assertEquals(METRIC_STRING, dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertEquals(TAGV_STRING, dps[0].getTags().get(TAGK_STRING)); + + int i = 600; + long ts = start_timestamp * 1000; + + for (final DataPoint dp : dps[0]) { + assertFalse(dp.isInteger()); + assertEquals(i, dp.doubleValue(), 0.0001); + assertEquals(ts, dp.timestamp()); + ts += time_interval * 1000; + i += time_interval; + } + assertEquals(73, dps[0].size()); + } + + @Test + public void run10mAvgLongSingleTS() throws Exception { + final RollupInterval interval = rollup_config.getRollupInterval("10m"); + Aggregator aggr = Aggregators.SUM; + long start_timestamp = 1356998400L; + long end_timestamp = 1357041600L; + storeLongRollup(start_timestamp, end_timestamp, false, false, interval, aggr); + storeCount(start_timestamp, end_timestamp, false, false, interval, 2); + + aggr = Aggregators.AVG; + setQuery(interval.getInterval(), aggr, tags, aggr); + query.configureFromQuery(ts_query, 0); + + final DataPoints[] dps = query.run(); + assertEquals(1, dps.length); + assertEquals(METRIC_STRING, dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertEquals(TAGV_STRING, dps[0].getTags().get(TAGK_STRING)); + + int i = 300; + long ts = start_timestamp * 1000; + + for (final DataPoint dp : dps[0]) { + assertFalse(dp.isInteger()); + assertEquals(i, dp.doubleValue(), 0.0001); + assertEquals(ts, dp.timestamp()); + ts += interval.getIntervalSeconds() * 1000; + i += interval.getIntervalSeconds() / 2; + } + assertEquals(73, dps[0].size()); + } + + @Test + public void run10mAvgLongSingleTSMissingCount() throws Exception { + final RollupInterval interval = rollup_config.getRollupInterval("10m"); + Aggregator aggr = Aggregators.SUM; + long start_timestamp = 1356998400L; + long end_timestamp = 1357041600L; + storeLongRollup(start_timestamp, end_timestamp, false, false, interval, aggr); + + aggr = Aggregators.AVG; + setQuery(interval.getInterval(), aggr, tags, aggr); + query.configureFromQuery(ts_query, 0); + + final DataPoints[] dps = query.run(); + assertEquals(1, dps.length); + assertEquals("", dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertNull(dps[0].getTags().get(TAGK_STRING)); + assertFalse(dps[0].iterator().hasNext()); + assertEquals(0, dps[0].size()); + } + + @Test + public void run10mAvgLongSingleTSMissingSum() throws Exception { + final RollupInterval interval = rollup_config.getRollupInterval("10m"); + final Aggregator aggr = Aggregators.AVG; + long start_timestamp = 1356998400L; + long end_timestamp = 1357041600L; + storeCount(start_timestamp, end_timestamp, false, false, interval, 1); + + setQuery(interval.getInterval(), aggr, tags, aggr); + query.configureFromQuery(ts_query, 0); + + final DataPoints[] dps = query.run(); + assertEquals(1, dps.length); + assertEquals("", dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertNull(dps[0].getTags().get(TAGK_STRING)); + assertFalse(dps[0].iterator().hasNext()); + assertEquals(0, dps[0].size()); + } + + @Test + public void run10mAvgLongSingleTSMissingACount() throws Exception { + final RollupInterval interval = rollup_config.getRollupInterval("10m"); + + storePoint(1356998400, 20, Aggregators.SUM, interval); + storePoint(1356998400, 2, Aggregators.COUNT, interval); + storePoint(1356999000, 40, Aggregators.SUM, interval); + //storePoint(1356999000, 5, Aggregators.COUNT, interval); + storePoint(1356999600, 60, Aggregators.SUM, interval); + storePoint(1356999600, 3, Aggregators.COUNT, interval); + storePoint(1357000200, 80, Aggregators.SUM, interval); + storePoint(1357000200, 4, Aggregators.COUNT, interval); + + Aggregator aggr = Aggregators.AVG; + setQuery(interval.getInterval(), aggr, tags, aggr); + query.configureFromQuery(ts_query, 0); + + final DataPoints[] dps = query.run(); + assertEquals(1, dps.length); + assertEquals(METRIC_STRING, dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertEquals(TAGV_STRING, dps[0].getTags().get(TAGK_STRING)); + + final SeekableView it = dps[0].iterator(); + DataPoint dp = it.next(); + assertEquals(1356998400000L, dp.timestamp()); + assertEquals(10, dp.doubleValue(), 0.0001); + dp = it.next(); + assertEquals(1356999600000L, dp.timestamp()); + assertEquals(20, dp.doubleValue(), 0.0001); + dp = it.next(); + assertEquals(1357000200000L, dp.timestamp()); + assertEquals(20, dp.doubleValue(), 0.0001); + assertFalse(it.hasNext()); + assertEquals(3, dps[0].size()); + } + + @Test + public void run10mAvgLongSingleTSMissingASum() throws Exception { + final RollupInterval interval = rollup_config.getRollupInterval("10m"); + + storePoint(1356998400, 20, Aggregators.SUM, interval); + storePoint(1356998400, 2, Aggregators.COUNT, interval); + //storePoint(1356999000, 40, Aggregators.SUM, interval); + storePoint(1356999000, 5, Aggregators.COUNT, interval); + storePoint(1356999600, 60, Aggregators.SUM, interval); + storePoint(1356999600, 3, Aggregators.COUNT, interval); + storePoint(1357000200, 80, Aggregators.SUM, interval); + storePoint(1357000200, 4, Aggregators.COUNT, interval); + + Aggregator aggr = Aggregators.AVG; + setQuery(interval.getInterval(), aggr, tags, aggr); + query.configureFromQuery(ts_query, 0); + + final DataPoints[] dps = query.run(); + assertEquals(1, dps.length); + assertEquals(METRIC_STRING, dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertEquals(TAGV_STRING, dps[0].getTags().get(TAGK_STRING)); + + final SeekableView it = dps[0].iterator(); + DataPoint dp = it.next(); + assertEquals(1356998400000L, dp.timestamp()); + assertEquals(10, dp.doubleValue(), 0.0001); + dp = it.next(); + assertEquals(1356999600000L, dp.timestamp()); + assertEquals(20, dp.doubleValue(), 0.0001); + dp = it.next(); + assertEquals(1357000200000L, dp.timestamp()); + assertEquals(20, dp.doubleValue(), 0.0001); + assertFalse(it.hasNext()); + assertEquals(3, dps[0].size()); + } + + @Test + public void run10mAvgLongSingleTSMissingToZero() throws Exception { + final RollupInterval interval = rollup_config.getRollupInterval("10m"); + + storePoint(1356998400, 20, Aggregators.SUM, interval); + //storePoint(1356998400, 2, Aggregators.COUNT, interval); + //storePoint(1356999000, 40, Aggregators.SUM, interval); + storePoint(1356999000, 5, Aggregators.COUNT, interval); + storePoint(1356999600, 60, Aggregators.SUM, interval); + //storePoint(1356999600, 3, Aggregators.COUNT, interval); + //storePoint(1357000200, 80, Aggregators.SUM, interval); + storePoint(1357000200, 4, Aggregators.COUNT, interval); + + Aggregator aggr = Aggregators.AVG; + setQuery(interval.getInterval(), aggr, tags, aggr); + query.configureFromQuery(ts_query, 0); + + final DataPoints[] dps = query.run(); + assertEquals(1, dps.length); + assertEquals("", dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertNull(dps[0].getTags().get(TAGK_STRING)); + + final SeekableView it = dps[0].iterator(); + assertFalse(it.hasNext()); + assertEquals(0, dps[0].size()); + } + + @Test + public void run10mAvgLongSingleTSMissingToZeroOneSpan() throws Exception { + final RollupInterval interval = rollup_config.getRollupInterval("10m"); + + // For this test, a span in the middle was zero'd out + storePoint(1356998400, 20, Aggregators.SUM, interval); + storePoint(1356998400, 2, Aggregators.COUNT, interval); + storePoint(1356999000, 40, Aggregators.SUM, interval); + storePoint(1356999000, 5, Aggregators.COUNT, interval); + + storePoint(1357084800, 60, Aggregators.SUM, interval); + //storePoint(1357084800, 3, Aggregators.COUNT, interval); + //storePoint(1357085400, 80, Aggregators.SUM, interval); + storePoint(1357085400, 4, Aggregators.COUNT, interval); + + storePoint(1357171200, 90, Aggregators.SUM, interval); + storePoint(1357171200, 3, Aggregators.COUNT, interval); + storePoint(1357171800, 100, Aggregators.SUM, interval); + storePoint(1357171800, 5, Aggregators.COUNT, interval); + + Aggregator aggr = Aggregators.AVG; + setQuery(interval.getInterval(), aggr, tags, aggr); + ts_query.setEnd("1359590400"); + ts_query.validateAndSetQuery(); + query.configureFromQuery(ts_query, 0); + + final DataPoints[] dps = query.run(); + assertEquals(1, dps.length); + assertEquals(METRIC_STRING, dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertEquals(TAGV_STRING, dps[0].getTags().get(TAGK_STRING)); + + final SeekableView it = dps[0].iterator(); + DataPoint dp = it.next(); + assertEquals(1356998400000L, dp.timestamp()); + assertEquals(10, dp.doubleValue(), 0.0001); + dp = it.next(); + assertEquals(1356999000000L, dp.timestamp()); + assertEquals(8, dp.doubleValue(), 0.0001); + dp = it.next(); + assertEquals(1357171200000L, dp.timestamp()); + assertEquals(30, dp.doubleValue(), 0.0001); + dp = it.next(); + assertEquals(1357171800000L, dp.timestamp()); + assertEquals(20, dp.doubleValue(), 0.0001); + assertFalse(it.hasNext()); + assertEquals(4, dps[0].size()); + } + + @Test + public void run10mAvgLongSingleTSMissingToZeroBookends() throws Exception { + final RollupInterval interval = rollup_config.getRollupInterval("10m"); + + // For this test, the spans on either side are zero'd + storePoint(1356998400, 20, Aggregators.SUM, interval); + //storePoint(1356998400, 2, Aggregators.COUNT, interval); + //storePoint(1356999000, 40, Aggregators.SUM, interval); + storePoint(1356999000, 5, Aggregators.COUNT, interval); + + storePoint(1357084800, 60, Aggregators.SUM, interval); + storePoint(1357084800, 3, Aggregators.COUNT, interval); + storePoint(1357085400, 80, Aggregators.SUM, interval); + storePoint(1357085400, 4, Aggregators.COUNT, interval); + + //storePoint(1357171200, 90, Aggregators.SUM, interval); + storePoint(1357171200, 3, Aggregators.COUNT, interval); + storePoint(1357171800, 100, Aggregators.SUM, interval); + //storePoint(1357171800, 5, Aggregators.COUNT, interval); + + Aggregator aggr = Aggregators.AVG; + setQuery(interval.getInterval(), aggr, tags, aggr); + ts_query.setEnd("1359590400"); + ts_query.validateAndSetQuery(); + query.configureFromQuery(ts_query, 0); + + final DataPoints[] dps = query.run(); + assertEquals(1, dps.length); + assertEquals(METRIC_STRING, dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertEquals(TAGV_STRING, dps[0].getTags().get(TAGK_STRING)); + + final SeekableView it = dps[0].iterator(); + DataPoint dp = it.next(); + assertEquals(1357084800000L, dp.timestamp()); + assertEquals(20, dp.doubleValue(), 0.0001); + dp = it.next(); + assertEquals(1357085400000L, dp.timestamp()); + assertEquals(20, dp.doubleValue(), 0.0001); + assertFalse(it.hasNext()); + assertEquals(2, dps[0].size()); + } + + @Test + public void runDupes() throws Exception { + storage.flushStorage(); + + final RollupInterval interval = rollup_config.getRollupInterval("10m"); + final Aggregator aggr = Aggregators.SUM; + + tsdb.addAggregatePoint(METRIC_STRING, 1357026600L, Integer.MAX_VALUE, tags, false, + interval.getInterval(), aggr.toString(), null).joinUninterruptibly(); + tsdb.addAggregatePoint(METRIC_STRING, 1357026600L, 42.5F, tags, false, + interval.getInterval(), aggr.toString(), null).joinUninterruptibly(); + + setQuery(interval.getInterval(), aggr, tags, aggr); + query.configureFromQuery(ts_query, 0); + try { + query.run(); + fail("Expected IllegalDataException"); + } catch (IllegalDataException e) { } + + config.setFixDuplicates(true); + DataPoints[] dps = query.run(); + + DataPoint dp = dps[0].iterator().next(); + assertEquals(1357026600000L, dp.timestamp()); + assertEquals(42.5F, dp.toDouble(), 0.0001); + } + + @Test + public void oldStringPrefix() throws Exception { + final RollupInterval interval = rollup_config.getRollupInterval("10m"); + final Aggregator aggr = Aggregators.SUM; + long start_timestamp = 1356998400000L; + + storage.addColumn("tsdb-rollup-10m".getBytes(), + getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_STRING), + "t".getBytes(), + new byte[] { 's', 'u', 'm', ':', 0, 0 }, new byte[] { 0x2A }); + + final int time_interval = interval.getIntervalSeconds(); + setQuery(interval.getInterval(), aggr, tags, aggr); + query.configureFromQuery(ts_query, 0); + + final DataPoints[] dps = query.run(); + assertEquals(1, dps.length); + assertEquals(METRIC_STRING, dps[0].metricName()); + assertTrue(dps[0].getAggregatedTags().isEmpty()); + assertNull(dps[0].getAnnotations()); + assertEquals(TAGV_STRING, dps[0].getTags().get(TAGK_STRING)); + + final DataPoint dp = dps[0].iterator().next(); + assertFalse(dp.isInteger()); + assertEquals(42, dp.doubleValue(), 0.001); + assertEquals(start_timestamp, dp.timestamp()); + assertEquals(1, dps[0].size()); + } + + // ----------------- // + // Helper functions. // + // ----------------- // + + private void storeLongRollup(final long start_timestamp, + final long end_timestamp, + final boolean two_metrics, + final boolean offset, + final RollupInterval interval, + final Aggregator aggr) throws Exception { + + // dump a bunch of rows of two metrics so that we can test filtering out + // on the metric + int time_interval = interval.getIntervalSeconds(); + long start_a = start_timestamp; + long start_b = start_timestamp + (offset ? time_interval : 0); + int i = 0; + + while (start_a <= end_timestamp) { + i += time_interval; + tsdb.addAggregatePoint(METRIC_STRING, start_a, i, tags, false, + interval.getInterval(), aggr.toString(), null).joinUninterruptibly(); + if (two_metrics) { + tsdb.addAggregatePoint(METRIC_B_STRING, start_b, i, tags, false, + interval.getInterval(), aggr.toString(), null).joinUninterruptibly(); + } + start_a += (offset ? time_interval * 2 : time_interval); + start_b += (offset ? time_interval * 2 : time_interval); + } + + // dump a parallel set but invert the values + start_a = start_timestamp; + start_b = start_timestamp + (offset ? time_interval : 0); + + while (start_a <= end_timestamp) { + i -= time_interval; + tsdb.addAggregatePoint(METRIC_STRING, start_a, i, tags2, false, + interval.getInterval(), aggr.toString(), null).joinUninterruptibly(); + if (two_metrics) { + tsdb.addAggregatePoint(METRIC_B_STRING, start_b, i, tags2, false, + interval.getInterval(), aggr.toString(), null).joinUninterruptibly(); + } + + start_a += (offset ? time_interval * 2 : time_interval); + start_b += (offset ? time_interval * 2 : time_interval); + } + } + + private void storeCount(final long start_timestamp, + final long end_timestamp, + final boolean two_metrics, + final boolean offset, + final RollupInterval interval, + final int value) throws Exception { + + // dump a bunch of rows of two metrics so that we can test filtering out + // on the metric + int time_interval = interval.getIntervalSeconds(); + long start_a = start_timestamp; + long start_b = start_timestamp + (offset ? time_interval : 0); + + while (start_a <= end_timestamp) { + tsdb.addAggregatePoint(METRIC_STRING, start_a, value, tags, false, + interval.getInterval(), Aggregators.COUNT.toString(), null) + .joinUninterruptibly(); + if (two_metrics) { + tsdb.addAggregatePoint(METRIC_B_STRING, start_b, value, tags, false, + interval.getInterval(), Aggregators.COUNT.toString(), null) + .joinUninterruptibly(); + } + start_a += (offset ? time_interval * 2 : time_interval); + start_b += (offset ? time_interval * 2 : time_interval); + } + + start_a = start_timestamp; + start_b = start_timestamp + (offset ? time_interval : 0); + + while (start_a <= end_timestamp) { + tsdb.addAggregatePoint(METRIC_STRING, start_a, value, tags2, false, + interval.getInterval(), Aggregators.COUNT.toString(), null) + .joinUninterruptibly(); + if (two_metrics) { + tsdb.addAggregatePoint(METRIC_B_STRING, start_b, value, tags2, false, + interval.getInterval(), Aggregators.COUNT.toString(), null) + .joinUninterruptibly(); + } + + start_a += (offset ? time_interval * 2 : time_interval); + start_b += (offset ? time_interval * 2 : time_interval); + } + } + + private void storeFloatRollup(final long start_timestamp, + final long end_timestamp, + final boolean two_metrics, + final boolean offset, + final RollupInterval interval, + final Aggregator aggr) throws Exception { + + // dump a bunch of rows of two metrics so that we can test filtering out + // on the metric + int time_interval = interval.getIntervalSeconds(); + long start_a = start_timestamp; + long start_b = start_timestamp + (offset ? time_interval : 0); + float i = 0.5F; + + while (start_a <= end_timestamp) { + i += time_interval; + tsdb.addAggregatePoint(METRIC_STRING, start_a, i, tags, false, + interval.getInterval(), aggr.toString(), null).joinUninterruptibly(); + + if (two_metrics) { + tsdb.addAggregatePoint(METRIC_B_STRING, start_b,i, tags, false, + interval.getInterval(), aggr.toString(), null).joinUninterruptibly(); + } + start_a += (offset ? time_interval * 2 : time_interval); + start_b += (offset ? time_interval * 2 : time_interval); + } + + // dump a parallel set but invert the values + start_a = start_timestamp; + start_b = start_timestamp + (offset ? time_interval : 0); + + while (start_a <= end_timestamp) { + i -= time_interval; + tsdb.addAggregatePoint(METRIC_STRING, start_a, i, tags2, false, + interval.getInterval(), aggr.toString(), null).joinUninterruptibly(); + if (two_metrics) { + tsdb.addAggregatePoint(METRIC_B_STRING, start_b, i, tags2, false, + interval.getInterval(), aggr.toString(), null).joinUninterruptibly(); + } + + start_a += (offset ? time_interval * 2 : time_interval); + start_b += (offset ? time_interval * 2 : time_interval); + } + } + + private void storePoint(final long ts, final long value, final Aggregator agg, + final RollupInterval interval) throws Exception { + tsdb.addAggregatePoint(METRIC_STRING, ts, value, tags, false, + interval.getInterval(), agg.toString(), null).joinUninterruptibly(); + } + + private void setQuery(final String ds_interval, final Aggregator ds_agg, + final Map<String, String> tags, final Aggregator group_by) { + ts_query = new TSQuery(); + ts_query.setStart("1356998400"); + ts_query.setEnd("1357041600"); + + final TSSubQuery sub = new TSSubQuery(); + sub.setMetric(METRIC_STRING); + sub.setDownsample(ds_interval + "-" + ds_agg); + sub.setTags(new HashMap<String, String>(tags)); + sub.setAggregator(group_by.toString()); + + ts_query.setQueries(Arrays.asList(sub)); + ts_query.validateAndSetQuery(); + } +} diff --git a/test/core/TestTsdbQueryRollupSalted.java b/test/core/TestTsdbQueryRollupSalted.java new file mode 100644 index 0000000000..0284baeaa1 --- /dev/null +++ b/test/core/TestTsdbQueryRollupSalted.java @@ -0,0 +1,74 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.junit.Before; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.reflect.Whitebox; + +import net.opentsdb.rollup.RollupConfig; +import net.opentsdb.rollup.RollupInterval; + +public class TestTsdbQueryRollupSalted extends TestTsdbQueryRollup { + + @Before + public void beforeLocal() throws Exception { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(2); + PowerMockito.when(Const.MAX_NUM_TAGS()).thenReturn((short) 8); + + storeLongTimeSeriesSeconds(false, false); + final List<byte[]> families = new ArrayList<byte[]>(); + families.add(FAMILY); + + storage.addTable("tsdb-rollup-10m".getBytes(), families); + storage.addTable("tsdb-rollup-agg-10m".getBytes(), families); + storage.addTable("tsdb-rollup-1h".getBytes(), families); + storage.addTable("tsdb-rollup-agg-1h".getBytes(), families); + storage.addTable("tsdb-rollup-1d".getBytes(), families); + storage.addTable("tsdb-rollup-agg-1d".getBytes(), families); + + query = new TsdbQuery(tsdb); + tags2 = new HashMap<String, String>(1); + tags2.put(TAGK_STRING, TAGV_B_STRING); + + rollup_config = RollupConfig.builder() + .addAggregationId("sum", 0) + .addAggregationId("count", 1) + .addAggregationId("max", 2) + .addAggregationId("min", 3) + .addInterval(RollupInterval.builder() + .setTable("tsdb-rollup-10m") + .setPreAggregationTable("tsdb-rollup-agg-10m") + .setInterval("10m") + .setRowSpan("6h")) + .addInterval(RollupInterval.builder() + .setTable("tsdb-rollup-1h") + .setPreAggregationTable("tsdb-rollup-agg-1h") + .setInterval("1h") + .setRowSpan("1d")) + .addInterval(RollupInterval.builder() + .setTable("tsdb-rollup-1d") + .setPreAggregationTable("tsdb-rollup-agg-1d") + .setInterval("1d") + .setRowSpan("1n")) + .build(); + Whitebox.setInternalState(tsdb, "rollup_config", rollup_config); + } + +} diff --git a/test/core/TestTsdbQuerySalted.java b/test/core/TestTsdbQuerySalted.java new file mode 100644 index 0000000000..3061da6221 --- /dev/null +++ b/test/core/TestTsdbQuerySalted.java @@ -0,0 +1,36 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import org.junit.Before; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.modules.junit4.PowerMockRunner; + +/** + * An integration test class that runs all of the tests in + * {@see TestTsdbQueryAggregators} but with salting enabled. + */ +@RunWith(PowerMockRunner.class) +public class TestTsdbQuerySalted extends TestTsdbQueryQueries { + + @Before + public void beforeLocal() throws Exception { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(2); + PowerMockito.when(Const.MAX_NUM_TAGS()).thenReturn((short) 8); + + query = new TsdbQuery(tsdb); + } +} diff --git a/test/core/TestTsdbQuerySaltedAppend.java b/test/core/TestTsdbQuerySaltedAppend.java new file mode 100644 index 0000000000..1f95bf0616 --- /dev/null +++ b/test/core/TestTsdbQuerySaltedAppend.java @@ -0,0 +1,30 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import org.junit.Before; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.reflect.Whitebox; + +public class TestTsdbQuerySaltedAppend extends TestTsdbQueryQueries { + + @Before + public void beforeLocal() { + Whitebox.setInternalState(config, "enable_appends", true); + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(2); + PowerMockito.when(Const.MAX_NUM_TAGS()).thenReturn((short) 8); + query = new TsdbQuery(tsdb); + } +} diff --git a/test/core/TestTsdbTSConfig.java b/test/core/TestTsdbTSConfig.java new file mode 100644 index 0000000000..5261a627c9 --- /dev/null +++ b/test/core/TestTsdbTSConfig.java @@ -0,0 +1,215 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.core; + +import com.google.common.collect.ImmutableMap; +import com.stumbleupon.async.Deferred; +import net.opentsdb.storage.MockBase; +import net.opentsdb.uid.UniqueId; +import net.opentsdb.utils.Config; +import org.hbase.async.HBaseClient; +import org.hbase.async.Scanner; +import org.jboss.netty.util.HashedWheelTimer; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.when; +import static org.powermock.api.mockito.PowerMockito.mock; + +/** + * Sets up a real TSDB with mocked client, compaction queue and timer along + * with mocked UID assignment, fetches for common unit tests. + */ +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({TSDB.class, Config.class, UniqueId.class, HBaseClient.class, + HashedWheelTimer.class, Scanner.class, Const.class }) +public class TestTsdbTSConfig { + /** A list of UIDs from A to Z for unit testing UIDs values */ + public static final Map<String, byte[]> METRIC_UIDS = + new HashMap<String, byte[]>(26); + public static final Map<String, byte[]> TAGK_UIDS = + new HashMap<String, byte[]>(26); + public static final Map<String, byte[]> TAGV_UIDS = + new HashMap<String, byte[]>(26); + static { + char letter = 'A'; + int uid = 10; + for (int i = 0; i < 26; i++) { + METRIC_UIDS.put(Character.toString(letter), + UniqueId.longToUID(uid, TSDB.metrics_width())); + TAGK_UIDS.put(Character.toString(letter), + UniqueId.longToUID(uid, TSDB.tagk_width())); + TAGV_UIDS.put(Character.toString(letter++), + UniqueId.longToUID(uid++, TSDB.tagv_width())); + } + } + + public static final String METRIC_STRING = "sys.cpu.user"; + public static final byte[] METRIC_BYTES = new byte[] { 0, 0, 1 }; + + public static final String TAGK_STRING = "host"; + public static final byte[] TAGK_BYTES = new byte[] { 0, 0, 1 }; + + public static final String TAGV_STRING = "web01"; + public static final byte[] TAGV_BYTES = new byte[] { 0, 0, 1 }; + + protected Config config; + protected TSDB tsdb; + protected HBaseClient client = mock(HBaseClient.class); + protected UniqueId metrics = mock(UniqueId.class); + protected UniqueId tag_names = mock(UniqueId.class); + protected UniqueId tag_values = mock(UniqueId.class); + protected Map<String, String> tags = new HashMap<String, String>(1); + protected MockBase storage; + + public void before(Map<String, String> overrideConfigs) throws Exception { + config = new Config(false); + for (Map.Entry<String, String> entry : overrideConfigs.entrySet()) { + config.overrideConfig(entry.getKey(), entry.getValue()); + } + + tsdb = PowerMockito.spy(new TSDB(config)); + + config.setAutoMetric(true); + + Whitebox.setInternalState(tsdb, "metrics", metrics); + Whitebox.setInternalState(tsdb, "tag_names", tag_names); + Whitebox.setInternalState(tsdb, "tag_values", tag_values); + + setupMetricMaps(); + setupTagkMaps(); + setupTagvMaps(); + + when(metrics.width()).thenReturn((short)3); + when(tag_names.width()).thenReturn((short)3); + when(tag_values.width()).thenReturn((short)3); + + tags.put(TAGK_STRING, TAGV_STRING); + } + + /** Adds the static UIDs to the metrics UID mock object */ + void setupMetricMaps() { + when(metrics.getId(METRIC_STRING)).thenReturn(METRIC_BYTES); + when(metrics.getIdAsync(METRIC_STRING)) + .thenAnswer(new Answer<Deferred<byte[]>>() { + @Override + public Deferred<byte[]> answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult(METRIC_BYTES); + } + }); + when(metrics.getOrCreateId(METRIC_STRING)) + .thenReturn(METRIC_BYTES); + } + + /** Adds the static UIDs to the tag keys UID mock object */ + void setupTagkMaps() { + when(tag_names.getId(TAGK_STRING)).thenReturn(TAGK_BYTES); + when(tag_names.getOrCreateId(TAGK_STRING)).thenReturn(TAGK_BYTES); + when(tag_names.getIdAsync(TAGK_STRING)) + .thenAnswer(new Answer<Deferred<byte[]>>() { + @Override + public Deferred<byte[]> answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult(TAGK_BYTES); + } + }); + when(tag_names.getOrCreateIdAsync(TAGK_STRING)) + .thenReturn(Deferred.fromResult(TAGK_BYTES)); + + } + + /** Adds the static UIDs to the tag values UID mock object */ + void setupTagvMaps() { + when(tag_values.getId(TAGV_STRING)).thenReturn(TAGV_BYTES); + when(tag_values.getOrCreateId(TAGV_STRING)).thenReturn(TAGV_BYTES); + when(tag_values.getIdAsync(TAGV_STRING)) + .thenAnswer(new Answer<Deferred<byte[]>>() { + @Override + public Deferred<byte[]> answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult(TAGV_BYTES); + } + }); + when(tag_values.getOrCreateIdAsync(TAGV_STRING)) + .thenReturn(Deferred.fromResult(TAGV_BYTES)); + + } + + @Test + public void scannerTimestampsEqualToQueryTimestamps() throws Exception { + before(ImmutableMap.of("tsd.storage.enable_compaction", "false")); + TSQuery q = new TSQuery(); + q.setStart("1h-ago"); + + TSSubQuery subQuery = new TSSubQuery(); + subQuery.setMetric(METRIC_STRING); + subQuery.setAggregator("none"); + + ArrayList<TSSubQuery> list = new ArrayList<TSSubQuery>(); + list.add(subQuery); + q.setQueries(list); + q.validateAndSetQuery(); + + TsdbQuery query = new TsdbQuery(tsdb); + query.configureFromQuery(q, 0); + + Scanner scanner = query.getScanner(); + long minTs = scanner.getMinTimestamp(); + long maxTs = scanner.getMaxTimestamp(); + // For 1h - ago, the TSDB will create a window of 2 hrs, For example if the current time is 2:45 PM, the window will be 1 PM to 3 PM + assert((maxTs - minTs) == (2 * 3600000)); + } + + @Test + public void scannerTimestampsAreOnlySetWhenDTCTimestampIsOlder() throws Exception { + String now = String.valueOf(System.currentTimeMillis()); + before(ImmutableMap.of("tsd.storage.get_date_tiered_compaction_start", now)); + TSQuery q = new TSQuery(); + q.setStart("1h-ago"); + + TSSubQuery subQuery = new TSSubQuery(); + subQuery.setMetric(METRIC_STRING); + subQuery.setAggregator("none"); + + ArrayList<TSSubQuery> list = new ArrayList<TSSubQuery>(); + list.add(subQuery); + q.setQueries(list); + q.validateAndSetQuery(); + + TsdbQuery query = new TsdbQuery(tsdb); + query.configureFromQuery(q, 0); + + Scanner scanner = query.getScanner(); + long minTs = scanner.getMinTimestamp(); + long maxTs = scanner.getMaxTimestamp(); + assertEquals(0L, minTs); + assertEquals(Long.MAX_VALUE, maxTs); + } + +} \ No newline at end of file diff --git a/test/meta/TestAnnotation.java b/test/meta/TestAnnotation.java index bc77a6831b..6d2cbe720f 100644 --- a/test/meta/TestAnnotation.java +++ b/test/meta/TestAnnotation.java @@ -15,24 +15,20 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -import static org.mockito.Matchers.anyString; -import static org.powermock.api.mockito.PowerMockito.mock; +import static org.junit.Assert.assertTrue; import java.util.List; +import java.util.Map; +import net.opentsdb.core.BaseTsdbTest; +import net.opentsdb.core.Const; +import net.opentsdb.core.RowKey; import net.opentsdb.core.TSDB; import net.opentsdb.storage.MockBase; -import net.opentsdb.uid.UniqueId; -import net.opentsdb.utils.Config; import net.opentsdb.utils.JSON; -import org.hbase.async.DeleteRequest; -import org.hbase.async.GetRequest; -import org.hbase.async.HBaseClient; -import org.hbase.async.KeyValue; -import org.hbase.async.PutRequest; +import org.hbase.async.Bytes; import org.hbase.async.Scanner; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.powermock.api.mockito.PowerMockito; @@ -40,73 +36,21 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import com.google.common.collect.Maps; + @PowerMockIgnore({"javax.management.*", "javax.xml.*", "ch.qos.*", "org.slf4j.*", "com.sum.*", "org.xml.*"}) @RunWith(PowerMockRunner.class) -@PrepareForTest({TSDB.class, Config.class, UniqueId.class, HBaseClient.class, - GetRequest.class, PutRequest.class, DeleteRequest.class, KeyValue.class, - Scanner.class, Annotation.class}) -public final class TestAnnotation { - private TSDB tsdb; - private HBaseClient client = mock(HBaseClient.class); - private MockBase storage; +@PrepareForTest({ Annotation.class, Const.class, Scanner.class }) +public final class TestAnnotation extends BaseTsdbTest { private Annotation note = new Annotation(); - - final private byte[] global_row_key = - new byte[] { 0, 0, 0, (byte) 0x4F, (byte) 0x29, (byte) 0xD2, 0 }; - final private byte[] tsuid_row_key = - new byte[] { 0, 0, 1, (byte) 0x52, (byte) 0xC2, (byte) 0x09, 0, 0, 0, - 1, 0, 0, 1 }; - + private final static String TSUID = "000001000001000001"; + private byte[] global_row_key; + private byte[] tsuid_row_key; // 1425715200 - Sat Mar 7 00:00:00 PST 2015 - final private byte[] global_row_key_2015_midnight = - new byte[] { 0, 0, 0, (byte) 0x54, (byte) 0xFA, (byte) 0xB0, 0 }; + private byte[] global_row_key_2015_midnight; - @Before - public void before() throws Exception { - final Config config = new Config(false); - PowerMockito.whenNew(HBaseClient.class) - .withArguments(anyString(), anyString()).thenReturn(client); - tsdb = new TSDB(config); - - storage = new MockBase(tsdb, client, true, true, true, true); - - // add a global - storage.addColumn(global_row_key, - new byte[] { 1, 0, 0 }, - ("{\"startTime\":1328140800,\"endTime\":1328140801,\"description\":" + - "\"Description\",\"notes\":\"Notes\",\"custom\":{\"owner\":" + - "\"ops\"}}").getBytes(MockBase.ASCII())); - - storage.addColumn(global_row_key, - new byte[] { 1, 0, 1 }, - ("{\"startTime\":1328140801,\"endTime\":1328140803,\"description\":" + - "\"Global 2\",\"notes\":\"Nothing\"}").getBytes(MockBase.ASCII())); - - // add a local - storage.addColumn(tsuid_row_key, - new byte[] { 1, 0x0A, 0x02 }, - ("{\"tsuid\":\"000001000001000001\",\"startTime\":1388450562," + - "\"endTime\":1419984000,\"description\":\"Hello!\",\"notes\":" + - "\"My Notes\",\"custom\":{\"owner\":\"ops\"}}") - .getBytes(MockBase.ASCII())); - - storage.addColumn(tsuid_row_key, - new byte[] { 1, 0x0A, 0x03 }, - ("{\"tsuid\":\"000001000001000001\",\"startTime\":1388450563," + - "\"endTime\":1419984000,\"description\":\"Note2\",\"notes\":" + - "\"Nothing\"}") - .getBytes(MockBase.ASCII())); - - // add some data points too - storage.addColumn(tsuid_row_key, - new byte[] { 0x50, 0x10 }, new byte[] { 1 }); - - storage.addColumn(tsuid_row_key, - new byte[] { 0x50, 0x18 }, new byte[] { 2 }); - } - @Test public void constructor() { assertNotNull(new Annotation()); @@ -129,26 +73,41 @@ public void deserialize() throws Exception { @Test public void getAnnotation() throws Exception { - note = Annotation.getAnnotation(tsdb, "000001000001000001", 1388450562L) + setupStorage(false); + + note = Annotation.getAnnotation(tsdb, TSUID, 1388450562L) .joinUninterruptibly(); assertNotNull(note); - assertEquals("000001000001000001", note.getTSUID()); + assertEquals(TSUID, note.getTSUID()); + assertEquals("Hello!", note.getDescription()); + assertEquals(1388450562L, note.getStartTime()); + } + + @Test + public void getAnnotationSalted() throws Exception { + setupStorage(true); + note = Annotation.getAnnotation(tsdb, TSUID, 1388450562L) + .joinUninterruptibly(); + assertNotNull(note); + assertEquals(TSUID, note.getTSUID()); assertEquals("Hello!", note.getDescription()); assertEquals(1388450562L, note.getStartTime()); } @Test public void getAnnotationNormalizeMs() throws Exception { - note = Annotation.getAnnotation(tsdb, "000001000001000001", 1388450562000L) + setupStorage(false); + note = Annotation.getAnnotation(tsdb, TSUID, 1388450562000L) .joinUninterruptibly(); assertNotNull(note); - assertEquals("000001000001000001", note.getTSUID()); + assertEquals(TSUID, note.getTSUID()); assertEquals("Hello!", note.getDescription()); assertEquals(1388450562L, note.getStartTime()); } @Test public void getAnnotationGlobal() throws Exception { + setupStorage(false); note = Annotation.getAnnotation(tsdb, 1328140800000L) .joinUninterruptibly(); assertNotNull(note); @@ -157,15 +116,28 @@ public void getAnnotationGlobal() throws Exception { assertEquals(1328140800L, note.getStartTime()); } + @Test + public void getAnnotationGlobalSalted() throws Exception { + setupStorage(true); + note = Annotation.getAnnotation(tsdb, 1328140800000L) + .joinUninterruptibly(); + assertNotNull(note); + assertEquals("", note.getTSUID()); + assertEquals("Description", note.getDescription()); + assertEquals(1328140800L, note.getStartTime()); + } + @Test public void getAnnotationNotFound() throws Exception { - note = Annotation.getAnnotation(tsdb, "000001000001000001", 1388450564L) + setupStorage(false); + note = Annotation.getAnnotation(tsdb, TSUID, 1388450564L) .joinUninterruptibly(); assertNull(note); } @Test public void getAnnotationGlobalNotFound() throws Exception { + setupStorage(false); note = Annotation.getAnnotation(tsdb, 1388450563L) .joinUninterruptibly(); assertNull(note); @@ -173,12 +145,13 @@ public void getAnnotationGlobalNotFound() throws Exception { @Test (expected = IllegalArgumentException.class) public void getAnnotationNoStartTime() throws Exception { - Annotation.getAnnotation(tsdb, "000001000001000001", 0L) + Annotation.getAnnotation(tsdb, TSUID, 0L) .joinUninterruptibly(); } @Test public void getGlobalAnnotations() throws Exception { + setupStorage(false); List<Annotation> notes = Annotation.getGlobalAnnotations(tsdb, 1328140000, 1328141000).joinUninterruptibly(); assertNotNull(notes); @@ -189,8 +162,22 @@ public void getGlobalAnnotations() throws Exception { assertEquals("Global 2", note1.getDescription()); } + @Test + public void getGlobalAnnotationsSalted() throws Exception { + setupStorage(true); + List<Annotation> notes = Annotation.getGlobalAnnotations(tsdb, 1328140000, + 1328141000).joinUninterruptibly(); + assertNotNull(notes); + assertEquals(2, notes.size()); + Annotation note0 = notes.get(0); + Annotation note1 = notes.get(1); + assertEquals("Description", note0.getDescription()); + assertEquals("Global 2", note1.getDescription()); + } + @Test public void getGlobalAnnotationOutsideCurrentHour() throws Exception { + setupStorage(false); // 1425716000 - Sat Mar 7 00:13:20 PST 2015 storage.addColumn(global_row_key_2015_midnight, new byte[] { 1, 3, (byte) 0x20 }, @@ -209,8 +196,30 @@ public void getGlobalAnnotationOutsideCurrentHour() throws Exception { assertEquals("Issue #457", note0.getNotes()); } + @Test + public void getGlobalAnnotationOutsideCurrentHourSalt() throws Exception { + setupStorage(true); + // 1425716000 - Sat Mar 7 00:13:20 PST 2015 + storage.addColumn(global_row_key_2015_midnight, + new byte[] { 1, 3, (byte) 0x20 }, + ("{\"startTime\":1425716000,\"endTime\":1425716001,\"description\":" + + "\"Global 3\",\"notes\":\"Issue #457\"}").getBytes(MockBase.ASCII())); + + int right_now = 1425717000; // Sat Mar 7 00:30:00 PST 2015 + int fourty_minutes_ago = 1425714600; // Fri Mar 6 23:50:00 PST 2015 + + List<Annotation> recentGlobalAnnotation = Annotation.getGlobalAnnotations( + tsdb, fourty_minutes_ago, right_now).joinUninterruptibly(); + assertNotNull(recentGlobalAnnotation); + assertEquals(1, recentGlobalAnnotation.size()); + Annotation note0 = recentGlobalAnnotation.get(0); + assertEquals("Global 3", note0.getDescription()); + assertEquals("Issue #457", note0.getNotes()); + } + @Test public void getGlobalAnnotationsEmpty() throws Exception { + setupStorage(false); List<Annotation> notes = Annotation.getGlobalAnnotations(tsdb, 1328150000, 1328160000).joinUninterruptibly(); assertNotNull(notes); @@ -229,28 +238,45 @@ public void getGlobalAnnotationsEndLessThanStart() throws Exception { @Test public void syncToStorage() throws Exception { - note.setTSUID("000001000001000001"); + setupStorage(false); + note.setTSUID(TSUID); + note.setStartTime(1388450562L); + note.setDescription("Synced!"); + note.syncToStorage(tsdb, false).joinUninterruptibly(); + final byte[] col = storage.getColumn(tsuid_row_key, + new byte[] { 1, 0x0A, 0x02 }); + note = JSON.parseToObject(col, Annotation.class); + assertEquals(TSUID, note.getTSUID()); + assertEquals("Synced!", note.getDescription()); + assertEquals("My Notes", note.getNotes()); + } + + @Test + public void syncToStorageSalted() throws Exception { + setupStorage(true); + note.setTSUID(TSUID); note.setStartTime(1388450562L); note.setDescription("Synced!"); note.syncToStorage(tsdb, false).joinUninterruptibly(); final byte[] col = storage.getColumn(tsuid_row_key, new byte[] { 1, 0x0A, 0x02 }); note = JSON.parseToObject(col, Annotation.class); - assertEquals("000001000001000001", note.getTSUID()); + assertEquals(TSUID, note.getTSUID()); assertEquals("Synced!", note.getDescription()); assertEquals("My Notes", note.getNotes()); } @Test public void syncToStorageMilliseconds() throws Exception { - note.setTSUID("000001000001000001"); + setupStorage(false); + note.setTSUID(TSUID); note.setStartTime(1388450562500L); note.setDescription("Synced!"); note.syncToStorage(tsdb, false).joinUninterruptibly(); final byte[] col = storage.getColumn(tsuid_row_key, new byte[] { 1, 0x00, 0x27, 0x19, (byte) 0xC4 }); note = JSON.parseToObject(col, Annotation.class); - assertEquals("000001000001000001", note.getTSUID()); + assertEquals(TSUID, note.getTSUID()); assertEquals("Synced!", note.getDescription()); assertEquals("", note.getNotes()); assertEquals(1388450562500L, note.getStartTime()); @@ -258,6 +284,21 @@ public void syncToStorageMilliseconds() throws Exception { @Test public void syncToStorageGlobal() throws Exception { + setupStorage(false); + note.setStartTime(1328140800L); + note.setDescription("Synced!"); + note.syncToStorage(tsdb, false).joinUninterruptibly(); + final byte[] col = storage.getColumn(global_row_key, + new byte[] { 1, 0, 0 }); + note = JSON.parseToObject(col, Annotation.class); + assertEquals("", note.getTSUID()); + assertEquals("Synced!", note.getDescription()); + assertEquals("Notes", note.getNotes()); + } + + @Test + public void syncToStorageGlobalSalted() throws Exception { + setupStorage(true); note.setStartTime(1328140800L); note.setDescription("Synced!"); note.syncToStorage(tsdb, false).joinUninterruptibly(); @@ -271,6 +312,7 @@ public void syncToStorageGlobal() throws Exception { @Test public void syncToStorageGlobalMilliseconds() throws Exception { + setupStorage(false); note.setStartTime(1328140800500L); note.setDescription("Synced!"); note.syncToStorage(tsdb, false).joinUninterruptibly(); @@ -284,21 +326,53 @@ public void syncToStorageGlobalMilliseconds() throws Exception { @Test (expected = IllegalArgumentException.class) public void syncToStorageMissingStart() throws Exception { - note.setTSUID("000001000001000001"); + note.setTSUID(TSUID); note.setDescription("Synced!"); note.syncToStorage(tsdb, false).joinUninterruptibly(); } @Test (expected = IllegalStateException.class) public void syncToStorageNoChanges() throws Exception { - note.setTSUID("000001000001000001"); + note.setTSUID(TSUID); note.setStartTime(1388450562L); note.syncToStorage(tsdb, false).joinUninterruptibly(); } + @Test + public void getStorageJSONTags() throws Exception { + Map<String, String> custom = Maps.newHashMap(); + custom.put("C", "C"); + custom.put("P", "P"); + custom.put("E", "E"); + System.out.println(custom); + + note.setTSUID(TSUID); + note.setStartTime(1388450562L); + note.setCustom(custom); + final String json = new String(note.getStorageJSON()); + assertTrue(json.contains("{\"C\":\"C\",\"E\":\"E\",\"P\":\"P\"}")); + } + @Test public void delete() throws Exception { - note.setTSUID("000001000001000001"); + setupStorage(false); + note.setTSUID(TSUID); + note.setStartTime(1388450562); + note.delete(tsdb).joinUninterruptibly(); + assertNull(storage.getColumn(tsuid_row_key, + new byte[] { 1, 0x0A, 0x02 })); + assertNotNull(storage.getColumn(tsuid_row_key, + new byte[] { 1, 0x0A, 0x03 })); + assertNotNull(storage.getColumn(tsuid_row_key, + new byte[] { 0x50, 0x10 })); + assertNotNull(storage.getColumn(tsuid_row_key, + new byte[] { 0x50, 0x18 })); + } + + @Test + public void deleteSalted() throws Exception { + setupStorage(true); + note.setTSUID(TSUID); note.setStartTime(1388450562); note.delete(tsdb).joinUninterruptibly(); assertNull(storage.getColumn(tsuid_row_key, @@ -313,7 +387,8 @@ public void delete() throws Exception { @Test public void deleteNormalizeMs() throws Exception { - note.setTSUID("000001000001000001"); + setupStorage(false); + note.setTSUID(TSUID); note.setStartTime(1388450562000L); note.delete(tsdb).joinUninterruptibly(); assertNull(storage.getColumn(tsuid_row_key, @@ -330,7 +405,8 @@ public void deleteNormalizeMs() throws Exception { // and it's ignored. @Test public void deleteNotFound() throws Exception { - note.setTSUID("000001000001000001"); + setupStorage(false); + note.setTSUID(TSUID); note.setStartTime(1388450561); note.delete(tsdb).joinUninterruptibly(); assertNotNull(storage.getColumn(tsuid_row_key, @@ -345,12 +421,24 @@ public void deleteNotFound() throws Exception { @Test (expected = IllegalArgumentException.class) public void deleteMissingStart() throws Exception { - note.setTSUID("000001000001000001"); + note.setTSUID(TSUID); note.delete(tsdb).joinUninterruptibly(); } @Test public void deleteGlobal() throws Exception { + setupStorage(false); + note.setStartTime(1328140800); + note.delete(tsdb).joinUninterruptibly(); + assertNull(storage.getColumn(global_row_key, + new byte[] { 1, 0, 0 })); + assertNotNull(storage.getColumn(global_row_key, + new byte[] { 1, 0, 1 })); + } + + @Test + public void deleteGlobalSalted() throws Exception { + setupStorage(true); note.setStartTime(1328140800); note.delete(tsdb).joinUninterruptibly(); assertNull(storage.getColumn(global_row_key, @@ -361,6 +449,7 @@ public void deleteGlobal() throws Exception { @Test public void deleteGlobalNotFound() throws Exception { + setupStorage(false); note.setStartTime(1328140803); note.delete(tsdb).joinUninterruptibly(); assertNotNull(storage.getColumn(global_row_key, @@ -371,6 +460,24 @@ public void deleteGlobalNotFound() throws Exception { @Test public void deleteRange() throws Exception { + setupStorage(false); + final int count = Annotation.deleteRange(tsdb, + new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1}, 1388450560000L, + 1388450562000L).joinUninterruptibly(); + assertEquals(1, count); + assertNull(storage.getColumn(tsuid_row_key, + new byte[] { 1, 0x0A, 0x02 })); + assertNotNull(storage.getColumn(tsuid_row_key, + new byte[] { 1, 0x0A, 0x03 })); + assertNotNull(storage.getColumn(tsuid_row_key, + new byte[] { 0x50, 0x10 })); + assertNotNull(storage.getColumn(tsuid_row_key, + new byte[] { 0x50, 0x18 })); + } + + @Test + public void deleteRangeSalted() throws Exception { + setupStorage(true); final int count = Annotation.deleteRange(tsdb, new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1}, 1388450560000L, 1388450562000L).joinUninterruptibly(); @@ -387,6 +494,24 @@ public void deleteRange() throws Exception { @Test public void deleteRangeNone() throws Exception { + setupStorage(false); + final int count = Annotation.deleteRange(tsdb, + new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1}, 1388450560000L, + 1388450561000L).joinUninterruptibly(); + assertEquals(0, count); + assertNotNull(storage.getColumn(tsuid_row_key, + new byte[] { 1, 0x0A, 0x02 })); + assertNotNull(storage.getColumn(tsuid_row_key, + new byte[] { 1, 0x0A, 0x03 })); + assertNotNull(storage.getColumn(tsuid_row_key, + new byte[] { 0x50, 0x10 })); + assertNotNull(storage.getColumn(tsuid_row_key, + new byte[] { 0x50, 0x18 })); + } + + @Test + public void deleteRangeNoneSalted() throws Exception { + setupStorage(true); final int count = Annotation.deleteRange(tsdb, new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1}, 1388450560000L, 1388450561000L).joinUninterruptibly(); @@ -403,6 +528,24 @@ public void deleteRangeNone() throws Exception { @Test public void deleteRangeMultiple() throws Exception { + setupStorage(false); + final int count = Annotation.deleteRange(tsdb, + new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1}, 1388450560000L, + 1388450568000L).joinUninterruptibly(); + assertEquals(2, count); + assertNull(storage.getColumn(tsuid_row_key, + new byte[] { 1, 0x0A, 0x02 })); + assertNull(storage.getColumn(tsuid_row_key, + new byte[] { 1, 0x0A, 0x03 })); + assertNotNull(storage.getColumn(tsuid_row_key, + new byte[] { 0x50, 0x10 })); + assertNotNull(storage.getColumn(tsuid_row_key, + new byte[] { 0x50, 0x18 })); + } + + @Test + public void deleteRangeMultipleSalted() throws Exception { + setupStorage(true); final int count = Annotation.deleteRange(tsdb, new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1}, 1388450560000L, 1388450568000L).joinUninterruptibly(); @@ -419,6 +562,19 @@ public void deleteRangeMultiple() throws Exception { @Test public void deleteRangeGlobal() throws Exception { + setupStorage(false); + final int count = Annotation.deleteRange(tsdb, null, 1328140799000L, + 1328140800000L).joinUninterruptibly(); + assertEquals(1, count); + assertNull(storage.getColumn(global_row_key, + new byte[] { 1, 0, 0 })); + assertNotNull(storage.getColumn(global_row_key, + new byte[] { 1, 0, 1 })); + } + + @Test + public void deleteRangeGlobalSalted() throws Exception { + setupStorage(true); final int count = Annotation.deleteRange(tsdb, null, 1328140799000L, 1328140800000L).joinUninterruptibly(); assertEquals(1, count); @@ -430,6 +586,7 @@ public void deleteRangeGlobal() throws Exception { @Test public void deleteRangeGlobalNone() throws Exception { + setupStorage(false); final int count = Annotation.deleteRange(tsdb, null, 1328140798000L, 1328140799000L).joinUninterruptibly(); assertEquals(0, count); @@ -441,6 +598,19 @@ public void deleteRangeGlobalNone() throws Exception { @Test public void deleteRangeGlobalMultiple() throws Exception { + setupStorage(false); + final int count = Annotation.deleteRange(tsdb, null, 1328140799000L, + 1328140900000L).joinUninterruptibly(); + assertEquals(2, count); + assertNull(storage.getColumn(global_row_key, + new byte[] { 1, 0, 0 })); + assertNull(storage.getColumn(global_row_key, + new byte[] { 1, 0, 1 })); + } + + @Test + public void deleteRangeGlobalMultipleSalted() throws Exception { + setupStorage(true); final int count = Annotation.deleteRange(tsdb, null, 1328140799000L, 1328140900000L).joinUninterruptibly(); assertEquals(2, count); @@ -460,4 +630,68 @@ public void deleteRangeEndLessThanStart() throws Exception { Annotation.deleteRange(tsdb, null, 1328140799000L, 1328140798000L) .joinUninterruptibly(); } + + /** + * Sets up storage with or without salting and writes a few bits of data + * @param salted Whether or not to use salting + */ + private void setupStorage(final boolean salted) { + if (salted) { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(2); + } + + global_row_key = new byte[Const.SALT_WIDTH() + TSDB.metrics_width() + + Const.TIMESTAMP_BYTES]; + System.arraycopy(Bytes.fromInt(1328140800), 0, global_row_key, + Const.SALT_WIDTH() + TSDB.metrics_width(), Const.TIMESTAMP_BYTES); + + global_row_key_2015_midnight = + new byte[Const.SALT_WIDTH() + TSDB.metrics_width() + + Const.TIMESTAMP_BYTES]; + System.arraycopy(Bytes.fromInt(1425715200), 0, global_row_key_2015_midnight, + Const.SALT_WIDTH() + TSDB.metrics_width(), Const.TIMESTAMP_BYTES); + + tsuid_row_key = getRowKeyTemplate(); + System.arraycopy(Bytes.fromInt(1388448000), 0, tsuid_row_key, + Const.SALT_WIDTH() + TSDB.metrics_width(), Const.TIMESTAMP_BYTES); + RowKey.prefixKeyWithSalt(tsuid_row_key); + + storage = new MockBase(tsdb, client, true, true, true, true); + + // add a global + storage.addColumn(global_row_key, + new byte[] { 1, 0, 0 }, + ("{\"startTime\":1328140800,\"endTime\":1328140801,\"description\":" + + "\"Description\",\"notes\":\"Notes\",\"custom\":{\"owner\":" + + "\"ops\"}}").getBytes(MockBase.ASCII()), 1328140799972L); + + storage.addColumn(global_row_key, + new byte[] { 1, 0, 1 }, + ("{\"startTime\":1328140801,\"endTime\":1328140803,\"description\":" + + "\"Global 2\",\"notes\":\"Nothing\"}").getBytes(MockBase.ASCII()), 1328140799973L); + + // add a local + storage.addColumn(tsuid_row_key, + new byte[] { 1, 0x0A, 0x02 }, + ("{\"tsuid\":\"000001000001000001\",\"startTime\":1388450562," + + "\"endTime\":1419984000,\"description\":\"Hello!\",\"notes\":" + + "\"My Notes\",\"custom\":{\"owner\":\"ops\"}}") + .getBytes(MockBase.ASCII()), 1388448000003L); + + storage.addColumn(tsuid_row_key, + new byte[] { 1, 0x0A, 0x03 }, + ("{\"tsuid\":\"000001000001000001\",\"startTime\":1388450563," + + "\"endTime\":1419984000,\"description\":\"Note2\",\"notes\":" + + "\"Nothing\"}") + .getBytes(MockBase.ASCII()), 1388448000004L); + + // add some data points too + storage.addColumn(tsuid_row_key, + new byte[] { 0x50, 0x10 }, new byte[] { 1 }, 1388448000005L); + + storage.addColumn(tsuid_row_key, + new byte[] { 0x50, 0x18 }, new byte[] { 2 }, 1388448000006L); + } } diff --git a/test/meta/TestTSMeta.java b/test/meta/TestTSMeta.java index 95c79da54d..3984495aee 100644 --- a/test/meta/TestTSMeta.java +++ b/test/meta/TestTSMeta.java @@ -19,11 +19,13 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.powermock.api.mockito.PowerMockito.mock; +import java.util.ArrayList; +import java.util.List; + import net.opentsdb.core.TSDB; import net.opentsdb.storage.MockBase; import net.opentsdb.uid.NoSuchUniqueId; @@ -42,7 +44,6 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; @@ -58,7 +59,10 @@ GetRequest.class, PutRequest.class, DeleteRequest.class, KeyValue.class, Scanner.class, UIDMeta.class, TSMeta.class, AtomicIncrementRequest.class}) public final class TestTSMeta { - private static byte[] NAME_FAMILY = "name".getBytes(MockBase.ASCII()); + private final static byte[] NAME_FAMILY = "name".getBytes(MockBase.ASCII()); + private final static byte[] META_TABLE = "tsdb-meta".getBytes(MockBase.ASCII()); + private final static byte[] UID_TABLE = "tsdb-uid".getBytes(MockBase.ASCII()); + private final static byte[] TSUID = new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }; private TSDB tsdb; private Config config; private HBaseClient client = mock(HBaseClient.class); @@ -74,17 +78,18 @@ public void before() throws Exception { when(config.getString("tsd.storage.hbase.tree_table")).thenReturn("tsdb-tree"); when(config.enable_tsuid_incrementing()).thenReturn(true); when(config.enable_realtime_ts()).thenReturn(true); - - PowerMockito.whenNew(HBaseClient.class) - .withArguments(anyString(), anyString()).thenReturn(client); - tsdb = new TSDB(config); + + tsdb = new TSDB(client, config); storage = new MockBase(tsdb, client, true, true, true, true); + final List<byte[]> families = new ArrayList<byte[]>(); + families.add(TSMeta.FAMILY); + storage.addTable(META_TABLE, families); - storage.addColumn(new byte[] { 0, 0, 1 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "metrics".getBytes(MockBase.ASCII()), "sys.cpu.0".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "metric_meta".getBytes(MockBase.ASCII()), ("{\"uid\":\"000001\",\"type\":\"METRIC\",\"name\":\"sys.cpu.0\"," + @@ -92,11 +97,11 @@ public void before() throws Exception { "1328140801,\"displayName\":\"System CPU\"}") .getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "tagk".getBytes(MockBase.ASCII()), "host".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "tagk_meta".getBytes(MockBase.ASCII()), ("{\"uid\":\"000001\",\"type\":\"TAGK\",\"name\":\"host\"," + @@ -104,11 +109,11 @@ public void before() throws Exception { "1328140801,\"displayName\":\"Host server name\"}") .getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "tagv".getBytes(MockBase.ASCII()), "web01".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "tagv_meta".getBytes(MockBase.ASCII()), ("{\"uid\":\"000001\",\"type\":\"TAGV\",\"name\":\"web01\"," + @@ -116,7 +121,7 @@ public void before() throws Exception { "1328140801,\"displayName\":\"Web server 1\"}") .getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }, + storage.addColumn(META_TABLE, TSUID, NAME_FAMILY, "ts_meta".getBytes(MockBase.ASCII()), ("{\"tsuid\":\"000001000001000001\",\"" + @@ -124,13 +129,13 @@ public void before() throws Exception { "\"custom\":null,\"units\":\"\",\"retention\":42,\"max\":1.0,\"min\":" + "\"NaN\",\"displayName\":\"Display\",\"dataType\":\"Data\"}") .getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }, - NAME_FAMILY, + storage.addColumn(META_TABLE, TSUID, + TSMeta.FAMILY, "ts_ctr".getBytes(MockBase.ASCII()), Bytes.fromLong(1L)); - storage.addColumn(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 2 }, - NAME_FAMILY, + storage.addColumn(META_TABLE, new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 2 }, + TSMeta.FAMILY, "ts_meta".getBytes(MockBase.ASCII()), ("{\"tsuid\":\"000001000001000002\",\"" + "description\":\"Description\",\"notes\":\"Notes\",\"created\":1328140800," + @@ -193,7 +198,7 @@ public void getTSMetaDoesNotExist() throws Exception { @Test (expected = NoSuchUniqueId.class) public void getTSMetaNSUMetric() throws Throwable { - storage.addColumn(new byte[] { 0, 0, 2, 0, 0, 1, 0, 0, 1 }, + storage.addColumn(META_TABLE, new byte[] { 0, 0, 2, 0, 0, 1, 0, 0, 1 }, NAME_FAMILY, "ts_meta".getBytes(MockBase.ASCII()), ("{\"tsuid\":\"000002000001000001\",\"" + @@ -210,7 +215,7 @@ public void getTSMetaNSUMetric() throws Throwable { @Test (expected = NoSuchUniqueId.class) public void getTSMetaNSUTagk() throws Throwable { - storage.addColumn(new byte[] { 0, 0, 1, 0, 0, 2, 0, 0, 1 }, + storage.addColumn(META_TABLE, new byte[] { 0, 0, 1, 0, 0, 2, 0, 0, 1 }, NAME_FAMILY, "ts_meta".getBytes(MockBase.ASCII()), ("{\"tsuid\":\"000001000002000001\",\"" + @@ -227,7 +232,7 @@ public void getTSMetaNSUTagk() throws Throwable { @Test (expected = NoSuchUniqueId.class) public void getTSMetaNSUTagv() throws Throwable { - storage.addColumn(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 2 }, + storage.addColumn(META_TABLE, new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 2 }, NAME_FAMILY, "ts_meta".getBytes(MockBase.ASCII()), ("{\"tsuid\":\"000001000001000002\",\"" + @@ -256,7 +261,7 @@ public void deleteNull() throws Exception { @Test public void syncToStorage() throws Exception { - meta = new TSMeta(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }, 1357300800000L); + meta = new TSMeta(TSUID, 1357300800000L); meta.setDisplayName("New DN"); meta.syncToStorage(tsdb, false).joinUninterruptibly(); assertEquals("New DN", meta.getDisplayName()); @@ -265,7 +270,7 @@ public void syncToStorage() throws Exception { @Test public void syncToStorageOverwrite() throws Exception { - meta = new TSMeta(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }, 1357300800000L); + meta = new TSMeta(TSUID, 1357300800000L); meta.setDisplayName("New DN"); meta.syncToStorage(tsdb, true).joinUninterruptibly(); assertEquals("New DN", meta.getDisplayName()); @@ -286,14 +291,14 @@ public void syncToStorageNullTSUID() throws Exception { @Test (expected = IllegalArgumentException.class) public void syncToStorageDoesNotExist() throws Exception { - storage.flushRow(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }); - meta = new TSMeta(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }, 1357300800000L); + storage.flushRow(META_TABLE, TSUID); + meta = new TSMeta(TSUID, 1357300800000L); meta.syncToStorage(tsdb, false).joinUninterruptibly(); } @Test public void storeNew() throws Exception { - meta = new TSMeta(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }, 1357300800000L); + meta = new TSMeta(TSUID, 1357300800000L); meta.setDisplayName("New DN"); meta.storeNew(tsdb); assertEquals("New DN", meta.getDisplayName()); @@ -319,7 +324,7 @@ public void metaExistsInStorage() throws Exception { @Test public void metaExistsInStorageNot() throws Exception { - storage.flushRow(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }); + storage.flushRow(META_TABLE, TSUID); assertFalse(TSMeta.metaExistsInStorage(tsdb, "000001000001000001") .joinUninterruptibly()); } @@ -327,14 +332,14 @@ public void metaExistsInStorageNot() throws Exception { @Test public void counterExistsInStorage() throws Exception { assertTrue(TSMeta.counterExistsInStorage(tsdb, - new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }).joinUninterruptibly()); + TSUID).joinUninterruptibly()); } @Test public void counterExistsInStorageNot() throws Exception { - storage.flushRow(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }); + storage.flushRow(META_TABLE, TSUID); assertFalse(TSMeta.counterExistsInStorage(tsdb, - new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }).joinUninterruptibly()); + TSUID).joinUninterruptibly()); } @Test @@ -374,12 +379,27 @@ public void COUNTER_QUALIFIER() throws Exception { TSMeta.COUNTER_QUALIFIER()); } + @Test + public void storeIfNecessaryExists() throws Exception { + assertTrue(TSMeta.storeIfNecessary(tsdb, TSUID).join()); + } + + @Test + public void storeIfNecessaryMissing() throws Exception { + storage.flushRow(META_TABLE, TSUID); + assertNull(storage.getColumn(META_TABLE, TSUID, NAME_FAMILY, + TSMeta.META_QUALIFIER())); + assertTrue(TSMeta.storeIfNecessary(tsdb, TSUID).join()); + assertNotNull(storage.getColumn(META_TABLE, TSUID, NAME_FAMILY, + TSMeta.META_QUALIFIER())); + } + @Test public void parseFromColumn() throws Exception { final KeyValue column = mock(KeyValue.class); - when(column.key()).thenReturn(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }); - when(column.value()).thenReturn(storage.getColumn( - new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }, + when(column.key()).thenReturn(TSUID); + when(column.value()).thenReturn(storage.getColumn(META_TABLE, + TSUID, NAME_FAMILY, "ts_meta".getBytes(MockBase.ASCII()))); final TSMeta meta = TSMeta.parseFromColumn(tsdb, column, false) @@ -392,9 +412,9 @@ public void parseFromColumn() throws Exception { @Test public void parseFromColumnWithUIDMeta() throws Exception { final KeyValue column = mock(KeyValue.class); - when(column.key()).thenReturn(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }); - when(column.value()).thenReturn(storage.getColumn( - new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }, + when(column.key()).thenReturn(TSUID); + when(column.value()).thenReturn(storage.getColumn(META_TABLE, + TSUID, NAME_FAMILY, "ts_meta".getBytes(MockBase.ASCII()))); final TSMeta meta = TSMeta.parseFromColumn(tsdb, column, true) diff --git a/test/meta/TestTSUIDQuery.java b/test/meta/TestTSUIDQuery.java index 0b7511b8b5..b92b01bcf4 100644 --- a/test/meta/TestTSUIDQuery.java +++ b/test/meta/TestTSUIDQuery.java @@ -12,20 +12,26 @@ // see <http://www.gnu.org/licenses/>. package net.opentsdb.meta; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import static org.mockito.Matchers.anyString; -import static org.mockito.Mockito.when; -import static org.powermock.api.mockito.PowerMockito.mock; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; -import java.lang.reflect.Field; -import java.util.HashMap; +import java.util.ArrayList; import java.util.List; +import net.opentsdb.core.BaseTsdbTest; +import net.opentsdb.core.IncomingDataPoint; import net.opentsdb.core.TSDB; import net.opentsdb.storage.MockBase; +import net.opentsdb.uid.NoSuchUniqueId; import net.opentsdb.uid.NoSuchUniqueName; import net.opentsdb.uid.UniqueId; import net.opentsdb.utils.Config; +import net.opentsdb.utils.DateTime; import org.hbase.async.AtomicIncrementRequest; import org.hbase.async.Bytes; @@ -35,6 +41,7 @@ import org.hbase.async.KeyValue; import org.hbase.async.PutRequest; import org.hbase.async.Scanner; +import org.hbase.async.Bytes.ByteMap; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -42,8 +49,9 @@ import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; -import com.stumbleupon.async.Deferred; +import com.stumbleupon.async.DeferredGroupException; @PowerMockIgnore({"javax.management.*", "javax.xml.*", "ch.qos.*", "org.slf4j.*", @@ -51,275 +59,812 @@ @RunWith(PowerMockRunner.class) @PrepareForTest({TSDB.class, Config.class, UniqueId.class, HBaseClient.class, GetRequest.class, PutRequest.class, DeleteRequest.class, KeyValue.class, - Scanner.class, TSMeta.class, AtomicIncrementRequest.class}) -public final class TestTSUIDQuery { - private static byte[] NAME_FAMILY = "name".getBytes(MockBase.ASCII()); - private TSDB tsdb; - private Config config; - private HBaseClient client = mock(HBaseClient.class); - private MockBase storage; - private UniqueId metrics = mock(UniqueId.class); - private UniqueId tag_names = mock(UniqueId.class); - private UniqueId tag_values = mock(UniqueId.class); - private TSUIDQuery query; + Scanner.class, TSMeta.class, AtomicIncrementRequest.class, DateTime.class }) +public final class TestTSUIDQuery extends BaseTsdbTest { + private final static byte[] NAME_FAMILY = "name".getBytes(MockBase.ASCII()); + private final static byte[] META_TABLE = "tsdb-meta".getBytes(MockBase.ASCII()); + private final static byte[] UID_TABLE = "tsdb-uid".getBytes(MockBase.ASCII()); + private static final byte[] TSUID = new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }; + private static final byte[] QUAL = new byte[] { 0, 0 }; + private static final byte[] VAL = new byte[] { 0x2A }; + private TSUIDQuery query; @Before - public void before() throws Exception { - config = mock(Config.class); - when(config.getString("tsd.storage.hbase.data_table")).thenReturn("tsdb"); - when(config.getString("tsd.storage.hbase.uid_table")).thenReturn("tsdb-uid"); - when(config.getString("tsd.storage.hbase.meta_table")).thenReturn("tsdb-meta"); - when(config.getString("tsd.storage.hbase.tree_table")).thenReturn("tsdb-tree"); - when(config.enable_tsuid_incrementing()).thenReturn(true); - when(config.enable_realtime_ts()).thenReturn(true); - - PowerMockito.whenNew(HBaseClient.class) - .withArguments(anyString(), anyString()).thenReturn(client); - tsdb = new TSDB(config); + public void beforeLocal() throws Exception { storage = new MockBase(tsdb, client, true, true, true, true); + setupStorage(tsdb, storage); + } + + @Test + public void ctorDefault() throws Exception { + query = new TSUIDQuery(tsdb); + assertNotNull(query); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNullTSDB() throws Exception { + query = new TSUIDQuery(null); + } + + @Test + public void ctorTSUID() throws Exception { + query = new TSUIDQuery(tsdb, TSUID); + assertNotNull(query); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNullTSUID() throws Exception { + query = new TSUIDQuery(tsdb, null); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNullTSDBforTSUID() throws Exception { + query = new TSUIDQuery(null, TSUID); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorEmptyTSUID() throws Exception { + query = new TSUIDQuery(tsdb, new byte[] { }); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorShortTSUID() throws Exception { + query = new TSUIDQuery(tsdb, new byte[] { 0, 0, 1, 0, 0, 1 }); + } + + @Test + public void ctorMetric() throws Exception { + query = new TSUIDQuery(tsdb, METRIC_STRING, tags); + assertNotNull(query); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorMetricNullTSDB() throws Exception { + query = new TSUIDQuery(null, METRIC_STRING, tags); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorMetricNullMetric() throws Exception { + query = new TSUIDQuery(tsdb, null, tags); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorMetricEmptyMetric() throws Exception { + query = new TSUIDQuery(tsdb, "", tags); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorMetricNullTags() throws Exception { + query = new TSUIDQuery(tsdb, METRIC_STRING, null); + } + + @Test + public void ctorMetricEmptyTags() throws Exception { + tags.clear(); + query = new TSUIDQuery(tsdb, METRIC_STRING, tags); + assertNotNull(query); + } + + @Test + public void getLastWriteTimes() throws Exception { + query = new TSUIDQuery(tsdb, METRIC_STRING, tags); + final ByteMap<Long> tsuids = query.getLastWriteTimes().joinUninterruptibly(); + assertEquals(1, tsuids.size()); + assertEquals(1388534400013L, (long)tsuids.get(TSUID)); + } + + @Test + public void getLastWriteTimesSetQuery() throws Exception { + query = new TSUIDQuery(tsdb); + query.setQuery(METRIC_STRING, tags); + final ByteMap<Long> tsuids = query.getLastWriteTimes().joinUninterruptibly(); + assertEquals(1, tsuids.size()); + assertEquals(1388534400013L, (long)tsuids.get(TSUID)); + } + + @Test + public void getLastWriteTimesEmptyTags() throws Exception { + tags.clear(); + query = new TSUIDQuery(tsdb, METRIC_STRING, tags); + final ByteMap<Long> tsuids = query.getLastWriteTimes().joinUninterruptibly(); + assertEquals(2, tsuids.size()); + assertEquals(1388534400013L, (long)tsuids.get(TSUID)); + assertEquals(1388534400015L, + (long)tsuids.get(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 2 })); + } + + @Test + public void getLastWriteTimesEmptyTagsSetQuery() throws Exception { + tags.clear(); + query = new TSUIDQuery(tsdb); + query.setQuery(METRIC_STRING, tags); + final ByteMap<Long> tsuids = query.getLastWriteTimes().joinUninterruptibly(); + assertEquals(2, tsuids.size()); + assertEquals(1388534400013L, (long)tsuids.get(TSUID)); + assertEquals(1388534400015L, + (long)tsuids.get(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 2 })); + } + + @Test (expected = IllegalArgumentException.class) + public void getLastWriteTimesQueryNotSet() throws Exception { + query = new TSUIDQuery(tsdb); + query.getLastWriteTimes().joinUninterruptibly(); + } + + @Test + public void getLastWriteTimesNoMatch() throws Exception { + storage.flushStorage(); + query = new TSUIDQuery(tsdb, METRIC_STRING, tags); + final ByteMap<Long> tsuids = query.getLastWriteTimes().joinUninterruptibly(); + assertTrue(tsuids.isEmpty()); + } + + @Test (expected = NoSuchUniqueName.class) + public void getLastWriteTimesNSUNMetric() throws Exception { + query = new TSUIDQuery(tsdb, NSUN_METRIC, tags); + query.getLastWriteTimes().joinUninterruptibly(); + } + + @Test (expected = DeferredGroupException.class) + public void getLastWriteTimesNSUNTagk() throws Exception { + tags.clear(); + tags.put("dc", TAGV_STRING); + query = new TSUIDQuery(tsdb, METRIC_STRING, tags); + query.getLastWriteTimes().joinUninterruptibly(); + } + + @Test (expected = DeferredGroupException.class) + public void getLastWriteTimesNSUNTagv() throws Exception { + tags.put(TAGK_STRING, "web03"); + query = new TSUIDQuery(tsdb, METRIC_STRING, tags); + query.getLastWriteTimes().joinUninterruptibly(); + } + + @Test + public void getTSMetasSingle() throws Exception { + query = new TSUIDQuery(tsdb, METRIC_STRING, tags); + final List<TSMeta> tsmetas = query.getTSMetas().joinUninterruptibly(); + assertEquals(1, tsmetas.size()); + assertEquals(METRIC_STRING, tsmetas.get(0).getMetric().getName()); + assertEquals(TAGK_STRING, tsmetas.get(0).getTags().get(0).getName()); + assertEquals(TAGV_STRING, tsmetas.get(0).getTags().get(1).getName()); + } + + @Test + public void getTSMetasSingleSetQuery() throws Exception { + query = new TSUIDQuery(tsdb); + query.setQuery(METRIC_STRING, tags); + final List<TSMeta> tsmetas = query.getTSMetas().joinUninterruptibly(); + assertEquals(1, tsmetas.size()); + assertEquals(METRIC_STRING, tsmetas.get(0).getMetric().getName()); + assertEquals(TAGK_STRING, tsmetas.get(0).getTags().get(0).getName()); + assertEquals(TAGV_STRING, tsmetas.get(0).getTags().get(1).getName()); + } + + @Test + public void getTSMetasMultipleResults() throws Exception { + tags.clear(); + query = new TSUIDQuery(tsdb, METRIC_STRING, tags); + final List<TSMeta> tsmetas = query.getTSMetas().joinUninterruptibly(); + assertEquals(2, tsmetas.size()); + assertEquals(METRIC_STRING, tsmetas.get(0).getMetric().getName()); + assertEquals(TAGK_STRING, tsmetas.get(0).getTags().get(0).getName()); + assertEquals(TAGV_STRING, tsmetas.get(0).getTags().get(1).getName()); + assertEquals(METRIC_STRING, tsmetas.get(1).getMetric().getName()); + assertEquals(TAGK_STRING, tsmetas.get(1).getTags().get(0).getName()); + assertEquals(TAGV_B_STRING, tsmetas.get(1).getTags().get(1).getName()); + } + + @Test + public void getTSMetasMultipleTags() throws Exception { + tags.put(TAGK_B_STRING, TAGV_B_STRING); + query = new TSUIDQuery(tsdb, METRIC_B_STRING, tags); + + final List<TSMeta> tsmetas = query.getTSMetas().joinUninterruptibly(); + assertEquals(1, tsmetas.size()); + assertEquals(METRIC_B_STRING, tsmetas.get(0).getMetric().getName()); + assertEquals(TAGK_STRING, tsmetas.get(0).getTags().get(0).getName()); + assertEquals(TAGV_STRING, tsmetas.get(0).getTags().get(1).getName()); + assertEquals(TAGK_B_STRING, tsmetas.get(0).getTags().get(2).getName()); + assertEquals(TAGV_B_STRING, tsmetas.get(0).getTags().get(3).getName()); + } + + @Test (expected = DeferredGroupException.class) + public void getTSMetasNSUITagk() throws Exception { + tags.put(NSUN_TAGK, TAGV_B_STRING); + query = new TSUIDQuery(tsdb, METRIC_B_STRING, tags); + query.getTSMetas().joinUninterruptibly(); + } + + @Test (expected = IllegalArgumentException.class) + public void getTSMetasNullMetric() throws Exception { + query = new TSUIDQuery(tsdb); + query.getTSMetas().joinUninterruptibly(); + } + + @Test + public void tsuidFromMetric() throws Exception { + byte[] tsuid = TSUIDQuery.tsuidFromMetric(tsdb, METRIC_STRING, tags).join(); + assertArrayEquals(TSUID, tsuid); + } + + @Test + public void tsuidFromMetricTwoTags() throws Exception { + tags.put(TAGK_B_STRING, TAGV_B_STRING); + byte[] tsuid = TSUIDQuery.tsuidFromMetric(tsdb, METRIC_STRING, tags).join(); + assertArrayEquals( + new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 3, 0, 0, 2 }, tsuid); + } + + @Test (expected = NoSuchUniqueName.class) + public void tsuidFromMetricNSUNMetric() throws Exception { + TSUIDQuery.tsuidFromMetric(tsdb, NSUN_METRIC, tags).join(); + } + + @Test (expected = DeferredGroupException.class) + public void tsuidFromMetricNSUNTagk() throws Exception { + tags.clear(); + tags.put("dc", TAGV_STRING); + TSUIDQuery.tsuidFromMetric(tsdb, METRIC_STRING, tags).join(); + } + + @Test (expected = DeferredGroupException.class) + public void tsuidFromMetricNSUNTagv() throws Exception { + tags.put(TAGK_STRING, "web03"); + TSUIDQuery.tsuidFromMetric(tsdb, METRIC_STRING, tags).join(); + } + + @Test (expected = NullPointerException.class) + public void tsuidFromMetricNullTSDB() throws Exception { + TSUIDQuery.tsuidFromMetric(null, METRIC_STRING, tags).join(); + } + + @Test (expected = IllegalArgumentException.class) + public void tsuidFromMetricNullMetric() throws Exception { + TSUIDQuery.tsuidFromMetric(tsdb, null, tags).join(); + } + + @Test (expected = IllegalArgumentException.class) + public void tsuidFromMetricEmptyMetric() throws Exception { + TSUIDQuery.tsuidFromMetric(tsdb, "", tags).join(); + } + + @Test (expected = IllegalArgumentException.class) + public void tsuidFromMetricNullTags() throws Exception { + TSUIDQuery.tsuidFromMetric(tsdb, METRIC_STRING, null).join(); + } + + @Test (expected = IllegalArgumentException.class) + public void tsuidFromMetricEmptyTags() throws Exception { + tags.clear(); + TSUIDQuery.tsuidFromMetric(tsdb, METRIC_STRING, tags).join(); + } + + @Test + public void getLastPointMetricZeroBackscanOnePoint() throws Exception { + Whitebox.setInternalState(config, "enable_tsuid_incrementing", false); + Whitebox.setInternalState(config, "enable_realtime_ts", false); + storage.flushStorage(); + tsdb.addPoint(METRIC_STRING, 1356998400L, 42, tags); + + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + query = new TSUIDQuery(tsdb, METRIC_STRING, tags); + final IncomingDataPoint dp = query.getLastPoint(false, 0).join(); + assertEquals(1356998400000L, dp.getTimestamp()); + assertNull(dp.getMetric()); + assertNull(dp.getTags()); + assertEquals("42", dp.getValue()); + assertEquals(UniqueId.uidToString(TSUID), dp.getTSUID()); + } + + @Test + public void getLastPointMetricZeroBackscanMostRecent() throws Exception { + Whitebox.setInternalState(config, "enable_tsuid_incrementing", false); + Whitebox.setInternalState(config, "enable_realtime_ts", false); + storage.flushStorage(); + tsdb.addPoint(METRIC_STRING, 1356998400L, 42, tags); + tsdb.addPoint(METRIC_STRING, 1356998401L, 24, tags); + tsdb.addPoint(METRIC_STRING, 1356998402L, 1, tags); + + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + query = new TSUIDQuery(tsdb, METRIC_STRING, tags); + final IncomingDataPoint dp = query.getLastPoint(false, 0).join(); + assertEquals(1356998402000L, dp.getTimestamp()); + assertNull(dp.getMetric()); + assertNull(dp.getTags()); + assertEquals("1", dp.getValue()); + assertEquals(UniqueId.uidToString(TSUID), dp.getTSUID()); + } + + @Test + public void getLastPointMetricZeroBackscanOutOfRange() throws Exception { + Whitebox.setInternalState(config, "enable_tsuid_incrementing", false); + Whitebox.setInternalState(config, "enable_realtime_ts", false); + storage.flushStorage(); + tsdb.addPoint(METRIC_STRING, 1356998400L, 42, tags); - storage.addColumn(new byte[] { 0, 0, 1 }, NAME_FAMILY, + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1357002000000L); + query = new TSUIDQuery(tsdb, METRIC_STRING, tags); + assertNull(query.getLastPoint(false, 0).join()); + } + + @Test + public void getLastPointMetricOneBackscanInRange() throws Exception { + Whitebox.setInternalState(config, "enable_tsuid_incrementing", false); + Whitebox.setInternalState(config, "enable_realtime_ts", false); + storage.flushStorage(); + tsdb.addPoint(METRIC_STRING, 1356998400L, 42, tags); + + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1357002000000L); + query = new TSUIDQuery(tsdb, METRIC_STRING, tags); + final IncomingDataPoint dp = query.getLastPoint(false, 1).join(); + assertEquals(1356998400000L, dp.getTimestamp()); + assertNull(dp.getMetric()); + assertNull(dp.getTags()); + assertEquals("42", dp.getValue()); + assertEquals(UniqueId.uidToString(TSUID), dp.getTSUID()); + } + + @Test + public void getLastPointMetricOneBackscanOutOfRange() throws Exception { + Whitebox.setInternalState(config, "enable_tsuid_incrementing", false); + Whitebox.setInternalState(config, "enable_realtime_ts", false); + storage.flushStorage(); + tsdb.addPoint(METRIC_STRING, 1356998400L, 42, tags); + + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1357010600000L); + query = new TSUIDQuery(tsdb, METRIC_STRING, tags); + assertNull(query.getLastPoint(false, 1).join()); + } + + @Test + public void getLastPointMetricManyBackscanInRange() throws Exception { + Whitebox.setInternalState(config, "enable_tsuid_incrementing", false); + Whitebox.setInternalState(config, "enable_realtime_ts", false); + storage.flushStorage(); + tsdb.addPoint(METRIC_STRING, 1356998400L, 42, tags); + + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1360681200000L); + query = new TSUIDQuery(tsdb, METRIC_STRING, tags); + final IncomingDataPoint dp = query.getLastPoint(false, 1024).join(); + assertEquals(1356998400000L, dp.getTimestamp()); + assertNull(dp.getMetric()); + assertNull(dp.getTags()); + assertEquals("42", dp.getValue()); + assertEquals(UniqueId.uidToString(TSUID), dp.getTSUID()); + } + + @Test + public void getLastPointMetricManyBackscanOutOfRange() throws Exception { + Whitebox.setInternalState(config, "enable_tsuid_incrementing", false); + Whitebox.setInternalState(config, "enable_realtime_ts", false); + storage.flushStorage(); + tsdb.addPoint(METRIC_STRING, 1356998400L, 42, tags); + + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1360681200000L); + query = new TSUIDQuery(tsdb, METRIC_STRING, tags); + assertNull(query.getLastPoint(false, 1022).join()); + } + + @Test (expected = IllegalArgumentException.class) + public void getLastPointMetricNegativeBackscan() throws Exception { + query = new TSUIDQuery(tsdb, METRIC_STRING, tags); + query.getLastPoint(false, -1).join(); + } + + @Test + public void getLastPointMetricResolve() throws Exception { + Whitebox.setInternalState(config, "enable_tsuid_incrementing", false); + Whitebox.setInternalState(config, "enable_realtime_ts", false); + storage.flushStorage(); + tsdb.addPoint(METRIC_STRING, 1356998400L, 42, tags); + + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + query = new TSUIDQuery(tsdb, METRIC_STRING, tags); + final IncomingDataPoint dp = query.getLastPoint(true, 0).join(); + assertEquals(1356998400000L, dp.getTimestamp()); + assertEquals(METRIC_STRING, dp.getMetric()); + assertSame(tags, dp.getTags()); + assertEquals("42", dp.getValue()); + assertEquals(UniqueId.uidToString(TSUID), dp.getTSUID()); + } + + @Test (expected = NoSuchUniqueName.class) + public void getLastPointMetricNSUNMetric() throws Exception { + Whitebox.setInternalState(config, "enable_tsuid_incrementing", false); + Whitebox.setInternalState(config, "enable_realtime_ts", false); + storage.flushStorage(); + tsdb.addPoint(NSUN_METRIC, 1356998400L, 42, tags); + + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + query = new TSUIDQuery(tsdb, METRIC_STRING, tags); + query.getLastPoint(false, 0).join(); + } + + @Test (expected = DeferredGroupException.class) + public void getLastPointMetricNSUNTagk() throws Exception { + tags.clear(); + tags.put(NSUN_TAGK, TAGV_STRING); + Whitebox.setInternalState(config, "enable_tsuid_incrementing", false); + Whitebox.setInternalState(config, "enable_realtime_ts", false); + + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + query = new TSUIDQuery(tsdb, METRIC_STRING, tags); + query.getLastPoint(false, 0).join(); + } + + @Test (expected = DeferredGroupException.class) + public void getLastPointMetricNSUNTagv() throws Exception { + tags.put(TAGK_STRING, NSUN_TAGV); + Whitebox.setInternalState(config, "enable_tsuid_incrementing", false); + Whitebox.setInternalState(config, "enable_realtime_ts", false); + + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + query = new TSUIDQuery(tsdb, METRIC_STRING, tags); + query.getLastPoint(false, 0).join(); + } + + @Test (expected = IllegalArgumentException.class) + public void getLastPointMetricEmptyTags() throws Exception { + Whitebox.setInternalState(config, "enable_tsuid_incrementing", false); + Whitebox.setInternalState(config, "enable_realtime_ts", false); + storage.flushStorage(); + tags.clear(); + tsdb.addPoint(METRIC_STRING, 1356998400L, 42, tags); + + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + query = new TSUIDQuery(tsdb, METRIC_STRING, tags); + query.getLastPoint(false, 0).join(); + } + + @Test + public void getLastPointTSUIDZeroBackscanRecent() throws Exception { + Whitebox.setInternalState(config, "enable_tsuid_incrementing", false); + Whitebox.setInternalState(config, "enable_realtime_ts", false); + storage.flushStorage(); + tsdb.addPoint(METRIC_STRING, 1356998400L, 42, tags); + + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + query = new TSUIDQuery(tsdb, TSUID); + final IncomingDataPoint dp = query.getLastPoint(false, 0).join(); + assertEquals(1356998400000L, dp.getTimestamp()); + assertNull(dp.getMetric()); + assertNull(dp.getTags()); + assertEquals("42", dp.getValue()); + assertEquals(UniqueId.uidToString(TSUID), dp.getTSUID()); + } + + @Test + public void getLastPointTSUIDZeroBackscanRecentOutOfRange() throws Exception { + Whitebox.setInternalState(config, "enable_tsuid_incrementing", false); + Whitebox.setInternalState(config, "enable_realtime_ts", false); + storage.flushStorage(); + tsdb.addPoint(METRIC_STRING, 1356998400L, 42, tags); + + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1357002000000L); + query = new TSUIDQuery(tsdb, TSUID); + assertNull(query.getLastPoint(false, 0).join()); + } + + @Test + public void getLastPointTSUIDOneBackscanInRange() throws Exception { + Whitebox.setInternalState(config, "enable_tsuid_incrementing", false); + Whitebox.setInternalState(config, "enable_realtime_ts", false); + storage.flushStorage(); + tsdb.addPoint(METRIC_STRING, 1356998400L, 42, tags); + + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1357002000000L); + query = new TSUIDQuery(tsdb, TSUID); + final IncomingDataPoint dp = query.getLastPoint(false, 1).join(); + assertEquals(1356998400000L, dp.getTimestamp()); + assertNull(dp.getMetric()); + assertNull(dp.getTags()); + assertEquals("42", dp.getValue()); + assertEquals(UniqueId.uidToString(TSUID), dp.getTSUID()); + } + + @Test + public void getLastPointTSUIDOneBackscanRecentOutOfRange() throws Exception { + Whitebox.setInternalState(config, "enable_tsuid_incrementing", false); + Whitebox.setInternalState(config, "enable_realtime_ts", false); + storage.flushStorage(); + tsdb.addPoint(METRIC_STRING, 1356998400L, 42, tags); + + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1357010600000L); + query = new TSUIDQuery(tsdb, TSUID); + assertNull(query.getLastPoint(false, 1).join()); + } + + @Test + public void getLastPointTSUIDManyBackscanInRange() throws Exception { + Whitebox.setInternalState(config, "enable_tsuid_incrementing", false); + Whitebox.setInternalState(config, "enable_realtime_ts", false); + storage.flushStorage(); + tsdb.addPoint(METRIC_STRING, 1356998400L, 42, tags); + + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1360681200000L); + query = new TSUIDQuery(tsdb, TSUID); + final IncomingDataPoint dp = query.getLastPoint(false, 1024).join(); + assertEquals(1356998400000L, dp.getTimestamp()); + assertNull(dp.getMetric()); + assertNull(dp.getTags()); + assertEquals("42", dp.getValue()); + assertEquals(UniqueId.uidToString(TSUID), dp.getTSUID()); + } + + @Test + public void getLastPointTSUIDManyBackscanRecentOutOfRange() throws Exception { + Whitebox.setInternalState(config, "enable_tsuid_incrementing", false); + Whitebox.setInternalState(config, "enable_realtime_ts", false); + storage.flushStorage(); + tsdb.addPoint(METRIC_STRING, 1356998400L, 42, tags); + + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1360681200000L); + query = new TSUIDQuery(tsdb, TSUID); + assertNull(query.getLastPoint(false, 1022).join()); + } + + // While these NSUI shouldn't happen, it's possible if someone deletes a metric + // or tag but not the actual data. + @Test + public void getLastPointTSUIDMetricNSUINotResolved() throws Exception { + Whitebox.setInternalState(config, "enable_tsuid_incrementing", false); + Whitebox.setInternalState(config, "enable_realtime_ts", false); + storage.flushStorage(); + storage.addColumn(MockBase.stringToBytes("00000350E22700000001000001"), + QUAL, VAL); + final byte[] tsuid = new byte[] { 0, 0, 3, 0, 0, 1, 0, 0, 1 }; + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + query = new TSUIDQuery(tsdb, tsuid); + final IncomingDataPoint dp = query.getLastPoint(false, 0).join(); + assertEquals(1356998400000L, dp.getTimestamp()); + assertNull(dp.getMetric()); + assertNull(dp.getTags()); + assertEquals("42", dp.getValue()); + assertEquals(UniqueId.uidToString(tsuid), dp.getTSUID()); + } + + @Test (expected = NoSuchUniqueId.class) + public void getLastPointTSUIDMetricNSUI() throws Exception { + Whitebox.setInternalState(config, "enable_tsuid_incrementing", false); + Whitebox.setInternalState(config, "enable_realtime_ts", false); + storage.flushStorage(); + storage.addColumn(MockBase.stringToBytes("00000350E22700000001000001"), + QUAL, VAL); + final byte[] tsuid = new byte[] { 0, 0, 3, 0, 0, 1, 0, 0, 1 }; + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + query = new TSUIDQuery(tsdb, tsuid); + query.getLastPoint(true, 0).join(); + } + + @Test + public void getLastPointTSUIDTagkNSUINotResolved() throws Exception { + Whitebox.setInternalState(config, "enable_tsuid_incrementing", false); + Whitebox.setInternalState(config, "enable_realtime_ts", false); + storage.flushStorage(); + storage.addColumn(MockBase.stringToBytes("00000150E22700000003000001"), + QUAL, VAL); + final byte[] tsuid = new byte[] { 0, 0, 1, 0, 0, 3, 0, 0, 1 }; + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + query = new TSUIDQuery(tsdb, tsuid); + final IncomingDataPoint dp = query.getLastPoint(false, 0).join(); + assertEquals(1356998400000L, dp.getTimestamp()); + assertNull(dp.getMetric()); + assertNull(dp.getTags()); + assertEquals("42", dp.getValue()); + assertEquals(UniqueId.uidToString(tsuid), dp.getTSUID()); + } + + @Test + public void getLastPointTSUIDTagkNSUI() throws Exception { + Whitebox.setInternalState(config, "enable_tsuid_incrementing", false); + Whitebox.setInternalState(config, "enable_realtime_ts", false); + storage.flushStorage(); + storage.addColumn(MockBase.stringToBytes("00000150E22700000004000001"), + QUAL, VAL); + final byte[] tsuid = new byte[] { 0, 0, 1, 0, 0, 4, 0, 0, 1 }; + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + query = new TSUIDQuery(tsdb, tsuid); + try { + query.getLastPoint(true, 0).join(); + fail("Expected DeferredGroupException"); + } catch (DeferredGroupException e) { + assertTrue(e.getCause() instanceof NoSuchUniqueId); + } + } + + @Test + public void getLastPointTSUIDTagvNSUINotResolved() throws Exception { + Whitebox.setInternalState(config, "enable_tsuid_incrementing", false); + Whitebox.setInternalState(config, "enable_realtime_ts", false); + storage.flushStorage(); + storage.addColumn(MockBase.stringToBytes("00000150E22700000001000003"), + QUAL, VAL); + final byte[] tsuid = new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 3 }; + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + query = new TSUIDQuery(tsdb, tsuid); + final IncomingDataPoint dp = query.getLastPoint(false, 0).join(); + assertEquals(1356998400000L, dp.getTimestamp()); + assertNull(dp.getMetric()); + assertNull(dp.getTags()); + assertEquals("42", dp.getValue()); + assertEquals(UniqueId.uidToString(tsuid), dp.getTSUID()); + } + + @Test + public void getLastPoitTSUIDTagvNSUI() throws Exception { + Whitebox.setInternalState(config, "enable_tsuid_incrementing", false); + Whitebox.setInternalState(config, "enable_realtime_ts", false); + storage.flushStorage(); + storage.addColumn(MockBase.stringToBytes("00000150E22700000001000003"), + QUAL, VAL); + final byte[] tsuid = new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 3 }; + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + query = new TSUIDQuery(tsdb, tsuid); + try { + query.getLastPoint(true, 0).join(); + fail("Expected DeferredGroupException"); + } catch (DeferredGroupException e) { + assertTrue(e.getCause() instanceof NoSuchUniqueId); + } + } + + @Test + public void getLastPointTSUIDMeta() throws Exception { + Whitebox.setInternalState(config, "enable_tsuid_incrementing", false); + Whitebox.setInternalState(config, "enable_realtime_ts", false); + tsdb.addPoint(METRIC_STRING, 1388534400L, 42, tags); + + Whitebox.setInternalState(config, "enable_tsuid_incrementing", true); + Whitebox.setInternalState(config, "enable_realtime_ts", true); + + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + query = new TSUIDQuery(tsdb, TSUID); + final IncomingDataPoint dp = query.getLastPoint(false, 0).join(); + assertEquals(1388534400000L, dp.getTimestamp()); + assertNull(dp.getMetric()); + assertNull(dp.getTags()); + assertEquals("42", dp.getValue()); + assertEquals(UniqueId.uidToString(TSUID), dp.getTSUID()); + } + + @Test + public void getLastPointTSUIDMetaNoPoint() throws Exception { + Whitebox.setInternalState(config, "enable_tsuid_incrementing", true); + Whitebox.setInternalState(config, "enable_realtime_ts", true); + + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + query = new TSUIDQuery(tsdb, TSUID); + assertNull(query.getLastPoint(false, 0).join()); + } + + /** + * Public for sharing with other UT classes + * @param tsdb The mock TSDB client + * @throws Exception If something went pear shaped + */ + public static void setupStorage(final TSDB tsdb, final MockBase storage) + throws Exception { + final List<byte[]> families = new ArrayList<byte[]>(); + families.add(TSMeta.FAMILY); + storage.addTable(META_TABLE, families); + + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "metrics".getBytes(MockBase.ASCII()), - "sys.cpu.user".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1 }, NAME_FAMILY, + METRIC_STRING.getBytes(MockBase.ASCII())); + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "metric_meta".getBytes(MockBase.ASCII()), ("{\"uid\":\"000001\",\"type\":\"METRIC\",\"name\":\"sys.cpu.user\"," + "\"description\":\"Description\",\"notes\":\"MyNotes\",\"created\":" + "1328140801,\"displayName\":\"System CPU\"}") .getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 2 }, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 2 }, NAME_FAMILY, "metrics".getBytes(MockBase.ASCII()), - "sys.cpu.nice".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 2 }, NAME_FAMILY, + METRIC_B_STRING.getBytes(MockBase.ASCII())); + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 2 }, NAME_FAMILY, "metric_meta".getBytes(MockBase.ASCII()), - ("{\"uid\":\"000002\",\"type\":\"METRIC\",\"name\":\"sys.cpu.nice\"," + + ("{\"uid\":\"000002\",\"type\":\"METRIC\",\"name\":\"sys.cpu.system\"," + "\"description\":\"Description\",\"notes\":\"MyNotes\",\"created\":" + "1328140801,\"displayName\":\"System CPU\"}") .getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1 }, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "tagk".getBytes(MockBase.ASCII()), - "host".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1 }, NAME_FAMILY, + TAGK_STRING.getBytes(MockBase.ASCII())); + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "tagk_meta".getBytes(MockBase.ASCII()), ("{\"uid\":\"000001\",\"type\":\"TAGK\",\"name\":\"host\"," + "\"description\":\"Description\",\"notes\":\"MyNotes\",\"created\":" + "1328140801,\"displayName\":\"Host server name\"}") .getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 2 }, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 2 }, NAME_FAMILY, "tagk".getBytes(MockBase.ASCII()), - "datacenter".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 2 }, NAME_FAMILY, + TAGK_B_STRING.getBytes(MockBase.ASCII())); + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 2 }, NAME_FAMILY, "tagk_meta".getBytes(MockBase.ASCII()), - ("{\"uid\":\"000002\",\"type\":\"TAGK\",\"name\":\"datacenter\"," + + ("{\"uid\":\"000002\",\"type\":\"TAGK\",\"name\":\"owner\"," + "\"description\":\"Description\",\"notes\":\"MyNotes\",\"created\":" + "1328140801,\"displayName\":\"Datecenter name\"}") .getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1 }, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "tagv".getBytes(MockBase.ASCII()), - "web01".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1 }, NAME_FAMILY, + TAGV_STRING.getBytes(MockBase.ASCII())); + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "tagv_meta".getBytes(MockBase.ASCII()), ("{\"uid\":\"000001\",\"type\":\"TAGV\",\"name\":\"web01\"," + "\"description\":\"Description\",\"notes\":\"MyNotes\",\"created\":" + "1328140801,\"displayName\":\"Web server 1\"}") .getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 2 }, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 2 }, NAME_FAMILY, "tagv".getBytes(MockBase.ASCII()), - "web02".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 2 }, NAME_FAMILY, + TAGV_B_STRING.getBytes(MockBase.ASCII())); + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 2 }, NAME_FAMILY, "tagv_meta".getBytes(MockBase.ASCII()), ("{\"uid\":\"000002\",\"type\":\"TAGV\",\"name\":\"web02\"," + "\"description\":\"Description\",\"notes\":\"MyNotes\",\"created\":" + "1328140801,\"displayName\":\"Web server 2\"}") .getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 3 }, NAME_FAMILY, - "tagv".getBytes(MockBase.ASCII()), - "dc01".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 3 }, NAME_FAMILY, - "tagv_meta".getBytes(MockBase.ASCII()), - ("{\"uid\":\"000003\",\"type\":\"TAGV\",\"name\":\"dc01\"," + - "\"description\":\"Description\",\"notes\":\"MyNotes\",\"created\":" + - "1328140801,\"displayName\":\"Web server 2\"}") - .getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }, NAME_FAMILY, + storage.addColumn(META_TABLE, TSUID, NAME_FAMILY, "ts_meta".getBytes(MockBase.ASCII()), ("{\"tsuid\":\"000001000001000001\",\"" + "description\":\"Description\",\"notes\":\"Notes\",\"created\":1328140800," + "\"custom\":null,\"units\":\"\",\"retention\":42,\"max\":1.0,\"min\":" + "\"NaN\",\"displayName\":\"Display\",\"dataType\":\"Data\"}") .getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }, NAME_FAMILY, + storage.addColumn(META_TABLE, TSUID, NAME_FAMILY, "ts_ctr".getBytes(MockBase.ASCII()), Bytes.fromLong(1L)); - storage.addColumn(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 2 }, NAME_FAMILY, + storage.addColumn(META_TABLE, + new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 2 }, NAME_FAMILY, "ts_meta".getBytes(MockBase.ASCII()), ("{\"tsuid\":\"000001000001000002\",\"" + "description\":\"Description\",\"notes\":\"Notes\",\"created\":1328140800," + "\"custom\":null,\"units\":\"\",\"retention\":42,\"max\":1.0,\"min\":" + "\"NaN\",\"displayName\":\"Display\",\"dataType\":\"Data\"}") .getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 2 }, NAME_FAMILY, + storage.addColumn(META_TABLE, + new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 2 }, NAME_FAMILY, "ts_ctr".getBytes(MockBase.ASCII()), Bytes.fromLong(1L)); - storage.addColumn(new byte[] { 0, 0, 2, 0, 0, 2, 0, 0, 3, 0, 0, 1, 0, 0, 1 }, + storage.addColumn(META_TABLE, + new byte[] { 0, 0, 2, 0, 0, 1, 0, 0, 1, 0, 0, 3, 0, 0, 2 }, NAME_FAMILY, "ts_meta".getBytes(MockBase.ASCII()), - ("{\"tsuid\":\"000002000002000003000001000001\",\"" + + ("{\"tsuid\":\"000002000001000001000003000002\",\"" + "description\":\"Description\",\"notes\":\"Notes\",\"created\":1328140800," + "\"custom\":null,\"units\":\"\",\"retention\":42,\"max\":1.0,\"min\":" + "\"NaN\",\"displayName\":\"Display\",\"dataType\":\"Data\"}") .getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 2, 0, 0, 2, 0, 0, 3, 0, 0, 1, 0, 0, 1 }, + storage.addColumn(META_TABLE, + new byte[] { 0, 0, 2, 0, 0, 1, 0, 0, 1, 0, 0, 3, 0, 0, 2 }, NAME_FAMILY, "ts_ctr".getBytes(MockBase.ASCII()), Bytes.fromLong(1L)); - - // replace the "real" field objects with mocks - Field cl = tsdb.getClass().getDeclaredField("client"); - cl.setAccessible(true); - cl.set(tsdb, client); - - Field met = tsdb.getClass().getDeclaredField("metrics"); - met.setAccessible(true); - met.set(tsdb, metrics); - - Field tagk = tsdb.getClass().getDeclaredField("tag_names"); - tagk.setAccessible(true); - tagk.set(tsdb, tag_names); - - Field tagv = tsdb.getClass().getDeclaredField("tag_values"); - tagv.setAccessible(true); - tagv.set(tsdb, tag_values); - - // mock UniqueId - when(metrics.getId("sys.cpu.user")).thenReturn(new byte[] { 0, 0, 1 }); - when(metrics.getNameAsync(new byte[] { 0, 0, 1 })) - .thenReturn(Deferred.fromResult("sys.cpu.user")); - when(metrics.getId("sys.cpu.system")) - .thenThrow(new NoSuchUniqueName("sys.cpu.system", "metric")); - when(metrics.getId("sys.cpu.nice")).thenReturn(new byte[] { 0, 0, 2 }); - when(metrics.getNameAsync(new byte[] { 0, 0, 2 })) - .thenReturn(Deferred.fromResult("sys.cpu.nice")); - - when(tag_names.getId("host")).thenReturn(new byte[] { 0, 0, 1 }); - when(tag_names.getIdAsync("host")).thenReturn( - Deferred.fromResult(new byte[] { 0, 0, 1 })); - when(tag_names.getNameAsync(new byte[] { 0, 0, 1 })) - .thenReturn(Deferred.fromResult("host")); - when(tag_names.getOrCreateIdAsync("host")).thenReturn( - Deferred.fromResult(new byte[] { 0, 0, 1 })); - when(tag_names.getId("dc")) - .thenThrow(new NoSuchUniqueName("dc", "metric")); - when(tag_names.getId("datacenter")).thenReturn(new byte[] { 0, 0, 2 }); - when(tag_names.getIdAsync("datacenter")) - .thenReturn(Deferred.fromResult(new byte[] { 0, 0, 2 })); - when(tag_names.getNameAsync(new byte[] { 0, 0, 2 })) - .thenReturn(Deferred.fromResult("datacenter")); - - when(tag_values.getId("web01")).thenReturn(new byte[] { 0, 0, 1 }); - when(tag_values.getIdAsync("web01")).thenReturn( - Deferred.fromResult(new byte[] { 0, 0, 1 })); - when(tag_values.getNameAsync(new byte[] { 0, 0, 1 })) - .thenReturn(Deferred.fromResult("web01")); - when(tag_values.getOrCreateIdAsync("web01")).thenReturn( - Deferred.fromResult(new byte[] { 0, 0, 1 })); - when(tag_values.getId("web02")).thenReturn(new byte[] { 0, 0, 2 }); - when(tag_values.getIdAsync("web02")).thenReturn( - Deferred.fromResult(new byte[] { 0, 0, 2 })); - when(tag_values.getNameAsync(new byte[] { 0, 0, 2 })) - .thenReturn(Deferred.fromResult("web02")); - when(tag_values.getOrCreateIdAsync("web02")).thenReturn( - Deferred.fromResult(new byte[] { 0, 0, 2 })); - when(tag_values.getId("web03")) - .thenThrow(new NoSuchUniqueName("web03", "metric")); - when(tag_values.getId("dc01")).thenReturn(new byte[] { 0, 0, 3 }); - when(tag_values.getIdAsync("dc01")) - .thenReturn(Deferred.fromResult(new byte[] { 0, 0, 3 })); - when(tag_values.getNameAsync(new byte[] { 0, 0, 3 })) - .thenReturn(Deferred.fromResult("dc01")); - - when(metrics.width()).thenReturn((short)3); - when(tag_names.width()).thenReturn((short)3); - when(tag_values.width()).thenReturn((short)3); - } - - @Test - public void setQuery() throws Exception { - query = new TSUIDQuery(tsdb); - final HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web01"); - query.setQuery("sys.cpu.user", tags); - } - - @Test - public void setQueryEmtpyTags() throws Exception { - query = new TSUIDQuery(tsdb); - query.setQuery("sys.cpu.user", new HashMap<String, String>(0)); } - - @Test (expected = NoSuchUniqueName.class) - public void setQueryNSUMetric() throws Exception { - query = new TSUIDQuery(tsdb); - query.setQuery("sys.cpu.system", new HashMap<String, String>(0)); - } - - @Test (expected = NoSuchUniqueName.class) - public void setQueryNSUTagk() throws Exception { - query = new TSUIDQuery(tsdb); - final HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("dc", "web01"); - query.setQuery("sys.cpu.user", tags); - } - - @Test (expected = NoSuchUniqueName.class) - public void setQueryNSUTagv() throws Exception { - query = new TSUIDQuery(tsdb); - final HashMap<String, String> tags = new HashMap<String, String>(1); - tags.put("host", "web03"); - query.setQuery("sys.cpu.user", tags); - } - - @Test (expected = IllegalArgumentException.class) - public void getLastWriteTimesQueryNotSet() throws Exception { - query = new TSUIDQuery(tsdb); - query.getLastWriteTimes().joinUninterruptibly(); - } - - @Test - public void getTSMetasSingle() throws Exception { - query = new TSUIDQuery(tsdb); - HashMap<String, String> tags = new HashMap<String, String>(); - tags.put("host", "web01"); - query.setQuery("sys.cpu.user", tags); - List<TSMeta> tsmetas = query.getTSMetas().joinUninterruptibly(); - assertEquals(1, tsmetas.size()); - } - - @Test - public void getTSMetasMulti() throws Exception { - query = new TSUIDQuery(tsdb); - HashMap<String, String> tags = new HashMap<String, String>(); - query.setQuery("sys.cpu.user", tags); - List<TSMeta> tsmetas = query.getTSMetas().joinUninterruptibly(); - assertEquals(2, tsmetas.size()); - } - - @Test - public void getTSMetasMultipleTags() throws Exception { - query = new TSUIDQuery(tsdb); - HashMap<String, String> tags = new HashMap<String, String>(); - query.setQuery("sys.cpu.nice", tags); - tags.put("host", "web01"); - tags.put("datacenter", "dc01"); - List<TSMeta> tsmetas = query.getTSMetas().joinUninterruptibly(); - assertEquals(1, tsmetas.size()); - } - - @Test (expected = IllegalArgumentException.class) - public void getTSMetasNullMetric() throws Exception { - query = new TSUIDQuery(tsdb); - query.getTSMetas().joinUninterruptibly(); - } - } diff --git a/test/meta/TestUIDMeta.java b/test/meta/TestUIDMeta.java index 85e5c0a977..c53338706c 100644 --- a/test/meta/TestUIDMeta.java +++ b/test/meta/TestUIDMeta.java @@ -15,7 +15,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.when; import static org.powermock.api.mockito.PowerMockito.mock; @@ -49,7 +48,8 @@ GetRequest.class, PutRequest.class, DeleteRequest.class, KeyValue.class, Scanner.class, UIDMeta.class}) public final class TestUIDMeta { - private static byte[] NAME_FAMILY = "name".getBytes(MockBase.ASCII()); + private final static byte[] NAME_FAMILY = "name".getBytes(MockBase.ASCII()); + private final static byte[] UID_TABLE = "tsdb-uid".getBytes(MockBase.ASCII()); private TSDB tsdb; private HBaseClient client = mock(HBaseClient.class); private MockBase storage; @@ -58,23 +58,21 @@ public final class TestUIDMeta { @Before public void before() throws Exception { final Config config = new Config(false); - PowerMockito.whenNew(HBaseClient.class) - .withArguments(anyString(), anyString()).thenReturn(client); - tsdb = new TSDB(config); + tsdb = new TSDB(client, config); storage = new MockBase(tsdb, client, true, true, true, true); - storage.addColumn(new byte[] { 0, 0, 1 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "metrics".getBytes(MockBase.ASCII()), "sys.cpu.0".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 3 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 3 }, NAME_FAMILY, "metrics".getBytes(MockBase.ASCII()), "sys.cpu.2".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "metric_meta".getBytes(MockBase.ASCII()), ("{\"uid\":\"000001\",\"type\":\"METRIC\",\"name\":\"sys.cpu.0\"," + @@ -253,7 +251,7 @@ public void storeNew() throws Exception { meta = new UIDMeta(UniqueIdType.METRIC, new byte[] { 0, 0, 1 }, "sys.cpu.1"); meta.setDisplayName("System CPU"); meta.storeNew(tsdb).joinUninterruptibly(); - meta = JSON.parseToObject(storage.getColumn(new byte[] { 0, 0, 1 }, + meta = JSON.parseToObject(storage.getColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "metric_meta".getBytes(MockBase.ASCII())), UIDMeta.class); assertEquals("System CPU", meta.getDisplayName()); diff --git a/test/query/TestQueryLimitOverride.java b/test/query/TestQueryLimitOverride.java new file mode 100644 index 0000000000..351da751e7 --- /dev/null +++ b/test/query/TestQueryLimitOverride.java @@ -0,0 +1,318 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.concurrent.TimeUnit; + +import org.jboss.netty.util.HashedWheelTimer; +import org.jboss.netty.util.TimerTask; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.google.common.io.Files; + +import net.opentsdb.core.TSDB; +import net.opentsdb.query.QueryLimitOverride.QueryLimitOverrideItem; +import net.opentsdb.utils.Config; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ TSDB.class, HashedWheelTimer.class, QueryLimitOverride.class, + File.class, Files.class }) +public class TestQueryLimitOverride { + + private TSDB tsdb; + private Config config; + private HashedWheelTimer timer; + private File file; + + @Before + public void before() throws Exception { + tsdb = PowerMockito.mock(TSDB.class); + config = new Config(false); + timer = mock(HashedWheelTimer.class); + PowerMockito.when(tsdb.getTimer()).thenReturn(timer); + when(tsdb.getConfig()).thenReturn(config); + PowerMockito.mockStatic(Files.class); + file = mock(File.class); + PowerMockito.whenNew(File.class).withAnyArguments().thenReturn(file); + + config.overrideConfig("tsd.query.limits.bytes.default", "42"); + config.overrideConfig("tsd.query.limits.data_points.default", "24"); + config.overrideConfig("tsd.query.limits.overrides.config", "/tmp/overrides.json"); + } + + @Test + public void ctorNoFileConfigured() throws Exception { + config.overrideConfig("tsd.query.limits.overrides.config", null); + final QueryLimitOverride limits = new QueryLimitOverride(tsdb); + assertEquals(42, limits.getDefaultByteLimit()); + assertEquals(24, limits.getDefaultDataPointsLimit()); + assertEquals(0, limits.getLimits().size()); + verify(file, never()).exists(); + verify(timer, never()) + .newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class)); + } + + @Test + public void ctorFileNoReload() throws Exception { + config.overrideConfig("tsd.query.limits.overrides.interval", null); + final QueryLimitOverride limits = new QueryLimitOverride(tsdb); + assertEquals(42, limits.getDefaultByteLimit()); + assertEquals(24, limits.getDefaultDataPointsLimit()); + verify(file, times(1)).exists(); + verify(timer, never()) + .newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class)); + } + + @Test + public void ctorFileDoesntExistException() throws Exception { + PowerMockito.when(Files.getFileExtension(anyString())) + .thenThrow(new IllegalStateException("Boo!")); + final QueryLimitOverride limits = new QueryLimitOverride(tsdb); + verify(file, times(1)).exists(); + assertEquals(42, limits.getDefaultByteLimit()); + assertEquals(24, limits.getDefaultDataPointsLimit()); + assertEquals(0, limits.getLimits().size()); + verify(timer, times(1)) + .newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class)); + } + + @Test + public void ctorNegativeDefaultsLimit() throws Exception { + config.overrideConfig("tsd.query.limits.bytes.default", "-42"); + config.overrideConfig("tsd.query.limits.data_points.default", "24"); + try { + new QueryLimitOverride(tsdb); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + + config.overrideConfig("tsd.query.limits.bytes.default", "42"); + config.overrideConfig("tsd.query.limits.data_points.default", "-24"); + try { + new QueryLimitOverride(tsdb); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + } + + @Test + public void ctorWithFile() throws Exception { + PowerMockito.when(Files.class, "toString", any(File.class), any(Charset.class)) + .thenReturn("[{\"regex\":\".*sys$\",\"byteLimit\":24,\"dataPointsLimit\":16}]"); + when(file.exists()).thenReturn(true); + final QueryLimitOverride limits = new QueryLimitOverride(tsdb); + verify(file, times(1)).exists(); + assertEquals(42, limits.getDefaultByteLimit()); + assertEquals(24, limits.getDefaultDataPointsLimit()); + assertEquals(1, limits.getLimits().size()); + final QueryLimitOverrideItem item = limits.getLimits().iterator().next(); + assertEquals(24, item.getByteLimit()); + assertEquals(16, item.getDataPointsLimit()); + assertEquals(".*sys$", item.getRegex()); + assertTrue(item.matches("namespace.app.sys")); + assertFalse(item.matches("namespace.app.sys.cpu")); + verify(timer, times(1)) + .newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class)); + } + + @Test + public void ctorWithFileException() throws Exception { + PowerMockito.when(Files.class, "toString", any(File.class), any(Charset.class)) + .thenThrow(new IOException("Boo!")); + when(file.exists()).thenReturn(true); + final QueryLimitOverride limits = new QueryLimitOverride(tsdb); + verify(file, times(1)).exists(); + assertEquals(42, limits.getDefaultByteLimit()); + assertEquals(24, limits.getDefaultDataPointsLimit()); + assertEquals(0, limits.getLimits().size()); + verify(timer, times(1)) + .newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class)); + } + + @Test + public void ctorWithFileBadJSON() throws Exception { + PowerMockito.when(Files.class, "toString", any(File.class), any(Charset.class)) + .thenReturn("[{\"regex\":\".*sys$\",\"byteLim"); + when(file.exists()).thenReturn(true); + final QueryLimitOverride limits = new QueryLimitOverride(tsdb); + verify(file, times(1)).exists(); + assertEquals(42, limits.getDefaultByteLimit()); + assertEquals(24, limits.getDefaultDataPointsLimit()); + assertEquals(0, limits.getLimits().size()); + verify(timer, times(1)) + .newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class)); + } + + @Test + public void ctorWithFileBadRegex() throws Exception { + PowerMockito.when(Files.class, "toString", any(File.class), any(Charset.class)) + .thenReturn("[{\"regex\":\".*sy(notclosed\",\"byteLimit\":24," + + "\"dataPointsLimit\":16}]"); + when(file.exists()).thenReturn(true); + final QueryLimitOverride limits = new QueryLimitOverride(tsdb); + verify(file, times(1)).exists(); + assertEquals(42, limits.getDefaultByteLimit()); + assertEquals(24, limits.getDefaultDataPointsLimit()); + assertEquals(0, limits.getLimits().size()); + verify(timer, times(1)) + .newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class)); + } + + @Test + public void timerTaskEmptySet() throws Exception { + PowerMockito.when(Files.class, "toString", any(File.class), any(Charset.class)) + .thenReturn("[{\"regex\":\".*sys$\",\"byteLimit\":24," + + "\"dataPointsLimit\":16}]"); + when(file.exists()) + .thenReturn(false) + .thenReturn(true); + final QueryLimitOverride limits = new QueryLimitOverride(tsdb); + assertEquals(0, limits.getLimits().size()); + limits.run(null); + assertEquals(1, limits.getLimits().size()); + final QueryLimitOverrideItem item = limits.getLimits().iterator().next(); + assertEquals(24, item.getByteLimit()); + assertEquals(16, item.getDataPointsLimit()); + assertEquals(".*sys$", item.getRegex()); + } + + @Test + public void timerTaskSame() throws Exception { + PowerMockito.when(Files.class, "toString", any(File.class), any(Charset.class)) + .thenReturn("[{\"regex\":\".*sys$\",\"byteLimit\":24," + + "\"dataPointsLimit\":16}]"); + when(file.exists()) + .thenReturn(true) + .thenReturn(true); + final QueryLimitOverride limits = new QueryLimitOverride(tsdb); + assertEquals(1, limits.getLimits().size()); + final QueryLimitOverrideItem item = limits.getLimits().iterator().next(); + assertEquals(24, item.getByteLimit()); + assertEquals(16, item.getDataPointsLimit()); + assertEquals(".*sys$", item.getRegex()); + + limits.run(null); + assertEquals(1, limits.getLimits().size()); + // same obj/address + assertSame(item, limits.getLimits().iterator().next()); + } + + @Test + public void timerTaskDiffLimit() throws Exception { + PowerMockito.when(Files.class, "toString", any(File.class), any(Charset.class)) + .thenReturn("[{\"regex\":\".*sys$\",\"byteLimit\":24," + + "\"dataPointsLimit\":16}]") + .thenReturn("[{\"regex\":\".*sys$\",\"byteLimit\":60," + + "\"dataPointsLimit\":16}]"); + when(file.exists()) + .thenReturn(true) + .thenReturn(true); + final QueryLimitOverride limits = new QueryLimitOverride(tsdb); + assertEquals(1, limits.getLimits().size()); + QueryLimitOverrideItem item = limits.getLimits().iterator().next(); + assertEquals(24, item.getByteLimit()); + assertEquals(16, item.getDataPointsLimit()); + assertEquals(".*sys$", item.getRegex()); + + limits.run(null); + assertEquals(1, limits.getLimits().size()); + item = limits.getLimits().iterator().next(); + assertEquals(60, item.getByteLimit()); + assertEquals(16, item.getDataPointsLimit()); + assertEquals(".*sys$", item.getRegex()); + } + + @Test + public void timerTaskCleared() throws Exception { + PowerMockito.when(Files.class, "toString", any(File.class), any(Charset.class)) + .thenReturn("[{\"regex\":\".*sys$\",\"byteLimit\":24," + + "\"dataPointsLimit\":16}]") + .thenReturn("[]"); + when(file.exists()) + .thenReturn(true) + .thenReturn(true); + final QueryLimitOverride limits = new QueryLimitOverride(tsdb); + assertEquals(1, limits.getLimits().size()); + limits.run(null); + assertEquals(0, limits.getLimits().size()); + } + + @Test + public void timerTaskAddOne() throws Exception { + PowerMockito.when(Files.class, "toString", any(File.class), any(Charset.class)) + .thenReturn("[{\"regex\":\".*sys$\",\"byteLimit\":24," + + "\"dataPointsLimit\":16}]") + .thenReturn("[{\"regex\":\".*sys$\",\"byteLimit\":24," + + "\"dataPointsLimit\":16}," + + "{\"regex\":\".*if$\",\"byteLimit\":96," + + "\"dataPointsLimit\":32}]"); + when(file.exists()) + .thenReturn(true) + .thenReturn(true); + final QueryLimitOverride limits = new QueryLimitOverride(tsdb); + assertEquals(1, limits.getLimits().size()); + limits.run(null); + assertEquals(2, limits.getLimits().size()); + } + + @Test + public void timerTaskRemoveOne() throws Exception { + PowerMockito.when(Files.class, "toString", any(File.class), any(Charset.class)) + .thenReturn("[{\"regex\":\".*sys$\",\"byteLimit\":24," + + "\"dataPointsLimit\":16}," + + "{\"regex\":\".*if$\",\"byteLimit\":96," + + "\"dataPointsLimit\":32}]") + .thenReturn("[{\"regex\":\".*sys$\",\"byteLimit\":24," + + "\"dataPointsLimit\":16}]"); + when(file.exists()) + .thenReturn(true) + .thenReturn(true); + final QueryLimitOverride limits = new QueryLimitOverride(tsdb); + assertEquals(2, limits.getLimits().size()); + limits.run(null); + assertEquals(1, limits.getLimits().size()); + } + + /** @return an override object with a couple of items */ + public QueryLimitOverride getTestObject(final boolean with_file) throws Exception { + PowerMockito.when(Files.class, "toString", any(File.class), any(Charset.class)) + .thenReturn("[{\"regex\":\".*sys$\",\"byteLimit\":24,\"dataPointsLimit\":16}," + + "{\"regex\":\".*perf.*\",\"byteLimit\":84,\"dataPointsLimit\":42}]"); + when(file.exists()).thenReturn(true); + return new QueryLimitOverride(tsdb); + } +} diff --git a/test/query/TestQueryUtil.java b/test/query/TestQueryUtil.java new file mode 100644 index 0000000000..ca0d12f25d --- /dev/null +++ b/test/query/TestQueryUtil.java @@ -0,0 +1,168 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import net.opentsdb.core.Query; +import net.opentsdb.utils.DateTime; +import org.hbase.async.Bytes.ByteMap; +import org.hbase.async.FilterList; +import org.hbase.async.KeyRegexpFilter; +import org.hbase.async.ScanFilter; +import org.hbase.async.Scanner; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.google.common.collect.Lists; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({ Scanner.class }) +public class TestQueryUtil { + private Scanner scanner; + + @Before + public void before() throws Exception { + scanner = mock(Scanner.class); + } + + @Test + public void setDataTableScanFilterNoOp() throws Exception { + QueryUtil.setDataTableScanFilter( + scanner, + Lists.<byte[]>newArrayList(), + new ByteMap<byte[][]>(), + false, + false, + 0); + verify(scanner, never()).getCurrentKey(); + verify(scanner, never()).setFilter(any(ScanFilter.class)); + verify(scanner, never()).setStartKey(any(byte[].class)); + verify(scanner, never()).setStopKey(any(byte[].class)); + } + + @Test + public void setDataTableScanFilterGroupBy() throws Exception { + QueryUtil.setDataTableScanFilter( + scanner, + Lists.<byte[]>newArrayList(new byte[] { 0, 0, 1 }), + new ByteMap<byte[][]>(), + false, + false, + 0); + verify(scanner, never()).getCurrentKey(); + // TODO - validate the regex + verify(scanner, times(1)).setFilter(any(KeyRegexpFilter.class)); + verify(scanner, never()).setStartKey(any(byte[].class)); + verify(scanner, never()).setStopKey(any(byte[].class)); + } + + @Test + public void setDataTableScanFilterTags() throws Exception { + final ByteMap<byte[][]> tags = new ByteMap<byte[][]>(); + tags.put(new byte[] { 0, 0, 1 }, new byte[][] { new byte[] {0, 0, 1} }); + QueryUtil.setDataTableScanFilter( + scanner, + Lists.<byte[]>newArrayList(), + tags, + false, + false, + 0); + verify(scanner, never()).getCurrentKey(); + // TODO - validate the regex + verify(scanner, times(1)).setFilter(any(KeyRegexpFilter.class)); + verify(scanner, never()).setStartKey(any(byte[].class)); + verify(scanner, never()).setStopKey(any(byte[].class)); + } + + @Test + public void setDataTableScanFilterEnableFuzzy() throws Exception { + final ByteMap<byte[][]> tags = new ByteMap<byte[][]>(); + tags.put(new byte[] { 0, 0, 1 }, new byte[][] { new byte[] {0, 0, 1} }); + QueryUtil.setDataTableScanFilter( + scanner, + Lists.<byte[]>newArrayList(), + tags, + false, + true, + 0); + verify(scanner, never()).getCurrentKey(); + // TODO - validate the regex + verify(scanner, times(1)).setFilter(any(KeyRegexpFilter.class)); + verify(scanner, never()).setStartKey(any(byte[].class)); + verify(scanner, never()).setStopKey(any(byte[].class)); + } + + @Test + public void setDataTableScanFilterEnableExplicit() throws Exception { + final ByteMap<byte[][]> tags = new ByteMap<byte[][]>(); + tags.put(new byte[] { 0, 0, 1 }, new byte[][] { new byte[] {0, 0, 1} }); + QueryUtil.setDataTableScanFilter( + scanner, + Lists.<byte[]>newArrayList(), + tags, + true, + false, + 0); + verify(scanner, never()).getCurrentKey(); + // TODO - validate the regex + verify(scanner, times(1)).setFilter(any(KeyRegexpFilter.class)); + verify(scanner, never()).setStartKey(any(byte[].class)); + verify(scanner, never()).setStopKey(any(byte[].class)); + } + + @Test + public void setDataTableScanFilterEnableBoth() throws Exception { + when(scanner.getCurrentKey()).thenReturn(new byte[] { 0, 0, 0, 0, 0, 0, 1 }); + final ByteMap<byte[][]> tags = new ByteMap<byte[][]>(); + tags.put(new byte[] { 0, 0, 1 }, new byte[][] { new byte[] {0, 0, 1} }); + QueryUtil.setDataTableScanFilter( + scanner, + Lists.<byte[]>newArrayList(), + tags, + true, + true, + 0); + verify(scanner, times(3)).getCurrentKey(); + // TODO - validate the regex and fuzzy filter + verify(scanner, times(1)).setFilter(any(FilterList.class)); + verify(scanner, times(1)).setStartKey(any(byte[].class)); + verify(scanner, times(1)).setStopKey(any(byte[].class)); + } + + @Test + public void timestampComparison() { + long now = DateTime.currentTimeMillis() / 1000L; + assertFalse(QueryUtil.isTimestampAfter(now*1000, now+1)); + assertFalse(QueryUtil.isTimestampAfter(now-1, now*1000L)); + assertFalse(QueryUtil.isTimestampAfter(now-1, now)); + assertFalse(QueryUtil.isTimestampAfter((now-1)*1000L, now*1000L)); + + assertTrue(QueryUtil.isTimestampAfter(now+1, now*1000L)); + assertTrue(QueryUtil.isTimestampAfter(now*1000L, now-1)); + assertTrue(QueryUtil.isTimestampAfter(now, now-1)); + assertTrue(QueryUtil.isTimestampAfter(now*1000L, (now-1)*1000L)); + + assertFalse(QueryUtil.isTimestampAfter(now, now)); + } +} diff --git a/test/query/expression/BaseTimeSyncedIteratorTest.java b/test/query/expression/BaseTimeSyncedIteratorTest.java new file mode 100644 index 0000000000..1e6be27dd4 --- /dev/null +++ b/test/query/expression/BaseTimeSyncedIteratorTest.java @@ -0,0 +1,651 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.expression; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import net.opentsdb.core.BaseTsdbTest; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.FillPolicy; +import net.opentsdb.core.Query; +import net.opentsdb.core.TSQuery; +import net.opentsdb.core.TSSubQuery; +import net.opentsdb.utils.Pair; + +/** + * A class for setting up a number of time series and queries for expression + * testing. + */ +public class BaseTimeSyncedIteratorTest extends BaseTsdbTest { + + /** Start time across all queries */ + protected static final long START_TS = 1388534400; + + /** The query object compiled after calling {@link #runQueries(ArrayList)} */ + protected TSQuery query; + + /** The results of our queries after calling {@link #runQueries(ArrayList)} */ + protected Map<String, Pair<TSSubQuery, DataPoints[]>> results; + + /** List of iterators */ + protected Map<String, ITimeSyncedIterator> iterators; + + /** + * Queries for metrics A and B with a group by all on the D tag + */ + protected void queryAB_Dstar() throws Exception { + final ArrayList<TSSubQuery> subs = new ArrayList<TSSubQuery>(2); + TSSubQuery sub = new TSSubQuery(); + + HashMap<String, String> query_tags = new HashMap<String, String>(1); + query_tags.put("D", "*"); + + sub = new TSSubQuery(); + sub.setMetric("A"); + sub.setTags(query_tags); + sub.setAggregator("sum"); + subs.add(sub); + + sub = new TSSubQuery(); + sub.setMetric("B"); + query_tags = new HashMap<String, String>(1); + query_tags.put("D", "*"); + sub.setTags(query_tags); + sub.setAggregator("sum"); + subs.add(sub); + + runQueries(subs); + } + + /** + * Queries for A and B but without a tag specifier, thus agging em all + */ + protected void queryAB_AggAll() throws Exception { + final ArrayList<TSSubQuery> subs = new ArrayList<TSSubQuery>(2); + TSSubQuery sub = new TSSubQuery(); + + HashMap<String, String> query_tags = new HashMap<String, String>(1); + + sub = new TSSubQuery(); + sub.setMetric("A"); + sub.setTags(query_tags); + sub.setAggregator("sum"); + subs.add(sub); + + sub = new TSSubQuery(); + sub.setMetric("B"); + query_tags = new HashMap<String, String>(1); + sub.setTags(query_tags); + sub.setAggregator("sum"); + subs.add(sub); + + runQueries(subs); + } + + /** + * Queries for A only with a filter of tag value "D" for tag key "D" + */ + protected void queryA_DD() throws Exception { + final ArrayList<TSSubQuery> subs = new ArrayList<TSSubQuery>(2); + TSSubQuery sub = new TSSubQuery(); + final HashMap<String, String> query_tags = new HashMap<String, String>(1); + query_tags.put("D", "D"); + sub = new TSSubQuery(); + sub.setMetric("A"); + sub.setTags(query_tags); + sub.setAggregator("sum"); + subs.add(sub); + + runQueries(subs); + } + + /** + * Executes the queries against MockBase through the regular pipeline and stores + * the results in {@link #results} + * @param subs The queries to execute + */ + protected void runQueries(final ArrayList<TSSubQuery> subs) throws Exception { + query = new TSQuery(); + query.setStart(Long.toString(START_TS)); + query.setQueries(subs); + query.validateAndSetQuery(); + + final Query[] compiled = query.buildQueries(tsdb); + results = new HashMap<String, Pair<TSSubQuery, DataPoints[]>>( + compiled.length); + iterators = new HashMap<String, ITimeSyncedIterator>(compiled.length); + + int index = 0; + for (final Query q : compiled) { + final DataPoints[] dps = q.runAsync().join(); + results.put(Integer.toString(index), + new Pair<TSSubQuery, DataPoints[]>( + query.getQueries().get(index), dps)); + final ITimeSyncedIterator it = new TimeSyncedIterator(Integer.toString(index), + query.getQueries().get(index).getFilterTagKs(), dps); + it.setFillPolicy(new NumericFillPolicy(FillPolicy.NOT_A_NUMBER)); + iterators.put(Integer.toString(index), it); + index++; + } + } + + /** + * A and B, each with two series. Common D values, different E values + */ + protected void twoSeriesAggedE() throws Exception { + setDataPointStorage(); + HashMap<String, String> tags = new HashMap<String, String>(2); + tags.put("D", "D"); + tags.put("E", "E"); + + tsdb.addPoint("A", 1431561600, 1, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 2, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 3, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "D"); + tags.put("E", "F"); + + tsdb.addPoint("A", 1431561600, 1, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 2, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 3, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "D"); + tags.put("E", "E"); + tsdb.addPoint("B", 1431561600, 11, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561660, 12, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561720, 13, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "D"); + tags.put("E", "F"); + tsdb.addPoint("B", 1431561600, 11, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561660, 12, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561720, 13, tags).joinUninterruptibly(); + } + + /** + * A and B, each with two series. Common D, different E values and the B metric + * series have an extra Z tag with different values. + */ + protected void twoSeriesAggedEandExtraTagK() throws Exception { + setDataPointStorage(); + HashMap<String, String> tags = new HashMap<String, String>(2); + tags.put("D", "D"); + tags.put("E", "E"); + + tsdb.addPoint("A", 1431561600, 1, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 2, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 3, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "D"); + tags.put("E", "F"); + + tsdb.addPoint("A", 1431561600, 1, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 2, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 3, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "D"); + tags.put("E", "E"); + tags.put("Z", "A"); + tsdb.addPoint("B", 1431561600, 11, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561660, 12, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561720, 13, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "D"); + tags.put("E", "F"); + tags.put("Z", "B"); + tsdb.addPoint("B", 1431561600, 11, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561660, 12, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561720, 13, tags).joinUninterruptibly(); + } + + /** + * A and B where A has two series and B only has one. Series A has two D + * values that will be agged. Different D and E values. + */ + protected void oneAggedTheOtherTagged() throws Exception { + setDataPointStorage(); + HashMap<String, String> tags = new HashMap<String, String>(2); + tags.put("D", "D"); + tags.put("E", "E"); + + tsdb.addPoint("A", 1431561600, 1, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 2, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 3, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "E"); + tags.put("E", "F"); + + tsdb.addPoint("A", 1431561600, 1, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 2, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 3, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "D"); + tags.put("E", "E"); + + tsdb.addPoint("B", 1431561600, 11, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561660, 12, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561720, 13, tags).joinUninterruptibly(); + } + + /** + * A only with three series. Different D values, commong E values. + */ + protected void threeSameENoB() throws Exception { + setDataPointStorage(); + HashMap<String, String> tags = new HashMap<String, String>(2); + tags.put("D", "D"); + tags.put("E", "E"); + + tsdb.addPoint("A", 1431561600, 1, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 2, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 3, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "F"); + tags.put("E", "E"); + tsdb.addPoint("A", 1431561600, 4, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 5, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 6, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "G"); + tags.put("E", "E"); + tsdb.addPoint("A", 1431561600, 7, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 8, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 9, tags).joinUninterruptibly(); + } + + /** + * A and B where A has two series, B has three. Different D values, common E. + */ + protected void oneExtraSameE() throws Exception { + setDataPointStorage(); + HashMap<String, String> tags = new HashMap<String, String>(2); + tags.put("D", "D"); + tags.put("E", "E"); + + tsdb.addPoint("A", 1431561600, 1, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 2, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 3, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "F"); + tags.put("E", "E"); + tsdb.addPoint("A", 1431561600, 4, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 5, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 6, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "D"); + tags.put("E", "E"); + tsdb.addPoint("B", 1431561600, 11, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561660, 12, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561720, 13, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "F"); + tags.put("E", "E"); + tsdb.addPoint("B", 1431561600, 14, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561660, 15, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561720, 16, tags).joinUninterruptibly(); + + // all by myself...... + tags = new HashMap<String, String>(2); + tags.put("D", "G"); + tags.put("E", "E"); + tsdb.addPoint("B", 1431561600, 17, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561660, 18, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561720, 19, tags).joinUninterruptibly(); + } + + /** + * A and B with two series each. Different D values, common E. + * A has values at T0 and T1, but then B has values at T2 and T3. Should + * throw NaNs after the intersection. + */ + protected void timeOffset() throws Exception { + setDataPointStorage(); + HashMap<String, String> tags = new HashMap<String, String>(2); + tags.put("D", "D"); + tags.put("E", "E"); + + tsdb.addPoint("A", 1431561600, 1, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 2, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "F"); + tags.put("E", "E"); + tsdb.addPoint("A", 1431561600, 4, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 5, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "D"); + tags.put("E", "E"); + tsdb.addPoint("B", 1431561720, 13, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561780, 14, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "F"); + tags.put("E", "E"); + tsdb.addPoint("B", 1431561720, 16, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561780, 17, tags).joinUninterruptibly(); + } + + /** + * A and B, each with three series. Different D values, commong E. + */ + protected void threeSameE() throws Exception { + setDataPointStorage(); + HashMap<String, String> tags = new HashMap<String, String>(2); + tags.put("D", "D"); + tags.put("E", "E"); + + tsdb.addPoint("A", 1431561600, 1, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 2, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 3, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "F"); + tags.put("E", "E"); + tsdb.addPoint("A", 1431561600, 4, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 5, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 6, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "G"); + tags.put("E", "E"); + tsdb.addPoint("A", 1431561600, 7, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 8, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 9, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "D"); + tags.put("E", "E"); + tsdb.addPoint("B", 1431561600, 11, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561660, 12, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561720, 13, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "F"); + tags.put("E", "E"); + tsdb.addPoint("B", 1431561600, 14, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561660, 15, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561720, 16, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "G"); + tags.put("E", "E"); + tsdb.addPoint("B", 1431561600, 17, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561660, 18, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561720, 19, tags).joinUninterruptibly(); + } + + /** + * A and B, each with three series. Different D values. Series in A are missing + * the E tag. Common values in E for B. + */ + protected void threeAMissingE() throws Exception { + setDataPointStorage(); + HashMap<String, String> tags = new HashMap<String, String>(2); + tags.put("D", "D"); + tsdb.addPoint("A", 1431561600, 1, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 2, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 3, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "F"); + tsdb.addPoint("A", 1431561600, 4, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 5, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 6, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "G"); + tsdb.addPoint("A", 1431561600, 7, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 8, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 9, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "D"); + tags.put("E", "E"); + tsdb.addPoint("B", 1431561600, 11, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561660, 12, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561720, 13, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "F"); + tags.put("E", "E"); + tsdb.addPoint("B", 1431561600, 14, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561660, 15, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561720, 16, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "G"); + tags.put("E", "E"); + tsdb.addPoint("B", 1431561600, 17, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561660, 18, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561720, 19, tags).joinUninterruptibly(); + } + + /** + * A and B, each with three series. Different D and E values + */ + protected void threeDifE() throws Exception { + setDataPointStorage(); + HashMap<String, String> tags = new HashMap<String, String>(2); + tags.put("D", "D"); + tags.put("E", "A"); + tsdb.addPoint("A", 1431561600, 1, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 2, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 3, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "F"); + tags.put("E", "B"); + tsdb.addPoint("A", 1431561600, 4, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 5, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 6, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "G"); + tags.put("E", "C"); + tsdb.addPoint("A", 1431561600, 7, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 8, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 9, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "D"); + tags.put("E", "D"); + tsdb.addPoint("B", 1431561600, 11, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561660, 12, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561720, 13, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "F"); + tags.put("E", "F"); + tsdb.addPoint("B", 1431561600, 14, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561660, 15, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561720, 16, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "G"); + tags.put("E", "G"); + tsdb.addPoint("B", 1431561600, 17, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561660, 18, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561720, 19, tags).joinUninterruptibly(); + } + + /** + * A and B, each with 3 series. Different D values, common E. + * Each set has one series that isn't in the other. D=G in A and D=Q in B. + */ + protected void threeDisjointSameE() throws Exception { + setDataPointStorage(); + + HashMap<String, String> tags = new HashMap<String, String>(2); + tags.put("D", "D"); + tags.put("E", "E"); + tsdb.addPoint("A", 1431561600, 1, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 2, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 3, tags).joinUninterruptibly(); + + // not in set 2 + tags = new HashMap<String, String>(2); + tags.put("D", "F"); + tags.put("E", "E"); + tsdb.addPoint("A", 1431561600, 4, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 5, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 6, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "G"); + tags.put("E", "E"); + tsdb.addPoint("A", 1431561600, 7, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 8, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 9, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "D"); + tags.put("E", "E"); + tsdb.addPoint("B", 1431561600, 11, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561660, 12, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561720, 13, tags).joinUninterruptibly(); + + // not in set 1 + tags = new HashMap<String, String>(2); + tags.put("D", "Q"); + tags.put("E", "E"); + tsdb.addPoint("B", 1431561600, 14, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561660, 15, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561720, 16, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "G"); + tags.put("E", "E"); + tsdb.addPoint("B", 1431561600, 17, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561660, 18, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561720, 19, tags).joinUninterruptibly(); + } + + /** + * A and B, each with three series. Different D values, common E values. + * D=G is the only common series between the sets + */ + protected void reduceToOne() throws Exception { + setDataPointStorage(); + HashMap<String, String> tags = new HashMap<String, String>(2); + tags.put("D", "D"); + tags.put("E", "E"); + + tsdb.addPoint("A", 1431561600, 1, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 2, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 3, tags).joinUninterruptibly(); + + // not in set 2 + tags = new HashMap<String, String>(2); + tags.put("D", "F"); + tags.put("E", "E"); + tsdb.addPoint("A", 1431561600, 4, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 5, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 6, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "G"); + tags.put("E", "E"); + tsdb.addPoint("A", 1431561600, 7, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 8, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 9, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "P"); + tags.put("E", "E"); + tsdb.addPoint("B", 1431561600, 11, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561660, 12, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561720, 13, tags).joinUninterruptibly(); + + // not in set 1 + tags = new HashMap<String, String>(2); + tags.put("D", "Q"); + tags.put("E", "E"); + tsdb.addPoint("B", 1431561600, 14, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561660, 15, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561720, 16, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "G"); + tags.put("E", "E"); + tsdb.addPoint("B", 1431561600, 17, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561660, 18, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561720, 19, tags).joinUninterruptibly(); + } + + /** + * A and B, each with three series. Different D values, common E values. + * Each series is "missing" a data point so that we can test time sync. + */ + protected void threeSameEGaps() throws Exception { + setDataPointStorage(); + HashMap<String, String> tags = new HashMap<String, String>(2); + tags.put("D", "D"); + tags.put("E", "E"); + + tsdb.addPoint("A", 1431561600, 1, tags).joinUninterruptibly(); + //tsdb.addPoint("A", 1431561660, 2, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 3, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "F"); + tags.put("E", "E"); + tsdb.addPoint("A", 1431561600, 4, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 5, tags).joinUninterruptibly(); + //tsdb.addPoint("A", 1431561720, 6, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "G"); + tags.put("E", "E"); + //tsdb.addPoint("A", 1431561600, 7, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561660, 8, tags).joinUninterruptibly(); + tsdb.addPoint("A", 1431561720, 9, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "D"); + tags.put("E", "E"); + //tsdb.addPoint("B", 1431561600, 11, tags).joinUninterruptibly(); + //tsdb.addPoint("B", 1431561660, 12, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561720, 13, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "F"); + tags.put("E", "E"); + //tsdb.addPoint("B", 1431561600, 14, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561660, 15, tags).joinUninterruptibly(); + //tsdb.addPoint("B", 1431561720, 16, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "G"); + tags.put("E", "E"); + //tsdb.addPoint("B", 1431561600, 17, tags).joinUninterruptibly(); + //tsdb.addPoint("B", 1431561660, 18, tags).joinUninterruptibly(); + tsdb.addPoint("B", 1431561720, 19, tags).joinUninterruptibly(); + } + +} diff --git a/test/query/expression/TestAbsolute.java b/test/query/expression/TestAbsolute.java new file mode 100644 index 0000000000..7d822fe4f4 --- /dev/null +++ b/test/query/expression/TestAbsolute.java @@ -0,0 +1,309 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.expression; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.SeekableView; +import net.opentsdb.core.SeekableViewsForTest; +import net.opentsdb.core.TSQuery; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.stumbleupon.async.Deferred; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ TSQuery.class }) +public class TestAbsolute { + + private static long START_TIME = 1356998400000L; + private static int INTERVAL = 60000; + private static int NUM_POINTS = 5; + private static String METRIC = "sys.cpu"; + + private TSQuery data_query; + private SeekableView view; + private DataPoints dps; + private DataPoints[] group_bys; + private List<DataPoints[]> query_results; + private List<String> params; + private Absolute func; + + @Before + public void before() throws Exception { + view = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 1, 1); + data_query = mock(TSQuery.class); + when(data_query.startTime()).thenReturn(START_TIME); + when(data_query.endTime()).thenReturn(START_TIME + (INTERVAL * NUM_POINTS)); + + dps = PowerMockito.mock(DataPoints.class); + when(dps.iterator()).thenReturn(view); + when(dps.metricNameAsync()).thenReturn(Deferred.fromResult(METRIC)); + + group_bys = new DataPoints[] { dps }; + + query_results = new ArrayList<DataPoints[]>(1); + query_results.add(group_bys); + + params = new ArrayList<String>(1); + func = new Absolute(); + } + + @Test + public void evaluatePositiveGroupByLong() throws Exception { + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals(METRIC, results[0].metricName()); + assertEquals("sys.mem", results[1].metricName()); + + long ts = START_TIME; + long v = 1; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + ts = START_TIME; + v = 10; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + } + + @Test + public void evaluatePositiveGroupByDouble() throws Exception { + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, false, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals(METRIC, results[0].metricName()); + assertEquals("sys.mem", results[1].metricName()); + + long ts = START_TIME; + double v = 1; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals((long)v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + ts = START_TIME; + v = 10; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + v += 1; + } + } + + @Test + public void evaluateFactorNegativeGroupByLong() throws Exception { + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, -10, -1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals(METRIC, results[0].metricName()); + assertEquals("sys.mem", results[1].metricName()); + + long ts = START_TIME; + long v = 1; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + ts = START_TIME; + v = 10; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + } + + @Test + public void evaluateNegativeGroupByDouble() throws Exception { + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, false, -10, -1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals(METRIC, results[0].metricName()); + assertEquals("sys.mem", results[1].metricName()); + + long ts = START_TIME; + double v = 1; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals((long)v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + ts = START_TIME; + v = 10; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + v += 1; + } + } + + @Test + public void evaluateNegativeSubQuerySeries() throws Exception { + params.add("1"); + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, -10, -1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals(METRIC, results[0].metricName()); + assertEquals("sys.mem", results[1].metricName()); + + long ts = START_TIME; + long v = 1; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + ts = START_TIME; + v = 10; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateNullQuery() throws Exception { + params.add("1"); + func.evaluate(null, query_results, params); + } + + @Test + public void evaluateNullResults() throws Exception { + params.add("1"); + final DataPoints[] results = func.evaluate(data_query, null, params); + assertEquals(0, results.length); + } + + @Test + public void evaluateNullParams() throws Exception { + assertNotNull(func.evaluate(data_query, query_results, null)); + } + + @Test + public void evaluateEmptyResults() throws Exception { + final DataPoints[] results = func.evaluate(data_query, + Collections.<DataPoints[]>emptyList(), params); + assertEquals(0, results.length); + } + + @Test + public void evaluateEmptyParams() throws Exception { + assertNotNull(func.evaluate(data_query, query_results, null)); + } + + @Test + public void writeStringField() throws Exception { + params.add("1"); + assertEquals("absolute(inner_expression)", + func.writeStringField(params, "inner_expression")); + assertEquals("absolute(null)", func.writeStringField(params, null)); + assertEquals("absolute()", func.writeStringField(params, "")); + assertEquals("absolute(inner_expression)", + func.writeStringField(null, "inner_expression")); + } +} diff --git a/test/query/expression/TestAlias.java b/test/query/expression/TestAlias.java new file mode 100644 index 0000000000..744287e50f --- /dev/null +++ b/test/query/expression/TestAlias.java @@ -0,0 +1,321 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.expression; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.SeekableView; +import net.opentsdb.core.SeekableViewsForTest; +import net.opentsdb.core.TSQuery; + +import org.hbase.async.Bytes.ByteMap; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.stumbleupon.async.Deferred; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ TSQuery.class }) +public class TestAlias { + + private static long START_TIME = 1356998400000L; + private static int INTERVAL = 60000; + private static int NUM_POINTS = 5; + private static String METRIC = "sys.cpu"; + + private TSQuery data_query; + private SeekableView view; + private DataPoints dps; + private DataPoints[] group_bys; + private List<DataPoints[]> query_results; + private List<String> params; + private Alias func; + private Map<String, String> tags; + private ByteMap<byte[]> tag_uids; + + @Before + public void before() throws Exception { + view = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 1, 1); + data_query = mock(TSQuery.class); + when(data_query.startTime()).thenReturn(START_TIME); + when(data_query.endTime()).thenReturn(START_TIME + (INTERVAL * NUM_POINTS)); + + tags = new HashMap<String, String>(2); + tags.put("host", "web01"); + tags.put("dc", "lga"); + tag_uids = new ByteMap<byte[]>(); + tag_uids.put(new byte[] { 0, 0, 1 }, new byte[] { 0, 0, 1 }); + tag_uids.put(new byte[] { 0, 0, 2 }, new byte[] { 0, 0, 2 }); + + dps = PowerMockito.mock(DataPoints.class); + when(dps.iterator()).thenReturn(view); + when(dps.metricName()).thenReturn(METRIC); + when(dps.getTagsAsync()).thenReturn(Deferred.fromResult(tags)); + when(dps.getTagUids()).thenReturn(tag_uids); + + group_bys = new DataPoints[] { dps }; + + query_results = new ArrayList<DataPoints[]>(1); + query_results.add(group_bys); + + params = new ArrayList<String>(1); + func = new Alias(); + } + + @Test + public void evaluateGroupByLong() throws Exception { + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricName()).thenReturn("sys.mem"); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + params.add("My Alias"); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals("My Alias", results[0].metricName()); + assertEquals("My Alias", results[1].metricName()); + + long ts = START_TIME; + long v = 1; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + ts = START_TIME; + v = 10; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + } + + @Test + public void evaluateGroupByDouble() throws Exception { + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, false, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricName()).thenReturn("sys.mem"); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + params.add("My Alias"); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals("My Alias", results[0].metricName()); + assertEquals("My Alias", results[1].metricName()); + + long ts = START_TIME; + double v = 1; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals((long)v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + ts = START_TIME; + v = 10; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + v += 1; + } + } + + @Test + public void evaluateSubQuerySeries() throws Exception { + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, -10, -1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricName()).thenReturn("sys.mem"); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + params.add("My Alias"); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals("My Alias", results[0].metricName()); + assertEquals("My Alias", results[1].metricName()); + + long ts = START_TIME; + long v = 1; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + ts = START_TIME; + v = 10; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + } + + @Test + public void evaluateWithTags() throws Exception { + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricName()).thenReturn("sys.mem"); + when(dps2.getTagsAsync()).thenReturn(Deferred.fromResult(tags)); + when(dps2.getTagUids()).thenReturn(tag_uids); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + params.add("My Alias.@host.@dc"); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals("My Alias.web01.lga", results[0].metricName()); + assertEquals("My Alias.web01.lga", results[1].metricName()); + } + + @Test + public void evaluateWithATag() throws Exception { + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricName()).thenReturn("sys.mem"); + when(dps2.getTagsAsync()).thenReturn(Deferred.fromResult(tags)); + when(dps2.getTagUids()).thenReturn(tag_uids); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + params.add("My Alias.@dc"); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals("My Alias.lga", results[0].metricName()); + assertEquals("My Alias.lga", results[1].metricName()); + } + + @Test + public void evaluateWithTagsJoined() throws Exception { + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricName()).thenReturn("sys.mem"); + when(dps2.getTagsAsync()).thenReturn(Deferred.fromResult(tags)); + when(dps2.getTagUids()).thenReturn(tag_uids); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + params.add("My Alias"); + params.add("@host"); + params.add("@dc"); + params.add("@none"); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals("My Alias,web01,lga,@none", results[0].metricName()); + assertEquals("My Alias,web01,lga,@none", results[1].metricName()); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateNullQuery() throws Exception { + params.add("1"); + func.evaluate(null, query_results, params); + } + + @Test + public void evaluateNullResults() throws Exception { + params.add("1"); + final DataPoints[] results = func.evaluate(data_query, null, params); + assertEquals(0, results.length); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateNullParams() throws Exception { + func.evaluate(data_query, query_results, null); + } + + @Test + public void evaluateEmptyResults() throws Exception { + final DataPoints[] results = func.evaluate(data_query, + Collections.<DataPoints[]>emptyList(), params); + assertEquals(0, results.length); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateEmptyParams() throws Exception { + func.evaluate(data_query, query_results, null); + } + + @Test + public void writeStringField() throws Exception { + assertEquals("alias(m)", func.writeStringField(params, "m")); + params.add("MyAlias"); + assertEquals("alias(m,MyAlias)", func.writeStringField(params, "m")); + params.clear(); + params.add("Alias"); + params.add("@host"); + assertEquals("alias(m,Alias,@host)", func.writeStringField(params, "m")); + params.clear(); + assertEquals("alias(null)", func.writeStringField(params, null)); + assertEquals("alias()", func.writeStringField(params, "")); + assertEquals("alias(inner_expression)", + func.writeStringField(null, "inner_expression")); + } +} diff --git a/test/query/expression/TestDiffSeries.java b/test/query/expression/TestDiffSeries.java new file mode 100644 index 0000000000..3c36e234c1 --- /dev/null +++ b/test/query/expression/TestDiffSeries.java @@ -0,0 +1,192 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.expression; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.SeekableView; +import net.opentsdb.core.SeekableViewsForTest; +import net.opentsdb.core.TSQuery; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.sun.java_cup.internal.runtime.Scanner; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ TSQuery.class, Scanner.class }) +public class TestDiffSeries extends BaseTimeSyncedIteratorTest { + + private static long START_TIME = 1356998400000L; + private static int INTERVAL = 60000; + private static int NUM_POINTS = 5; + + private TSQuery data_query; + private SeekableView view; + private DataPoints dps; + private DataPoints[] group_bys; + private List<DataPoints[]> query_results; + private List<String> params; + private DiffSeries func; + + @Before + public void beforeLocal() throws Exception { + view = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 1, 1); + data_query = mock(TSQuery.class); + when(data_query.startTime()).thenReturn(START_TIME); + when(data_query.endTime()).thenReturn(START_TIME + (INTERVAL * NUM_POINTS)); + + dps = PowerMockito.mock(DataPoints.class); + when(dps.iterator()).thenReturn(view); + when(dps.metricName()).thenReturn(METRIC_STRING); + when(dps.metricUID()).thenReturn(new byte[] {0,0,1}); + + group_bys = new DataPoints[] { dps }; + + query_results = new ArrayList<DataPoints[]>(1); + query_results.add(group_bys); + + params = new ArrayList<String>(1); + func = new DiffSeries(tsdb); + } + + @Test + public void diffOneSeriesEach() throws Exception { + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricName()).thenReturn("sys.mem"); + when(dps2.metricUID()).thenReturn(new byte[] {0,0,2}); + group_bys = new DataPoints[] { dps2 }; + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(1, results.length); + assertEquals(METRIC_STRING, results[0].metricName()); + + long ts = START_TIME; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertEquals(-9, dp.toDouble(), 0.001); + ts += INTERVAL; + } + } + + @Test + public void diffMultipleSeriesEach() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + + query_results.clear(); + query_results.add(results.get("1").getValue()); + query_results.add(results.get("0").getValue()); + final DataPoints[] results = func.evaluate(data_query, + query_results, params); + + assertEquals(3, results.length); + + double val = 17; + for (int i = 0; i < results.length; i++) { + long ts = 1431561600000l; + final SeekableView it = results[i].iterator(); + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + if (i < 2) { + assertEquals(10, dp.toDouble(), 0.0001); + } else { + assertEquals(val++, dp.toDouble(), 0.0001); + } + ts += INTERVAL; + } + } + } + + @Test (expected = IllegalArgumentException.class) + public void diffOneResultSet() throws Exception { + func.evaluate(data_query, query_results, params); + } + + @Test (expected = IllegalArgumentException.class) + public void diffTooManyResultSets() throws Exception { + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricName()).thenReturn("sys.mem"); + when(dps2.metricUID()).thenReturn(new byte[] {0,0,2}); + group_bys = new DataPoints[] { dps2 }; + // doesn't matter what they are + for (int i = 0; i < 100; i++) { + query_results.add(group_bys); + } + + func.evaluate(data_query, query_results, params); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateNullQuery() throws Exception { + params.add("1"); + func.evaluate(null, query_results, params); + } + + @Test + public void evaluateNullResults() throws Exception { + params.add("1"); + final DataPoints[] results = func.evaluate(data_query, null, params); + assertEquals(0, results.length); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateNullParams() throws Exception { + func.evaluate(data_query, query_results, null); + } + + @Test + public void evaluateEmptyResults() throws Exception { + params.add("1"); + final DataPoints[] results = func.evaluate(data_query, + Collections.<DataPoints[]>emptyList(), params); + assertEquals(0, results.length); + } + + @Test + public void writeStringField() throws Exception { + params.add("1"); + assertEquals("diffSeries(inner_expression)", + func.writeStringField(params, "inner_expression")); + assertEquals("diffSeries(null)", func.writeStringField(params, null)); + assertEquals("diffSeries()", func.writeStringField(params, "")); + assertEquals("diffSeries(inner_expression)", + func.writeStringField(null, "inner_expression")); + } +} diff --git a/test/query/expression/TestDivideSeries.java b/test/query/expression/TestDivideSeries.java new file mode 100644 index 0000000000..e880a6c31e --- /dev/null +++ b/test/query/expression/TestDivideSeries.java @@ -0,0 +1,197 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.expression; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.SeekableView; +import net.opentsdb.core.SeekableViewsForTest; +import net.opentsdb.core.TSQuery; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ TSQuery.class }) +public class TestDivideSeries extends BaseTimeSyncedIteratorTest { + + private static long START_TIME = 1356998400000L; + private static int INTERVAL = 60000; + private static int NUM_POINTS = 5; + + private TSQuery data_query; + private SeekableView view; + private DataPoints dps; + private DataPoints[] group_bys; + private List<DataPoints[]> query_results; + private List<String> params; + private DivideSeries func; + + @Before + public void beforeLocal() throws Exception { + view = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 1, 1); + data_query = mock(TSQuery.class); + when(data_query.startTime()).thenReturn(START_TIME); + when(data_query.endTime()).thenReturn(START_TIME + (INTERVAL * NUM_POINTS)); + + dps = PowerMockito.mock(DataPoints.class); + when(dps.iterator()).thenReturn(view); + when(dps.metricName()).thenReturn(METRIC_STRING); + when(dps.metricUID()).thenReturn(new byte[] {0,0,1}); + + group_bys = new DataPoints[] { dps }; + + query_results = new ArrayList<DataPoints[]>(1); + query_results.add(group_bys); + + params = new ArrayList<String>(1); + func = new DivideSeries(tsdb); + } + + @Test + public void divideOneSeriesEach() throws Exception { + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricName()).thenReturn("sys.mem"); + when(dps2.metricUID()).thenReturn(new byte[] {0,0,2}); + group_bys = new DataPoints[] { dps2 }; + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(1, results.length); + assertEquals(METRIC_STRING, results[0].metricName()); + + double[] vals= new double[] { 0.1, 0.181, 0.25, 0.307, 0.357 }; + + long ts = START_TIME; + int i = 0; + for (final DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertEquals(vals[i++], dp.toDouble(), 0.001); + ts += INTERVAL; + } + } + + @Test + public void divideMultipleSeriesEach() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + + query_results.clear(); + query_results.add(results.get("1").getValue()); + query_results.add(results.get("0").getValue()); + final DataPoints[] results = func.evaluate(data_query, + query_results, params); + + assertEquals(3, results.length); + + final int vals[][] = new int[2][]; + vals[0] = new int[] { 11, 1 }; + vals[1] = new int[] { 14, 4 }; + for (int i = 0; i < results.length; i++) { + long ts = 1431561600000l; + final SeekableView it = results[i].iterator(); + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + if (i < 2) { + assertEquals(((double)vals[i][0]++ / (double)vals[i][1]++), + dp.toDouble(), 0.001); + } else { + assertEquals(0, dp.toDouble(), 0.0001); + } + ts += INTERVAL; + } + } + } + + @Test (expected = IllegalArgumentException.class) + public void divideOneResultSet() throws Exception { + func.evaluate(data_query, query_results, params); + } + + @Test (expected = IllegalArgumentException.class) + public void divideTooManyResultSets() throws Exception { + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricName()).thenReturn("sys.mem"); + when(dps2.metricUID()).thenReturn(new byte[] {0,0,2}); + group_bys = new DataPoints[] { dps2 }; + query_results.add(group_bys); + // doesn't matter what they are + for (int i = 0; i < 100; i++) { + query_results.add(group_bys); + } + + func.evaluate(data_query, query_results, params); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateNullQuery() throws Exception { + params.add("1"); + func.evaluate(null, query_results, params); + } + + @Test + public void evaluateNullResults() throws Exception { + params.add("1"); + final DataPoints[] results = func.evaluate(data_query, null, params); + assertEquals(0, results.length); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateNullParams() throws Exception { + func.evaluate(data_query, query_results, null); + } + + @Test + public void evaluateEmptyResults() throws Exception { + params.add("1"); + final DataPoints[] results = func.evaluate(data_query, + Collections.<DataPoints[]>emptyList(), params); + assertEquals(0, results.length); + } + + @Test + public void writeStringField() throws Exception { + params.add("1"); + assertEquals("divideSeries(inner_expression)", + func.writeStringField(params, "inner_expression")); + assertEquals("divideSeries(null)", func.writeStringField(params, null)); + assertEquals("divideSeries()", func.writeStringField(params, "")); + assertEquals("divideSeries(inner_expression)", + func.writeStringField(null, "inner_expression")); + } +} diff --git a/test/query/expression/TestExpressionFactory.java b/test/query/expression/TestExpressionFactory.java new file mode 100644 index 0000000000..b0024fa44f --- /dev/null +++ b/test/query/expression/TestExpressionFactory.java @@ -0,0 +1,93 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.expression; + +import static org.junit.Assert.assertTrue; + +import java.util.List; + +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.TSQuery; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +public class TestExpressionFactory { + + @Test + public void getByName() throws Exception { + // pick a couple of implementations + Expression e = ExpressionFactory.getByName("scale"); + assertTrue(e instanceof Scale); + + e = ExpressionFactory.getByName("highestMax"); + assertTrue(e instanceof HighestMax); + } + + @Test (expected = UnsupportedOperationException.class) + public void getByNameNoSuchFunction() throws Exception { + ExpressionFactory.getByName("I don't exist"); + } + + @Test (expected = UnsupportedOperationException.class) + public void getByNameNullName() throws Exception { + ExpressionFactory.getByName(null); + } + + @Test (expected = UnsupportedOperationException.class) + public void getByNameEmptyName() throws Exception { + ExpressionFactory.getByName(""); + } + + @Test + public void addFunction() throws Exception { + ExpressionFactory.addFunction("testExpr", new TestExpr()); + final Expression e = ExpressionFactory.getByName("testExpr"); + assertTrue(e instanceof TestExpr); + } + + @Test (expected = IllegalArgumentException.class) + public void addFunctionNullName() throws Exception { + ExpressionFactory.addFunction(null, new TestExpr()); + } + + @Test (expected = IllegalArgumentException.class) + public void addFunctionEmptyName() throws Exception { + ExpressionFactory.addFunction("", new TestExpr()); + } + + @Test (expected = IllegalArgumentException.class) + public void addFunctionNullFunction() throws Exception { + ExpressionFactory.addFunction("testExpr", null); + } + + /** Dummy expression class used for testing */ + private static class TestExpr implements Expression { + @Override + public DataPoints[] evaluate(TSQuery data_query, + List<DataPoints[]> results, List<String> params) { + return null; + } + @Override + public String writeStringField(List<String> params, + String inner_expression) { + return null; + } + } +} diff --git a/test/query/expression/TestExpressionIterator.java b/test/query/expression/TestExpressionIterator.java new file mode 100644 index 0000000000..c18dcbcd2b --- /dev/null +++ b/test/query/expression/TestExpressionIterator.java @@ -0,0 +1,1196 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.expression; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import net.opentsdb.core.FillPolicy; +import net.opentsdb.core.IllegalDataException; +import net.opentsdb.query.expression.VariableIterator.SetOperator; + +import org.apache.commons.jexl2.JexlException; +import org.hbase.async.Bytes; +import org.junit.Test; + +public class TestExpressionIterator extends BaseTimeSyncedIteratorTest { + + @Test + public void ctor() throws Exception { + final ExpressionIterator exp = new ExpressionIterator("ei", "a + b", + SetOperator.INTERSECTION, false, false); + assertEquals(2, exp.getVariableNames().size()); + assertTrue(exp.getVariableNames().contains("a")); + assertTrue(exp.getVariableNames().contains("b")); + assertFalse(exp.getVariableNames().contains("+")); // I'm not a variable :( + assertNull(exp.values()); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNoVariables() throws Exception { + new ExpressionIterator("ei", "1 + 1", SetOperator.INTERSECTION, false, false); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNullExpression() throws Exception { + new ExpressionIterator("ei", null, SetOperator.INTERSECTION, false, false); + } + + @Test (expected = JexlException.class) + public void ctorBadExpression() throws Exception { + new ExpressionIterator("ei", " a / ", SetOperator.INTERSECTION, false, false); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorEmptyExpression() throws Exception { + new ExpressionIterator("ei", "", SetOperator.INTERSECTION, false, false); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNullOperator() throws Exception { + new ExpressionIterator("ei", "a + b", null, false, false); + } + + @Test + public void aPlusBWithTwoSeries() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + remapResults(); + storage.dumpToSystemOut(); + ExpressionIterator exp = new ExpressionIterator("ei", "a + b", + SetOperator.INTERSECTION, false, false); + exp.addResults("a", iterators.get("a")); + exp.addResults("b", iterators.get("b")); + + exp.compile(); + final ExpressionDataPoint[] dps = exp.values(); + assertEquals(2, dps.length); + validateMeta(dps, true); + + long ts = 1431561600000L; + double[] values = new double[] { 12, 18 }; + long its = exp.nextTimestamp(); + while (exp.hasNext()) { + exp.next(its); + + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(values[0], dps[0].toDouble(), 0.0001); + assertEquals(values[1], dps[1].toDouble(), 0.0001); + + values[0] += 2; + values[1] += 2; + ts += 60000; + its = exp.nextTimestamp(); + } + + for (int i = 0; i < dps.length; i++) { + assertEquals(2, dps[i].tags().size()); + assertTrue(dps[i].aggregatedTags().isEmpty()); + } + assertArrayEquals(UIDS.get("D"), dps[0].tags().get(UIDS.get("D"))); + assertArrayEquals(UIDS.get("F"), dps[1].tags().get(UIDS.get("D"))); + } + + @Test + public void aMinusBWithTwoSeries() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + remapResults(); + + ExpressionIterator exp = new ExpressionIterator("ei", "a - b", + SetOperator.INTERSECTION, false, false); + exp.addResults("a", iterators.get("a")); + exp.addResults("b", iterators.get("b")); + + exp.compile(); + final ExpressionDataPoint[] dps = exp.values(); + assertEquals(2, dps.length); + validateMeta(dps, true); + + long ts = 1431561600000L; + long its = exp.nextTimestamp(); + while (exp.hasNext()) { + exp.next(its); + + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(-10, dps[0].toDouble(), 0.0001); + assertEquals(-10, dps[1].toDouble(), 0.0001); + ts += 60000; + its = exp.nextTimestamp(); + } + + for (int i = 0; i < dps.length; i++) { + assertEquals(2, dps[i].tags().size()); + assertTrue(dps[i].aggregatedTags().isEmpty()); + } + assertArrayEquals(UIDS.get("D"), dps[0].tags().get(UIDS.get("D"))); + assertArrayEquals(UIDS.get("F"), dps[1].tags().get(UIDS.get("D"))); + } + + @Test + public void aTimesBWithTwoSeries() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + remapResults(); + + ExpressionIterator exp = new ExpressionIterator("ei", "a * b", + SetOperator.INTERSECTION, false, false); + exp.addResults("a", iterators.get("a")); + exp.addResults("b", iterators.get("b")); + + exp.compile(); + final ExpressionDataPoint[] dps = exp.values(); + assertEquals(2, dps.length); + validateMeta(dps, true); + + long ts = 1431561600000L; + long its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(11, dps[0].toDouble(), 0.0001); + assertEquals(56, dps[1].toDouble(), 0.0001); + ts += 60000; + + its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(24, dps[0].toDouble(), 0.0001); + assertEquals(75, dps[1].toDouble(), 0.0001); + ts += 60000; + + its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(39, dps[0].toDouble(), 0.0001); + assertEquals(96, dps[1].toDouble(), 0.0001); + + assertFalse(exp.hasNext()); + + for (int i = 0; i < dps.length; i++) { + assertEquals(2, dps[i].tags().size()); + assertTrue(dps[i].aggregatedTags().isEmpty()); + } + assertArrayEquals(UIDS.get("D"), dps[0].tags().get(UIDS.get("D"))); + assertArrayEquals(UIDS.get("F"), dps[1].tags().get(UIDS.get("D"))); + } + + @Test + public void aDivideBWithTwoSeries() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + remapResults(); + + ExpressionIterator exp = new ExpressionIterator("ei", "a / b", + SetOperator.INTERSECTION, false, false); + exp.addResults("a", iterators.get("a")); + exp.addResults("b", iterators.get("b")); + + exp.compile(); + final ExpressionDataPoint[] dps = exp.values(); + assertEquals(2, dps.length); + validateMeta(dps, true); + + long ts = 1431561600000L; + long its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(0.0909, dps[0].toDouble(), 0.0001); + assertEquals(0.2857, dps[1].toDouble(), 0.0001); + ts += 60000; + + its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(0.1666, dps[0].toDouble(), 0.0001); + assertEquals(0.3333, dps[1].toDouble(), 0.0001); + ts += 60000; + + its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(0.2307, dps[0].toDouble(), 0.0001); + assertEquals(0.375, dps[1].toDouble(), 0.0001); + + assertFalse(exp.hasNext()); + + for (int i = 0; i < dps.length; i++) { + assertEquals(2, dps[i].tags().size()); + assertTrue(dps[i].aggregatedTags().isEmpty()); + } + assertArrayEquals(UIDS.get("D"), dps[0].tags().get(UIDS.get("D"))); + assertArrayEquals(UIDS.get("F"), dps[1].tags().get(UIDS.get("D"))); + } + + @Test + public void aModBWithTwoSeries() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + remapResults(); + + ExpressionIterator exp = new ExpressionIterator("ei", "a % b", + SetOperator.INTERSECTION, false, false); + exp.addResults("a", iterators.get("a")); + exp.addResults("b", iterators.get("b")); + + exp.compile(); + final ExpressionDataPoint[] dps = exp.values(); + assertEquals(2, dps.length); + validateMeta(dps, true); + + long ts = 1431561600000L; + long its = exp.nextTimestamp(); + double[] values = new double[] { 1, 4 }; + while (exp.hasNext()) { + exp.next(its); + + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(values[0]++, dps[0].toDouble(), 0.0001); + assertEquals(values[1]++, dps[1].toDouble(), 0.0001); + ts += 60000; + its = exp.nextTimestamp(); + } + + for (int i = 0; i < dps.length; i++) { + assertEquals(2, dps[i].tags().size()); + assertTrue(dps[i].aggregatedTags().isEmpty()); + } + assertArrayEquals(UIDS.get("D"), dps[0].tags().get(UIDS.get("D"))); + assertArrayEquals(UIDS.get("F"), dps[1].tags().get(UIDS.get("D"))); + } + + @Test + public void aDivideByZeroWithTwoSeries() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + remapResults(); + + // Jexl apparently happily allows this, just emits a zero + ExpressionIterator exp = new ExpressionIterator("ei", "a / 0", + SetOperator.INTERSECTION, false, false); + exp.addResults("a", iterators.get("a")); + exp.addResults("b", iterators.get("b")); + + exp.compile(); + final ExpressionDataPoint[] dps = exp.values(); + assertEquals(2, dps.length); + validateMeta(dps, true); + + long ts = 1431561600000L; + long its = exp.nextTimestamp(); + while (exp.hasNext()) { + exp.next(its); + + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(0, dps[0].toDouble(), 0.0001); + assertEquals(0, dps[1].toDouble(), 0.0001); + ts += 60000; + its = exp.nextTimestamp(); + } + + for (int i = 0; i < dps.length; i++) { + assertEquals(2, dps[i].tags().size()); + assertTrue(dps[i].aggregatedTags().isEmpty()); + } + assertArrayEquals(UIDS.get("D"), dps[0].tags().get(UIDS.get("D"))); + assertArrayEquals(UIDS.get("F"), dps[1].tags().get(UIDS.get("D"))); + } + + @Test + public void doubleVariableAndPrecedence() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + remapResults(); + + ExpressionIterator exp = new ExpressionIterator("ei", "a + (b * b)", + SetOperator.INTERSECTION, false, false); + exp.addResults("a", iterators.get("a")); + exp.addResults("b", iterators.get("b")); + + exp.compile(); + final ExpressionDataPoint[] dps = exp.values(); + assertEquals(2, dps.length); + validateMeta(dps, true); + + long ts = 1431561600000L; + long its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(122, dps[0].toDouble(), 0.0001); + assertEquals(200, dps[1].toDouble(), 0.0001); + ts += 60000; + + its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(146, dps[0].toDouble(), 0.0001); + assertEquals(230, dps[1].toDouble(), 0.0001); + ts += 60000; + + its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(172, dps[0].toDouble(), 0.0001); + assertEquals(262, dps[1].toDouble(), 0.0001); + + assertFalse(exp.hasNext()); + + for (int i = 0; i < dps.length; i++) { + assertEquals(2, dps[i].tags().size()); + assertTrue(dps[i].aggregatedTags().isEmpty()); + } + assertArrayEquals(UIDS.get("D"), dps[0].tags().get(UIDS.get("D"))); + assertArrayEquals(UIDS.get("F"), dps[1].tags().get(UIDS.get("D"))); + } + + @Test + public void doubleVariableAndPrecedenceChanged() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + remapResults(); + + ExpressionIterator exp = new ExpressionIterator("ei", "(a + b) * b", + SetOperator.INTERSECTION, false, false); + exp.addResults("a", iterators.get("a")); + exp.addResults("b", iterators.get("b")); + + exp.compile(); + final ExpressionDataPoint[] dps = exp.values(); + assertEquals(2, dps.length); + validateMeta(dps, true); + + long ts = 1431561600000L; + long its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(132, dps[0].toDouble(), 0.0001); + assertEquals(252, dps[1].toDouble(), 0.0001); + ts += 60000; + + its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(168, dps[0].toDouble(), 0.0001); + assertEquals(300, dps[1].toDouble(), 0.0001); + ts += 60000; + + its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(208, dps[0].toDouble(), 0.0001); + assertEquals(352, dps[1].toDouble(), 0.0001); + + assertFalse(exp.hasNext()); + + for (int i = 0; i < dps.length; i++) { + assertEquals(2, dps[i].tags().size()); + assertTrue(dps[i].aggregatedTags().isEmpty()); + } + assertArrayEquals(UIDS.get("D"), dps[0].tags().get(UIDS.get("D"))); + assertArrayEquals(UIDS.get("F"), dps[1].tags().get(UIDS.get("D"))); + } + + @Test + public void aPlusScalarDropB() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + remapResults(); + + ExpressionIterator exp = new ExpressionIterator("ei", "a + 1", + SetOperator.INTERSECTION, false, false); + exp.addResults("a", iterators.get("a")); + exp.addResults("b", iterators.get("b")); + + exp.compile(); + final ExpressionDataPoint[] dps = exp.values(); + assertEquals(2, dps.length); + validateMeta(dps, true); + + long ts = 1431561600000L; + double[] values = new double[] { 2, 5 }; + long its = exp.nextTimestamp(); + while (exp.hasNext()) { + exp.next(its); + + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(values[0]++, dps[0].toDouble(), 0.0001); + assertEquals(values[1]++, dps[1].toDouble(), 0.0001); + ts += 60000; + its = exp.nextTimestamp(); + } + + for (int i = 0; i < dps.length; i++) { + assertEquals(2, dps[i].tags().size()); + assertTrue(dps[i].aggregatedTags().isEmpty()); + } + assertArrayEquals(UIDS.get("D"), dps[0].tags().get(UIDS.get("D"))); + assertArrayEquals(UIDS.get("F"), dps[1].tags().get(UIDS.get("D"))); + } + + @Test (expected = IllegalArgumentException.class) + public void missingRequiredVariable() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + remapResults(); + + ExpressionIterator exp = new ExpressionIterator("ei", "a + b + c", + SetOperator.INTERSECTION, false, false); + exp.addResults("a", iterators.get("a")); + exp.addResults("b", iterators.get("b")); + + exp.compile(); + } + + @Test + public void aPlusBMissingPointsDefaultFillZero() throws Exception { + threeSameEGaps(); + queryAB_Dstar(); + remapResults(); + + ExpressionIterator exp = new ExpressionIterator("ei", "a + b", + SetOperator.INTERSECTION, false, false); + exp.addResults("a", iterators.get("a")); + exp.addResults("b", iterators.get("b")); + + exp.compile(); + final ExpressionDataPoint[] dps = exp.values(); + assertEquals(3, dps.length); + validateMeta(dps, true); + + long ts = 1431561600000L; + long its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + assertEquals(1, dps[0].toDouble(), 0.0001); + assertEquals(4, dps[1].toDouble(), 0.0001); + assertEquals(0, dps[2].toDouble(), 0.0001); + ts += 60000; + + its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + assertEquals(0, dps[0].toDouble(), 0.0001); + assertEquals(20, dps[1].toDouble(), 0.0001); + assertEquals(8, dps[2].toDouble(), 0.0001); + ts += 60000; + + its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + assertEquals(16, dps[0].toDouble(), 0.0001); + assertEquals(0, dps[1].toDouble(), 0.0001); + assertEquals(28, dps[2].toDouble(), 0.0001); + + assertFalse(exp.hasNext()); + + for (int i = 0; i < dps.length; i++) { + assertEquals(2, dps[i].tags().size()); + assertTrue(dps[i].aggregatedTags().isEmpty()); + } + assertArrayEquals(UIDS.get("D"), dps[0].tags().get(UIDS.get("D"))); + assertArrayEquals(UIDS.get("F"), dps[1].tags().get(UIDS.get("D"))); + assertArrayEquals(UIDS.get("G"), dps[2].tags().get(UIDS.get("D"))); + } + + @Test + public void aPlusBMissingPointsFillOne() throws Exception { + threeSameEGaps(); + queryAB_Dstar(); + remapResults(); + + ExpressionIterator exp = new ExpressionIterator("ei", "a + b", + SetOperator.INTERSECTION, false, false); + exp.addResults("a", iterators.get("a")); + exp.addResults("b", iterators.get("b")); + iterators.get("a").setFillPolicy(new NumericFillPolicy(FillPolicy.SCALAR, 1)); + iterators.get("b").setFillPolicy(new NumericFillPolicy(FillPolicy.SCALAR, 1)); + + exp.compile(); + final ExpressionDataPoint[] dps = exp.values(); + assertEquals(3, dps.length); + validateMeta(dps, true); + + long ts = 1431561600000L; + long its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + assertEquals(2, dps[0].toDouble(), 0.0001); + assertEquals(5, dps[1].toDouble(), 0.0001); + assertEquals(2, dps[2].toDouble(), 0.0001); + ts += 60000; + + its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + assertEquals(2, dps[0].toDouble(), 0.0001); + assertEquals(20, dps[1].toDouble(), 0.0001); + assertEquals(9, dps[2].toDouble(), 0.0001); + ts += 60000; + + its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + assertEquals(16, dps[0].toDouble(), 0.0001); + assertEquals(2, dps[1].toDouble(), 0.0001); + assertEquals(28, dps[2].toDouble(), 0.0001); + + assertFalse(exp.hasNext()); + + for (int i = 0; i < dps.length; i++) { + assertEquals(2, dps[i].tags().size()); + assertTrue(dps[i].aggregatedTags().isEmpty()); + } + assertArrayEquals(UIDS.get("D"), dps[0].tags().get(UIDS.get("D"))); + assertArrayEquals(UIDS.get("F"), dps[1].tags().get(UIDS.get("D"))); + assertArrayEquals(UIDS.get("G"), dps[2].tags().get(UIDS.get("D"))); + } + + @Test + public void aPlusBMissingPointsFillInfectiousNaN() throws Exception { + threeSameEGaps(); + queryAB_Dstar(); + remapResults(); + + ExpressionIterator exp = new ExpressionIterator("ei", "a + b", + SetOperator.INTERSECTION, false, false); + exp.addResults("a", iterators.get("a")); + exp.addResults("b", iterators.get("b")); + iterators.get("a").setFillPolicy( + new NumericFillPolicy(FillPolicy.NOT_A_NUMBER, Double.NaN)); + iterators.get("b").setFillPolicy( + new NumericFillPolicy(FillPolicy.NOT_A_NUMBER, Double.NaN)); + + exp.compile(); + final ExpressionDataPoint[] dps = exp.values(); + assertEquals(3, dps.length); + validateMeta(dps, true); + + long ts = 1431561600000L; + long its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + assertTrue(Double.isNaN(dps[0].toDouble())); + assertTrue(Double.isNaN(dps[1].toDouble())); + assertTrue(Double.isNaN(dps[2].toDouble())); + ts += 60000; + + its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + assertTrue(Double.isNaN(dps[0].toDouble())); + assertEquals(20, dps[1].toDouble(), 0.0001); + assertTrue(Double.isNaN(dps[2].toDouble())); + ts += 60000; + + its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + assertEquals(16, dps[0].toDouble(), 0.0001); + assertTrue(Double.isNaN(dps[1].toDouble())); + assertEquals(28, dps[2].toDouble(), 0.0001); + + assertFalse(exp.hasNext()); + + for (int i = 0; i < dps.length; i++) { + assertEquals(2, dps[i].tags().size()); + assertTrue(dps[i].aggregatedTags().isEmpty()); + } + assertArrayEquals(UIDS.get("D"), dps[0].tags().get(UIDS.get("D"))); + assertArrayEquals(UIDS.get("F"), dps[1].tags().get(UIDS.get("D"))); + assertArrayEquals(UIDS.get("G"), dps[2].tags().get(UIDS.get("D"))); + } + + @Test + public void aPlusBResultsOffsetDefaultFill() throws Exception { + timeOffset(); + queryAB_Dstar(); + remapResults(); + + ExpressionIterator exp = new ExpressionIterator("ei", "a + b", + SetOperator.INTERSECTION, false, false); + exp.addResults("a", iterators.get("a")); + exp.addResults("b", iterators.get("b")); + + exp.compile(); + final ExpressionDataPoint[] dps = exp.values(); + assertEquals(2, dps.length); + validateMeta(dps, true); + + long ts = 1431561600000L; + long its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(1, dps[0].toDouble(), 0.0001); + assertEquals(4, dps[1].toDouble(), 0.0001); + ts += 60000; + + its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(2, dps[0].toDouble(), 0.0001); + assertEquals(5, dps[1].toDouble(), 0.0001); + ts += 60000; + + its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(13, dps[0].toDouble(), 0.0001); + assertEquals(16, dps[1].toDouble(), 0.0001); + ts += 60000; + + its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(14, dps[0].toDouble(), 0.0001); + assertEquals(17, dps[1].toDouble(), 0.0001); + + assertFalse(exp.hasNext()); + + for (int i = 0; i < dps.length; i++) { + assertEquals(2, dps[i].tags().size()); + assertTrue(dps[i].aggregatedTags().isEmpty()); + } + assertArrayEquals(UIDS.get("D"), dps[0].tags().get(UIDS.get("D"))); + assertArrayEquals(UIDS.get("F"), dps[1].tags().get(UIDS.get("D"))); + } + + @Test + public void aPlusBOneAggedOneTaggedUseQueryTagsWoutQueryTags() throws Exception { + oneAggedTheOtherTagged(); + queryAB_AggAll(); + remapResults(); + + ExpressionIterator exp = new ExpressionIterator("ei", "a + b", + SetOperator.INTERSECTION, true, false); + exp.addResults("a", iterators.get("a")); + exp.addResults("b", iterators.get("b")); + + exp.compile(); + final ExpressionDataPoint[] dps = exp.values(); + assertEquals(1, dps.length); + // TODO - fix the TODO in the set operators to join tags + //validateMeta(dps, true); + + long ts = 1431561600000L; + double value = 13; + long its = exp.nextTimestamp(); + while (exp.hasNext()) { + exp.next(its); + + assertEquals(ts, dps[0].timestamp()); + assertEquals(value, dps[0].toDouble(), 0.0001); + + value += 3; + ts += 60000; + its = exp.nextTimestamp(); + } + + // TODO - fix the TODO in the set operators to join tags + //assertEquals(0, dps[0].tags().size()); + assertEquals(2, dps[0].aggregatedTags().size()); + assertTrue(dps[0].aggregatedTags().contains(UIDS.get("D"))); + assertTrue(dps[0].aggregatedTags().contains(UIDS.get("E"))); + // TODO - make sure the tags are empty once the expression data does it's + // thing + //assertTrue(dps[0].tags().isEmpty()); + } + + @Test + public void singleNestedExpression() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + remapResults(); + + ExpressionIterator ei = new ExpressionIterator("ei", "a + b", + SetOperator.INTERSECTION, false, false); + ei.addResults("a", iterators.get("a")); + ei.addResults("b", iterators.get("b")); + ei.compile(); + + ExpressionIterator exp = new ExpressionIterator("ei", "x * 2", + SetOperator.INTERSECTION, false, false); + exp.addResults("x", ei); + + exp.compile(); + final ExpressionDataPoint[] dps = exp.values(); + assertEquals(2, dps.length); + validateMeta(dps, true); + + long ts = 1431561600000L; + double[] values = new double[] { 24, 36 }; + long its = exp.nextTimestamp(); + while (exp.hasNext()) { + exp.next(its); + + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(values[0], dps[0].toDouble(), 0.0001); + assertEquals(values[1], dps[1].toDouble(), 0.0001); + + values[0] += 4; + values[1] += 4; + ts += 60000; + its = exp.nextTimestamp(); + } + + for (int i = 0; i < dps.length; i++) { + assertEquals(2, dps[i].tags().size()); + assertTrue(dps[i].aggregatedTags().isEmpty()); + } + assertArrayEquals(UIDS.get("D"), dps[0].tags().get(UIDS.get("D"))); + assertArrayEquals(UIDS.get("F"), dps[1].tags().get(UIDS.get("D"))); + } + + @Test + public void doubleNestedExpression() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + remapResults(); + + ExpressionIterator e1 = new ExpressionIterator("e1", "a + b", + SetOperator.INTERSECTION, false, false); + e1.addResults("a", iterators.get("a")); + e1.addResults("b", iterators.get("b")); + e1.compile(); + + ExpressionIterator e2 = new ExpressionIterator("e2", "e1 * 2", + SetOperator.INTERSECTION, false, false); + e2.addResults("e1", e1); + e2.compile(); + + ExpressionIterator e3 = new ExpressionIterator("e3", "e2 * 2", + SetOperator.INTERSECTION, false, false); + e3.addResults("e2", e2); + + e3.compile(); + final ExpressionDataPoint[] dps = e3.values(); + assertEquals(2, dps.length); + validateMeta(dps, true); + + long ts = 1431561600000L; + double[] values = new double[] { 48, 72 }; + long its = e3.nextTimestamp(); + while (e3.hasNext()) { + e3.next(its); + + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(values[0], dps[0].toDouble(), 0.0001); + assertEquals(values[1], dps[1].toDouble(), 0.0001); + + values[0] += 8; + values[1] += 8; + ts += 60000; + its = e3.nextTimestamp(); + } + + for (int i = 0; i < dps.length; i++) { + assertEquals(2, dps[i].tags().size()); + assertTrue(dps[i].aggregatedTags().isEmpty()); + } + assertArrayEquals(UIDS.get("D"), dps[0].tags().get(UIDS.get("D"))); + assertArrayEquals(UIDS.get("F"), dps[1].tags().get(UIDS.get("D"))); + } + + @Test (expected = IllegalDataException.class) + public void noIntersectionFound() throws Exception { + threeDifE(); + queryAB_Dstar(); + remapResults(); + + ExpressionIterator exp = new ExpressionIterator("ei", "a + b", + SetOperator.INTERSECTION, false, false); + exp.addResults("a", iterators.get("a")); + exp.addResults("b", iterators.get("b")); + + exp.compile(); + } + + @Test (expected = IllegalArgumentException.class) + public void addResultsMissingId() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + + ExpressionIterator exp = new ExpressionIterator("ei", "a + b + c", + SetOperator.INTERSECTION, false, false); + exp.addResults("a", iterators.get("a")); + } + + @Test (expected = IllegalArgumentException.class) + public void addResultsMissingSubQuery() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + + ExpressionIterator exp = new ExpressionIterator("ei", "a + b + c", + SetOperator.INTERSECTION, false, false); + exp.addResults("a", iterators.get("a")); + } + + @Test (expected = IllegalArgumentException.class) + public void addResultsMissingResults() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + + ExpressionIterator exp = new ExpressionIterator("ei", "a + b + c", + SetOperator.INTERSECTION, false, false); + exp.addResults("a", iterators.get("a")); + } + + @Test + public void unionOneExtraSeries() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + remapResults(); + + ExpressionIterator exp = new ExpressionIterator("ei", "a + b", + SetOperator.UNION, false, false); + exp.addResults("a", iterators.get("a")); + exp.addResults("b", iterators.get("b")); + + exp.compile(); + final ExpressionDataPoint[] dps = exp.values(); + assertEquals(3, dps.length); + // TODO - fix the TODO in the set operators to join tags + //validateMeta(dps, true); + + long ts = 1431561600000L; + double[] values = new double[] { 12, 18, 17 }; + long its = exp.nextTimestamp(); + while (exp.hasNext()) { + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + assertEquals(values[0], dps[0].toDouble(), 0.0001); + assertEquals(values[1], dps[1].toDouble(), 0.0001); + assertEquals(values[2], dps[2].toDouble(), 0.0001); + + values[0] += 2; + values[1] += 2; + values[2] += 1; + ts += 60000; + its = exp.nextTimestamp(); + } + + for (int i = 0; i < dps.length; i++) { + // TODO - fix the TODO in the set operators to join tags + //assertEquals(2, dps[i].tags().size()); + assertTrue(dps[i].aggregatedTags().isEmpty()); + } + assertArrayEquals(UIDS.get("D"), dps[0].tags().get(UIDS.get("D"))); + assertArrayEquals(UIDS.get("F"), dps[1].tags().get(UIDS.get("D"))); + } + + @Test + public void unionOffset() throws Exception { + timeOffset(); + queryAB_Dstar(); + remapResults(); + + ExpressionIterator exp = new ExpressionIterator("ei", "a + b", + SetOperator.UNION, false, false); + exp.addResults("a", iterators.get("a")); + exp.addResults("b", iterators.get("b")); + + exp.compile(); + final ExpressionDataPoint[] dps = exp.values(); + assertEquals(2, dps.length); + validateMeta(dps, true); + + long ts = 1431561600000L; + long its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(1, dps[0].toDouble(), 0.0001); + assertEquals(4, dps[1].toDouble(), 0.0001); + ts += 60000; + + its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(2, dps[0].toDouble(), 0.0001); + assertEquals(5, dps[1].toDouble(), 0.0001); + ts += 60000; + + its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(13, dps[0].toDouble(), 0.0001); + assertEquals(16, dps[1].toDouble(), 0.0001); + ts += 60000; + + its = exp.nextTimestamp(); + exp.next(its); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(14, dps[0].toDouble(), 0.0001); + assertEquals(17, dps[1].toDouble(), 0.0001); + + assertFalse(exp.hasNext()); + + for (int i = 0; i < dps.length; i++) { + assertEquals(2, dps[i].tags().size()); + assertTrue(dps[i].aggregatedTags().isEmpty()); + } + assertArrayEquals(UIDS.get("D"), dps[0].tags().get(UIDS.get("D"))); + assertArrayEquals(UIDS.get("F"), dps[1].tags().get(UIDS.get("D"))); + } + + @Test + public void unionNoIntersection() throws Exception { + threeDifE(); + queryAB_Dstar(); + remapResults(); + + ExpressionIterator exp = new ExpressionIterator("ei", "a + b", + SetOperator.UNION, false, false); + exp.addResults("a", iterators.get("a")); + exp.addResults("b", iterators.get("b")); + + exp.compile(); + final ExpressionDataPoint[] dps = exp.values(); + assertEquals(6, dps.length); + validateMeta(dps, false); + + long ts = 1431561600000L; + double[] values = new double[] { 1, 11, 4, 14, 7, 17 }; + long its = exp.nextTimestamp(); + while (exp.hasNext()) { + exp.next(its); + for (int i = 0; i < values.length; i++) { + assertEquals(ts, dps[i].timestamp()); + assertEquals(values[i], dps[i].toDouble(), 0.0001); + ++values[i]; + } + + ts += 60000; + its = exp.nextTimestamp(); + } + } + + @Test + public void unionSingleSeriesIteration() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + remapResults(); + + ExpressionIterator exp = new ExpressionIterator("ei", "a + b", + SetOperator.UNION, false, false); + exp.addResults("a", iterators.get("a")); + exp.addResults("b", iterators.get("b")); + + exp.compile(); + final ExpressionDataPoint[] dps = exp.values(); + double[] values = new double[] { 12, 18, 17 }; + + for (int i = 0; i < dps.length; i++) { + long ts = 1431561600000L; + while (exp.hasNext(i)) { + exp.next(i); + assertEquals(ts, dps[i].timestamp()); + assertEquals(values[i], dps[i].toDouble(), 0.001); + + ts += 60000; + if (i < dps.length - 1) { + values[i] += 2; + } else { + values[i]++; + } + } + } + } + + @Test + public void intersectionSingleSeriesIteration() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + remapResults(); + + ExpressionIterator exp = new ExpressionIterator("ei", "a + b", + SetOperator.INTERSECTION, false, false); + exp.addResults("a", iterators.get("a")); + exp.addResults("b", iterators.get("b")); + + exp.compile(); + final ExpressionDataPoint[] dps = exp.values(); + double[] values = new double[] { 12, 18 }; + + for (int i = 0; i < dps.length; i++) { + long ts = 1431561600000L; + while (exp.hasNext(i)) { + exp.next(i); + assertEquals(ts, dps[i].timestamp()); + assertEquals(values[i], dps[i].toDouble(), 0.001); + ts += 60000; + values[i] += 2; + } + } + + } + + @Test + public void aGreaterThanb() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + remapResults(); + + ExpressionIterator exp = new ExpressionIterator("ei", "a > b", + SetOperator.INTERSECTION, false, false); + exp.addResults("a", iterators.get("a")); + exp.addResults("b", iterators.get("b")); + + exp.compile(); + final ExpressionDataPoint[] dps = exp.values(); + assertEquals(2, dps.length); + validateMeta(dps, true); + + long ts = 1431561600000L; + double[] values = new double[] { 0, 0 }; + long its = exp.nextTimestamp(); + while (exp.hasNext()) { + exp.next(its); + + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(values[0], dps[0].toDouble(), 0.0001); + assertEquals(values[1], dps[1].toDouble(), 0.0001); + + ts += 60000; + its = exp.nextTimestamp(); + } + + for (int i = 0; i < dps.length; i++) { + assertEquals(2, dps[i].tags().size()); + assertTrue(dps[i].aggregatedTags().isEmpty()); + } + } + + @Test + public void aLessThanb() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + remapResults(); + + ExpressionIterator exp = new ExpressionIterator("ei", "a < b", + SetOperator.INTERSECTION, false, false); + exp.addResults("a", iterators.get("a")); + exp.addResults("b", iterators.get("b")); + + exp.compile(); + final ExpressionDataPoint[] dps = exp.values(); + assertEquals(2, dps.length); + validateMeta(dps, true); + + long ts = 1431561600000L; + double[] values = new double[] { 1, 1 }; + long its = exp.nextTimestamp(); + while (exp.hasNext()) { + exp.next(its); + + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(values[0], dps[0].toDouble(), 0.0001); + assertEquals(values[1], dps[1].toDouble(), 0.0001); + + ts += 60000; + its = exp.nextTimestamp(); + } + + for (int i = 0; i < dps.length; i++) { + assertEquals(2, dps[i].tags().size()); + assertTrue(dps[i].aggregatedTags().isEmpty()); + } + } + + /** + * Makes sure the series contain both metrics + * @param dps The results to validate + * @param common_e The common e + */ + private void validateMeta(final ExpressionDataPoint[] dps, + final boolean common_e) { + for (int i = 0; i < dps.length; i++) { + // TODO - change this guy to a byteset :( Since it's a bloody list we + // can't do a "contains" because it checks for the address of the byte + // arrays + boolean found = false; + for (final byte[] metric : dps[i].metricUIDs()) { + if (Bytes.memcmp(UIDS.get("A"), metric) == 0) { + found = true; + } else if (Bytes.memcmp(UIDS.get("B"), metric) == 0) { + found = true; + break; + } + } + if (!found) { + fail("Missing a metric"); + } + + if (common_e) { + assertArrayEquals(UIDS.get("E"), dps[i].tags().get(UIDS.get("E"))); + } + } + } + + private void remapResults() { + iterators.clear(); + iterators.put("a", new TimeSyncedIterator("a", + query.getQueries().get(0).getFilterTagKs(), results.get("0").getValue())); + iterators.put("b", new TimeSyncedIterator("b", + query.getQueries().get(1).getFilterTagKs(), results.get("1").getValue())); + } +} diff --git a/test/query/expression/TestExpressionReader.java b/test/query/expression/TestExpressionReader.java new file mode 100644 index 0000000000..3e4e7bd653 --- /dev/null +++ b/test/query/expression/TestExpressionReader.java @@ -0,0 +1,205 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.expression; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.NoSuchElementException; + +import org.junit.Test; + +public class TestExpressionReader { + final static String EXP = "test(sys.cpu.user)"; + + @Test + public void ctor() throws Exception { + final ExpressionReader reader = new ExpressionReader(EXP.toCharArray()); + assertEquals(EXP, reader.toString()); + assertEquals(0, reader.getMark()); + assertEquals('t', reader.peek()); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNull() throws Exception { + new ExpressionReader(null); + } + + @Test + public void ctorEmptyString() throws Exception { + final ExpressionReader reader = new ExpressionReader(new char[] { }); + assertEquals(0, reader.getMark()); + assertTrue(reader.isEOF()); + try { + reader.peek(); + fail("Expected a NoSuchElementException"); + } catch (NoSuchElementException e) { } + try { + reader.next(); + fail("Expected a NoSuchElementException"); + } catch (NoSuchElementException e) { } + try { + reader.isNextChar('o'); + fail("Expected a NoSuchElementException"); + } catch (NoSuchElementException e) { } + try { + reader.readFuncName(); + fail("Expected a NoSuchElementException"); + } catch (NoSuchElementException e) { } + reader.readNextParameter(); + // always false + assertFalse(reader.isNextSeq("laska")); + // no-op + reader.skipWhitespaces(); + // doesn't hurt anything + reader.skip(52); + } + + @Test + public void peek() throws Exception { + final ExpressionReader reader = new ExpressionReader(EXP.toCharArray()); + assertEquals(0, reader.getMark()); + assertEquals('t', reader.peek()); + reader.next(); + assertEquals(1, reader.getMark()); + assertEquals('e', reader.peek()); + assertEquals(1, reader.getMark()); + } + + @Test + public void nextTillEOF() throws Exception { + final char[] chars = EXP.toCharArray(); + final ExpressionReader reader = new ExpressionReader(EXP.toCharArray()); + for (final char c : chars) { + assertEquals(c, reader.next()); + } + assertTrue(reader.isEOF()); + } + + @Test + public void skip() throws Exception { + final ExpressionReader reader = new ExpressionReader(EXP.toCharArray()); + reader.skip(4); + assertEquals(4, reader.getMark()); + assertEquals('(', reader.peek()); + } + + @Test + public void skipOutOfBounds() throws Exception { + final ExpressionReader reader = new ExpressionReader(EXP.toCharArray()); + reader.skip(EXP.length()); + assertEquals(EXP.length(), reader.getMark()); + assertTrue(reader.isEOF()); + } + + @Test (expected = UnsupportedOperationException.class) + public void skipBackwards() throws Exception { + final ExpressionReader reader = new ExpressionReader(EXP.toCharArray()); + reader.skip(-1); + } + + @Test + public void isNextChar() throws Exception { + final ExpressionReader reader = new ExpressionReader(EXP.toCharArray()); + assertTrue(reader.isNextChar('t')); + reader.skip(4); + assertTrue(reader.isNextChar('(')); + assertFalse(reader.isNextChar('t')); + } + + @Test + public void isNextSeq() throws Exception { + final ExpressionReader reader = new ExpressionReader(EXP.toCharArray()); + assertTrue(reader.isNextSeq("test(")); + assertFalse(reader.isNextSeq("est(")); + assertTrue(reader.isNextSeq(EXP)); + assertFalse(reader.isNextSeq(EXP + "morestuff")); + } + + @Test + public void readFuncName() { + ExpressionReader reader = new ExpressionReader(EXP.toCharArray()); + assertEquals("test", reader.readFuncName()); + assertEquals("", reader.readFuncName()); + assertEquals("", reader.readFuncName()); + assertFalse(reader.isEOF()); + + // space between method and parens + reader = new ExpressionReader("test (foo)".toCharArray()); + assertEquals("test", reader.readFuncName()); + assertEquals("", reader.readFuncName()); + + // consume initial whitespace + reader = new ExpressionReader(" test(foo)".toCharArray()); + assertEquals("test", reader.readFuncName()); + assertEquals("", reader.readFuncName()); + + // whitespace everywhere!! + reader = new ExpressionReader(" test(foo) ".toCharArray()); + assertEquals("test", reader.readFuncName()); + assertEquals("", reader.readFuncName()); + + // nesting fails unless we consume the parens + reader = new ExpressionReader("test(foo(bar()))".toCharArray()); + assertEquals("test", reader.readFuncName()); + assertEquals("", reader.readFuncName()); + assertEquals("", reader.readFuncName()); + + reader = new ExpressionReader("test(foo(bar()))".toCharArray()); + assertEquals("test", reader.readFuncName()); + reader.next(); + assertEquals("foo", reader.readFuncName()); + reader.next(); + assertEquals("bar", reader.readFuncName()); + + // parens with space + reader = new ExpressionReader("test ( foo ( bar()))".toCharArray()); + assertEquals("test", reader.readFuncName()); + reader.next(); + assertEquals("foo", reader.readFuncName()); + reader.next(); + assertEquals("bar", reader.readFuncName()); + + // TODO - Watch out for the following gotchas + reader = new ExpressionReader("test ( foo bar()))".toCharArray()); + assertEquals("test", reader.readFuncName()); + reader.next(); + assertEquals("foo", reader.readFuncName()); + reader.next(); + assertEquals("ar", reader.readFuncName()); + + reader = new ExpressionReader("test ".toCharArray()); + assertEquals("test", reader.readFuncName()); + } + + // TODO - more UTs around this guy + @Test + public void readNextParameter() { + // will read the whole thing, so watch out! + ExpressionReader reader = new ExpressionReader(EXP.toCharArray()); + assertEquals(EXP, reader.readNextParameter()); + + // TODO - ok? + reader = new ExpressionReader(EXP.toCharArray()); + reader.readFuncName(); + assertEquals("(sys.cpu.user)", reader.readNextParameter()); + + // TODO - ok? + reader = new ExpressionReader("test(foo,1,2)".toCharArray()); + reader.readFuncName(); + reader.next(); + assertEquals("foo,1,2", reader.readNextParameter()); + } +} diff --git a/test/query/expression/TestExpressionTree.java b/test/query/expression/TestExpressionTree.java new file mode 100644 index 0000000000..ad1a5f37c7 --- /dev/null +++ b/test/query/expression/TestExpressionTree.java @@ -0,0 +1,319 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.expression; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.List; + +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.IllegalDataException; +import net.opentsdb.core.SeekableView; +import net.opentsdb.core.SeekableViewsForTest; +import net.opentsdb.core.TSQuery; +import net.opentsdb.query.expression.ExpressionTree.Parameter; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ TSQuery.class }) +public class TestExpressionTree { + private final static String EXPR_NAME = "treeTestExpr"; + private static long START_TIME = 1356998400000L; + private static int INTERVAL = 60000; + private static int NUM_POINTS = 5; + private static String METRIC = "sys.cpu"; + + private TSQuery data_query; + private SeekableView view; + private DataPoints dps; + private DataPoints[] group_bys; + private List<DataPoints[]> query_results; + private TreeTestExpr test_expression; + + @Before + public void before() throws Exception { + view = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 1, 1); + data_query = mock(TSQuery.class); + when(data_query.startTime()).thenReturn(START_TIME); + when(data_query.endTime()).thenReturn(START_TIME + (INTERVAL * NUM_POINTS)); + + dps = PowerMockito.mock(DataPoints.class); + when(dps.iterator()).thenReturn(view); + when(dps.metricName()).thenReturn(METRIC); + + group_bys = new DataPoints[] { dps }; + + query_results = new ArrayList<DataPoints[]>(1); + query_results.add(group_bys); + + test_expression = new TreeTestExpr(); + ExpressionFactory.addFunction(EXPR_NAME, test_expression); + } + + @Test + public void ctorString() throws Exception { + final ExpressionTree tree = new ExpressionTree(EXPR_NAME, data_query); + assertEquals(EXPR_NAME + "()", tree.toString()); + assertNull(tree.subExpressions()); + assertNull(tree.funcParams()); + assertNull(tree.subMetricQueries()); + assertTrue(tree.parameterIndex().isEmpty()); + } + + @Test (expected = UnsupportedOperationException.class) + public void ctorStringNull() throws Exception { + new ExpressionTree((String)null, data_query); + } + + @Test (expected = UnsupportedOperationException.class) + public void ctorStringEmpty() throws Exception { + new ExpressionTree("", data_query); + } + + @Test (expected = UnsupportedOperationException.class) + public void ctorStringUnknown() throws Exception { + new ExpressionTree("No such method", data_query); + } + + @Test + public void ctorExpression() throws Exception { + final ExpressionTree tree = new ExpressionTree(test_expression, data_query); + assertEquals(EXPR_NAME + "()", tree.toString()); + assertNull(tree.subExpressions()); + assertNull(tree.funcParams()); + assertNull(tree.subMetricQueries()); + assertTrue(tree.parameterIndex().isEmpty()); + } + + @Test + public void addSubExpression() throws Exception { + final ExpressionTree tree = new ExpressionTree(EXPR_NAME, data_query); + final ExpressionTree child = new ExpressionTree("scale", data_query); + tree.addSubExpression(child, 1); + assertEquals(1, tree.subExpressions().size()); + assertSame(child, tree.subExpressions().get(0)); + assertNull(tree.funcParams()); + assertNull(tree.subMetricQueries()); + assertEquals(1, tree.parameterIndex().size()); + assertEquals(Parameter.SUB_EXPRESSION, tree.parameterIndex().get(1)); + } + + @Test (expected = IllegalArgumentException.class) + public void addSubExpressionNullTree() throws Exception { + final ExpressionTree tree = new ExpressionTree(EXPR_NAME, data_query); + tree.addSubExpression(null, 1); + } + + @Test (expected = IllegalArgumentException.class) + public void addSubExpressionNegativeIndex() throws Exception { + final ExpressionTree tree = new ExpressionTree(EXPR_NAME, data_query); + tree.addSubExpression(null, 1); + } + + @Test (expected = IllegalDataException.class) + public void addSubExpressionRecursion() throws Exception { + final ExpressionTree tree = new ExpressionTree(EXPR_NAME, data_query); + tree.addSubExpression(tree, 1); + } + + @Test + public void addSubMetricQuery() throws Exception { + final ExpressionTree tree = new ExpressionTree(EXPR_NAME, data_query); + tree.addSubMetricQuery(METRIC, 1, 1); + assertEquals(EXPR_NAME + "(" + METRIC + ")", tree.toString()); + assertNull(tree.subExpressions()); + assertNull(tree.funcParams()); + assertEquals(1, tree.subMetricQueries().size()); + assertEquals(METRIC, tree.subMetricQueries().get(1)); + assertEquals(1, tree.parameterIndex().size()); + assertEquals(Parameter.METRIC_QUERY, tree.parameterIndex().get(1)); + } + + @Test (expected = IllegalArgumentException.class) + public void addSubMetricQueryNullMetric() throws Exception { + final ExpressionTree tree = new ExpressionTree(EXPR_NAME, data_query); + tree.addSubMetricQuery(null, 1, 1); + } + + @Test (expected = IllegalArgumentException.class) + public void addSubMetricQueryEmptyMetric() throws Exception { + final ExpressionTree tree = new ExpressionTree(EXPR_NAME, data_query); + tree.addSubMetricQuery("", 1, 1); + } + + @Test (expected = IllegalArgumentException.class) + public void addSubMetricQueryNegativeQueryIndex() throws Exception { + final ExpressionTree tree = new ExpressionTree(EXPR_NAME, data_query); + tree.addSubMetricQuery(METRIC, -1, 1); + } + + @Test (expected = IllegalArgumentException.class) + public void addSubMetricQueryNegativeParamIndex() throws Exception { + final ExpressionTree tree = new ExpressionTree(EXPR_NAME, data_query); + tree.addSubMetricQuery(METRIC, 1, -1); + } + + @Test + public void addFunctionParameter() throws Exception { + final ExpressionTree tree = new ExpressionTree(EXPR_NAME, data_query); + tree.addFunctionParameter("vimes"); + assertEquals(EXPR_NAME + "()", tree.toString()); + assertNull(tree.subExpressions()); + assertEquals(1, tree.funcParams().size()); + assertEquals("vimes", tree.funcParams().get(0)); + assertNull(tree.subMetricQueries()); + assertTrue(tree.parameterIndex().isEmpty()); + } + + @Test (expected = IllegalArgumentException.class) + public void addFunctionParameterNull() throws Exception { + final ExpressionTree tree = new ExpressionTree(EXPR_NAME, data_query); + tree.addFunctionParameter(null); + } + + @Test (expected = IllegalArgumentException.class) + public void addFunctionParameterEmpty() throws Exception { + final ExpressionTree tree = new ExpressionTree(EXPR_NAME, data_query); + tree.addFunctionParameter(""); + } + + @Test + public void evaluateNothingSet() throws Exception { + final ExpressionTree tree = new ExpressionTree(EXPR_NAME, data_query); + assertEquals(EXPR_NAME + "()", tree.toString()); + + final DataPoints[] response = tree.evaluate(query_results); + assertEquals(1, response.length); + assertSame(data_query, test_expression.data_query); + assertEquals(0, test_expression.results.size()); + assertNull(test_expression.params); + } + + @Test + public void evaluateSubMetricQuerySet() throws Exception { + final ExpressionTree tree = new ExpressionTree(EXPR_NAME, data_query); + tree.addSubMetricQuery(METRIC, 0, 0); + assertEquals(EXPR_NAME + "(" + METRIC + ")", tree.toString()); + + final DataPoints[] response = tree.evaluate(query_results); + assertEquals(1, response.length); + assertSame(data_query, test_expression.data_query); + assertEquals(1, test_expression.results.size()); + assertNull(test_expression.params); + } + + @Test + public void evaluateSubMetricQuerySetWithParam() throws Exception { + final ExpressionTree tree = new ExpressionTree(EXPR_NAME, data_query); + tree.addSubMetricQuery(METRIC, 0, 0); + tree.addFunctionParameter("foo"); + assertEquals(EXPR_NAME + "(" + METRIC + ")", tree.toString()); + + final DataPoints[] response = tree.evaluate(query_results); + assertEquals(1, response.length); + assertSame(data_query, test_expression.data_query); + assertEquals(1, test_expression.results.size()); + assertEquals(1, test_expression.params.size()); + assertEquals("foo", test_expression.params.get(0)); + } + + @Test + public void evaluateSubExpressionSet() throws Exception { + final ExpressionTree tree = new ExpressionTree(EXPR_NAME, data_query); + final ExpressionTree child = spy(new ExpressionTree("scale", data_query)); + child.addSubMetricQuery(METRIC, 0, 0); + child.addFunctionParameter("1"); + tree.addSubExpression(child, 0); + assertEquals(EXPR_NAME + "(scale(" + METRIC + "))", tree.toString()); + + final DataPoints[] response = tree.evaluate(query_results); + assertEquals(1, response.length); + assertSame(data_query, test_expression.data_query); + assertEquals(1, test_expression.results.size()); + assertNull(test_expression.params); + verify(child, times(1)).evaluate(query_results); + } + +// TODO - fix this up +// @Test +// public void evaluateSubExpressionAndSubMetricSet() throws Exception { +// final ExpressionTree tree = new ExpressionTree(EXPR_NAME, data_query); +// final ExpressionTree child = spy(new ExpressionTree("scale", data_query)); +// child.addSubMetricQuery(METRIC, 0, 0); +// child.addFunctionParameter("1"); +// tree.addSubExpression(child, 0); +// +// tree.addSubMetricQuery(METRIC, 1, 0); +// tree.addFunctionParameter("foo"); +// +// assertEquals(EXPR_NAME + "(scale(" + METRIC + ")," + METRIC + ")", +// tree.toString()); +// +// final DataPoints[] response = tree.evaluate(query_results); +// assertEquals(1, response.length); +// assertSame(data_query, test_expression.data_query); +// assertEquals(1, test_expression.results.size()); +// assertEquals(1, test_expression.params.size()); +// assertEquals("foo", test_expression.params.get(0)); +// verify(child, times(1)).evaluate(query_results); +// } + + // TODO - more tests around indexes, etc unless we cleanup the class + + private class TreeTestExpr implements Expression { + TSQuery data_query; + List<DataPoints[]> results; + List<String> params; + + @Override + public DataPoints[] evaluate(TSQuery data_query, + List<DataPoints[]> results, List<String> params) { + this.data_query = data_query; + this.results = results; + this.params = params; + + // Returns an array the size of the total incoming results array + int num_results = 0; + for (DataPoints[] r: query_results) { + num_results += r.length; + } + final DataPoints[] response = new DataPoints[num_results]; + return response; + } + @Override + public String writeStringField(List<String> params, + String inner_expression) { + return EXPR_NAME + "(" + inner_expression + ")"; + } + } +} diff --git a/test/query/expression/TestExpressions.java b/test/query/expression/TestExpressions.java new file mode 100644 index 0000000000..60dd6ec7c1 --- /dev/null +++ b/test/query/expression/TestExpressions.java @@ -0,0 +1,147 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.expression; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.mockito.Mockito.mock; + +import java.util.ArrayList; +import java.util.List; + +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.TSQuery; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ TSQuery.class }) +public class TestExpressions { + private TSQuery data_query; + private List<String> metric_queries; + + @Before + public void before() throws Exception { + data_query = mock(TSQuery.class); + metric_queries = new ArrayList<String>(); + ExpressionFactory.addFunction("foo", new FooExpression()); + } + + @Test + public void parse() throws Exception { + final ExpressionTree tree = Expressions.parse( + "scale(sys.cpu)", metric_queries, data_query); + assertEquals("scale()", tree.toString()); + } + + @Test + public void parseWithWhitespace() throws Exception { + final ExpressionTree tree = Expressions.parse( + " scale(sys.cpu)", metric_queries, data_query); + assertEquals("scale()", tree.toString()); + } + + @Test + public void parseMultiParameter() { + final String expr = "foo(sum:proc.sys.cpu,, sum:proc.meminfo.memfree)"; + final ExpressionTree tree = Expressions.parse(expr, metric_queries, null); + assertEquals("foo(proc.sys.cpu,proc.meminfo.memfree)", tree.toString()); + assertEquals(2, metric_queries.size()); + assertEquals("sum:proc.sys.cpu", metric_queries.get(0)); + assertEquals("sum:proc.meminfo.memfree", metric_queries.get(1)); + assertNull(tree.funcParams()); + } + + @Test + public void parseNestedExpr() { + final String expr = "foo(sum:proc.sys.cpu,, foo(sum:proc.a.b))"; + final ExpressionTree tree = Expressions.parse(expr, metric_queries, null); + assertEquals("foo(foo(proc.a.b),proc.sys.cpu)", tree.toString()); + assertEquals(2, metric_queries.size()); + assertEquals("sum:proc.sys.cpu", metric_queries.get(0)); + assertEquals("sum:proc.a.b", metric_queries.get(1)); + assertNull(tree.funcParams()); + } + + @Test + public void parseExprWithParam() { + final String expr = "foo(sum:proc.sys.cpu,, 100,, 3.1415)"; + final ExpressionTree tree = Expressions.parse(expr, metric_queries, null); + assertEquals("foo(proc.sys.cpu)", tree.toString()); + assertEquals(1, metric_queries.size()); + assertEquals("sum:proc.sys.cpu", metric_queries.get(0)); + assertEquals(2, tree.funcParams().size()); + assertEquals("100", tree.funcParams().get(0)); + assertEquals("3.1415", tree.funcParams().get(1)); + } + + @Test (expected = IllegalArgumentException.class) + public void parseNullExpression() throws Exception { + Expressions.parse(null, metric_queries, data_query); + } + + @Test (expected = IllegalArgumentException.class) + public void parseEmptyExpression() throws Exception { + Expressions.parse("", metric_queries, data_query); + } + + @Test (expected = IllegalArgumentException.class) + public void parseMissingOpenParens() throws Exception { + Expressions.parse("scalesys.cpu)", metric_queries, data_query); + } + + @Test (expected = IllegalArgumentException.class) + public void parseMissingClosingParens() throws Exception { + Expressions.parse("scale(sys.cpu", metric_queries, data_query); + } + + // TODO - These two may be problematic and need validation/fixing? + @Test + public void parseNullMetricQueries() throws Exception { + final ExpressionTree tree = Expressions.parse( + "scale(sys.cpu)", null, data_query); + assertEquals("scale()", tree.toString()); + } + + @Test + public void parseNullTSQuery() throws Exception { + final ExpressionTree tree = Expressions.parse( + "scale(sys.cpu)", metric_queries, null); + assertEquals("scale()", tree.toString()); + } + + //TODO - Need to add more tests around parsing nested functions and params + + /** Dummy test expression implementation */ + private static class FooExpression implements Expression { + @Override + public DataPoints[] evaluate(final TSQuery data_query, + final List<DataPoints[]> query_results, final List<String> params) { + return new DataPoints[0]; + } + + @Override + public String writeStringField(final List<String> query_params, + final String inner_expressions) { + return "foo(" + inner_expressions + ")"; + } + } +} diff --git a/test/query/expression/TestFirstDifference.java b/test/query/expression/TestFirstDifference.java new file mode 100644 index 0000000000..031f0e5e34 --- /dev/null +++ b/test/query/expression/TestFirstDifference.java @@ -0,0 +1,348 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.expression; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.SeekableView; +import net.opentsdb.core.SeekableViewsForTest; +import net.opentsdb.core.TSQuery; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.stumbleupon.async.Deferred; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({TSQuery.class}) +public class TestFirstDifference { + + private static long START_TIME = 1356998400000L; + private static int INTERVAL = 60000; + private static int NUM_POINTS = 5; + private static String METRIC = "sys.cpu"; + + private TSQuery data_query; + private SeekableView view; + private DataPoints dps; + private DataPoints[] group_bys; + private List<DataPoints[]> query_results; + private List<String> params; + private net.opentsdb.query.expression.FirstDifference func; + + @Before + public void before() throws Exception { + view = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 1, 1); + data_query = mock(TSQuery.class); + when(data_query.startTime()).thenReturn(START_TIME); + when(data_query.endTime()).thenReturn(START_TIME + (INTERVAL * NUM_POINTS)); + + dps = PowerMockito.mock(DataPoints.class); + when(dps.iterator()).thenReturn(view); + when(dps.metricNameAsync()).thenReturn(Deferred.fromResult(METRIC)); + + group_bys = new DataPoints[]{dps}; + + query_results = new ArrayList<DataPoints[]>(1); + query_results.add(group_bys); + + params = new ArrayList<String>(1); + func = new net.opentsdb.query.expression.FirstDifference(); + } + + @Test + public void evaluatePositiveGroupByLong() throws Exception { + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[]{dps, dps2}; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals(METRIC, results[0].metricName()); + assertEquals("sys.mem", results[1].metricName()); + + long ts = START_TIME; + long v = 0; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(),0.001); + ts += INTERVAL; + v = 1; + } + ts = START_TIME; + v = 0; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + v = 1; + } + } + + @Test + public void evaluatePositiveGroupByDouble() throws Exception { + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, false, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[]{dps, dps2}; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals(METRIC, results[0].metricName()); + assertEquals("sys.mem", results[1].metricName()); + + long ts = START_TIME; + double v = 0; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + v =1; + } + ts = START_TIME; + v = 0; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + v = 1; + } + } + + @Test + public void evaluatePositiveGroupBy1point5Double() throws Exception { + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, false, 10, 1.5); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[]{dps, dps2}; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals(METRIC, results[0].metricName()); + assertEquals("sys.mem", results[1].metricName()); + + long ts = START_TIME; + double v = 0; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + v =1; + } + ts = START_TIME; + v = 0; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + v = 1.5; + } + } + + @Test + public void evaluateFactorNegativeGroupByLong() throws Exception { + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, -10, -1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[]{dps, dps2}; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals(METRIC, results[0].metricName()); + assertEquals("sys.mem", results[1].metricName()); + + long ts = START_TIME; + long v = 0; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + v = 1; + } + ts = START_TIME; + v = 0; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + v = -1; + } + } + + @Test + public void evaluateNegativeGroupByDouble() throws Exception { + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, false, -10, -1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[]{dps, dps2}; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals(METRIC, results[0].metricName()); + assertEquals("sys.mem", results[1].metricName()); + + long ts = START_TIME; + double v = 0; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + v = 1; + } + ts = START_TIME; + v = 0; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + v = -1; + } + } + + @Test + public void evaluateNegativeSubQuerySeries() throws Exception { + params.add("1"); + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, -10, -1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[]{dps, dps2}; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals(METRIC, results[0].metricName()); + assertEquals("sys.mem", results[1].metricName()); + + long ts = START_TIME; + long v = 0; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + v = 1; + } + ts = START_TIME; + v = 0; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + v = -1; + } + } + + @Test(expected = IllegalArgumentException.class) + public void evaluateNullQuery() throws Exception { + params.add("1"); + func.evaluate(null, query_results, params); + } + + @Test + public void evaluateNullResults() throws Exception { + params.add("1"); + final DataPoints[] results = func.evaluate(data_query, null, params); + assertEquals(0, results.length); + } + + @Test + public void evaluateNullParams() throws Exception { + assertNotNull(func.evaluate(data_query, query_results, null)); + } + + @Test + public void evaluateEmptyResults() throws Exception { + params.add("1"); + final DataPoints[] results = func.evaluate(data_query, + Collections.<DataPoints[]>emptyList(), params); + assertEquals(0, results.length); + } + + @Test + public void evaluateEmptyParams() throws Exception { + assertNotNull(func.evaluate(data_query, query_results, null)); + } + + @Test + public void writeStringField() throws Exception { + params.add("1"); + assertEquals("firstDiff(inner_expression)", + func.writeStringField(params, "inner_expression")); + assertEquals("firstDiff(null)", func.writeStringField(params, null)); + assertEquals("firstDiff()", func.writeStringField(params, "")); + assertEquals("firstDiff(inner_expression)", + func.writeStringField(null, "inner_expression")); + } +} diff --git a/test/query/expression/TestHighestCurrent.java b/test/query/expression/TestHighestCurrent.java new file mode 100644 index 0000000000..876ee6356d --- /dev/null +++ b/test/query/expression/TestHighestCurrent.java @@ -0,0 +1,395 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.expression; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.SeekableView; +import net.opentsdb.core.SeekableViewsForTest; +import net.opentsdb.core.TSQuery; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.stumbleupon.async.Deferred; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ TSQuery.class }) +public class TestHighestCurrent { + + private static long START_TIME = 1356998400000L; + private static int INTERVAL = 60000; + private static int NUM_POINTS = 5; + private static String METRIC = "sys.cpu"; + + private TSQuery data_query; + private SeekableView view; + private DataPoints dps; + private DataPoints[] group_bys; + private List<DataPoints[]> query_results; + private List<String> params; + private HighestCurrent func; + + @Before + public void before() throws Exception { + view = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 1, 1); + data_query = mock(TSQuery.class); + when(data_query.startTime()).thenReturn(START_TIME); + when(data_query.endTime()).thenReturn(START_TIME + (INTERVAL * NUM_POINTS)); + + dps = PowerMockito.mock(DataPoints.class); + when(dps.iterator()).thenReturn(view); + when(dps.metricNameAsync()).thenReturn(Deferred.fromResult(METRIC)); + + group_bys = new DataPoints[] { dps }; + + query_results = new ArrayList<DataPoints[]>(1); + query_results.add(group_bys); + + params = new ArrayList<String>(1); + func = new HighestCurrent(); + } + + @Test + public void evaluateTopN1with2SeriesLong() throws Exception { + params.add("1"); + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(1, results.length); + assertEquals("sys.mem", results[0].metricName()); + + long ts = START_TIME; + long v = 10; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + } + + @Test + public void evaluateTopN2with2SeriesLong() throws Exception { + params.add("2"); + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals("sys.mem", results[0].metricName()); + assertEquals(METRIC, results[1].metricName()); + + long ts = START_TIME; + long v = 10; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + + ts = START_TIME; + v = 1; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + } + + @Test + public void evaluateTopN100with2SeriesLong() throws Exception { + params.add("100"); + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + DataPoints[] group_bys2 = new DataPoints[] { dps2 }; + query_results.add(group_bys2); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals("sys.mem", results[0].metricName()); + assertEquals(METRIC, results[1].metricName()); + + long ts = START_TIME; + long v = 10; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + + ts = START_TIME; + v = 1; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + } + + @Test + public void evaluateTopN100with2SubQuerySeriesLong() throws Exception { + params.add("100"); + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals("sys.mem", results[0].metricName()); + assertEquals(METRIC, results[1].metricName()); + + long ts = START_TIME; + long v = 10; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + + ts = START_TIME; + v = 1; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + } + + @Test + public void evaluateTopN2with2SeriesDouble() throws Exception { + params.add("2"); + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, false, 10, 1.5); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals("sys.mem", results[0].metricName()); + assertEquals(METRIC, results[1].metricName()); + + long ts = START_TIME; + double v = 10; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + v += 1.5; + } + + ts = START_TIME; + v = 1; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.toDouble(), 0.001); + ts += INTERVAL; + v += 1; + } + } + + @Test + public void evaluateTopN1with2SeriesLongDoubleMixed() throws Exception { + params.add("1"); + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, false, 10, 1.5, true); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(1, results.length); + assertEquals("sys.mem", results[0].metricName()); + + long ts = START_TIME; + double v = 10; + boolean toggle = true; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + if (toggle) { + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue(), 0.001); + } else { + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + } + toggle = !toggle; + ts += INTERVAL; + v += 1.5; + } + } + + @Test + public void evaluateTopN1with2SeriesDiffSpan() throws Exception { + params.add("1"); + // in this case one series ends earlier than the other so it's removed + // even though it's last value was greater than the winners. + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + 3, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(1, results.length); + assertEquals(METRIC, results[0].metricName()); + + long ts = START_TIME; + long v = 1; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateNullQuery() throws Exception { + params.add("1"); + func.evaluate(null, query_results, params); + } + + @Test + public void evaluateNullResults() throws Exception { + params.add("1"); + final DataPoints[] results = func.evaluate(data_query, null, params); + assertEquals(0, results.length); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateNullParams() throws Exception { + func.evaluate(data_query, query_results, null); + } + + @Test + public void evaluateEmptyResults() throws Exception { + params.add("1"); + final DataPoints[] results = func.evaluate(data_query, + Collections.<DataPoints[]>emptyList(), params); + assertEquals(0, results.length); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateEmptyParams() throws Exception { + func.evaluate(data_query, query_results, params); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateTopnNull() throws Exception { + params.add(null); + func.evaluate(data_query, query_results, params); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateTopnEmpty() throws Exception { + params.add(""); + func.evaluate(data_query, query_results, params); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateTopnZero() throws Exception { + params.add("0"); + func.evaluate(data_query, query_results, params); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateTopnNotaNumber() throws Exception { + params.add("not a number"); + func.evaluate(data_query, query_results, params); + } + + @Test + public void writeStringField() throws Exception { + params.add("1"); + assertEquals("highestCurrent(inner_expression)", + func.writeStringField(params, "inner_expression")); + assertEquals("highestCurrent(null)", func.writeStringField(params, null)); + assertEquals("highestCurrent()", func.writeStringField(params, "")); + assertEquals("highestCurrent(inner_expression)", + func.writeStringField(null, "inner_expression")); + } +} diff --git a/test/query/expression/TestHighestMax.java b/test/query/expression/TestHighestMax.java new file mode 100644 index 0000000000..66e38689a7 --- /dev/null +++ b/test/query/expression/TestHighestMax.java @@ -0,0 +1,365 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.expression; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.SeekableView; +import net.opentsdb.core.SeekableViewsForTest; +import net.opentsdb.core.TSQuery; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.stumbleupon.async.Deferred; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ TSQuery.class }) +public class TestHighestMax { + + private static long START_TIME = 1356998400000L; + private static int INTERVAL = 60000; + private static int NUM_POINTS = 5; + private static String METRIC = "sys.cpu"; + + private TSQuery data_query; + private SeekableView view; + private DataPoints dps; + private DataPoints[] group_bys; + private List<DataPoints[]> query_results; + private List<String> params; + private HighestMax func; + + @Before + public void before() throws Exception { + view = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 1, 1); + data_query = mock(TSQuery.class); + when(data_query.startTime()).thenReturn(START_TIME); + when(data_query.endTime()).thenReturn(START_TIME + (INTERVAL * NUM_POINTS)); + + dps = PowerMockito.mock(DataPoints.class); + when(dps.iterator()).thenReturn(view); + when(dps.metricNameAsync()).thenReturn(Deferred.fromResult(METRIC)); + + group_bys = new DataPoints[] { dps }; + + query_results = new ArrayList<DataPoints[]>(1); + query_results.add(group_bys); + + params = new ArrayList<String>(1); + func = new HighestMax(); + } + + @Test + public void evaluateTopN1with2SeriesLong() throws Exception { + params.add("1"); + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(1, results.length); + assertEquals("sys.mem", results[0].metricName()); + + long ts = START_TIME; + long v = 10; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + } + + @Test + public void evaluateTopN2with2SeriesLong() throws Exception { + params.add("2"); + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals("sys.mem", results[0].metricName()); + assertEquals(METRIC, results[1].metricName()); + + long ts = START_TIME; + long v = 10; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + + ts = START_TIME; + v = 1; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + } + + @Test + public void evaluateTopN100with2SeriesLong() throws Exception { + params.add("100"); + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + DataPoints[] group_bys2 = new DataPoints[] { dps2 }; + query_results.add(group_bys2); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals("sys.mem", results[0].metricName()); + assertEquals(METRIC, results[1].metricName()); + + long ts = START_TIME; + long v = 10; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + + ts = START_TIME; + v = 1; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + } + + @Test + public void evaluateTopN100with2SubQuerySeriesLong() throws Exception { + params.add("100"); + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals("sys.mem", results[0].metricName()); + assertEquals(METRIC, results[1].metricName()); + + long ts = START_TIME; + long v = 10; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + + ts = START_TIME; + v = 1; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + } + + @Test + public void evaluateTopN2with2SeriesDouble() throws Exception { + params.add("2"); + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, false, 10, 1.5); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals("sys.mem", results[0].metricName()); + assertEquals(METRIC, results[1].metricName()); + + long ts = START_TIME; + double v = 10; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + v += 1.5; + } + + ts = START_TIME; + v = 1; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.toDouble(), 0.001); + ts += INTERVAL; + v += 1; + } + } + + @Test + public void evaluateTopN1with2SeriesLongDoubleMixed() throws Exception { + params.add("1"); + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, false, 10, 1.5, true); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(1, results.length); + assertEquals("sys.mem", results[0].metricName()); + + long ts = START_TIME; + double v = 10; + boolean toggle = true; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + if (toggle) { + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue(), 0.001); + } else { + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + } + toggle = !toggle; + ts += INTERVAL; + v += 1.5; + } + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateNullQuery() throws Exception { + params.add("1"); + func.evaluate(null, query_results, params); + } + + @Test + public void evaluateNullResults() throws Exception { + params.add("1"); + final DataPoints[] results = func.evaluate(data_query, null, params); + assertEquals(0, results.length); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateNullParams() throws Exception { + func.evaluate(data_query, query_results, null); + } + + @Test + public void evaluateEmptyResults() throws Exception { + params.add("1"); + final DataPoints[] results = func.evaluate(data_query, + Collections.<DataPoints[]>emptyList(), params); + assertEquals(0, results.length); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateEmptyParams() throws Exception { + func.evaluate(data_query, query_results, params); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateTopnNull() throws Exception { + params.add(null); + func.evaluate(data_query, query_results, params); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateTopnEmpty() throws Exception { + params.add(""); + func.evaluate(data_query, query_results, params); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateTopnZero() throws Exception { + params.add("0"); + func.evaluate(data_query, query_results, params); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateTopnNotaNumber() throws Exception { + params.add("not a number"); + func.evaluate(data_query, query_results, params); + } + + @Test + public void writeStringField() throws Exception { + params.add("1"); + assertEquals("highestMax(inner_expression)", + func.writeStringField(params, "inner_expression")); + assertEquals("highestMax(null)", func.writeStringField(params, null)); + assertEquals("highestMax()", func.writeStringField(params, "")); + assertEquals("highestMax(inner_expression)", + func.writeStringField(null, "inner_expression")); + } +} diff --git a/test/query/expression/TestIntersectionIterator.java b/test/query/expression/TestIntersectionIterator.java new file mode 100644 index 0000000000..80adf6ca43 --- /dev/null +++ b/test/query/expression/TestIntersectionIterator.java @@ -0,0 +1,813 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.expression; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.HashMap; +import java.util.Map; + +import org.hbase.async.HBaseClient; +import org.hbase.async.Bytes.ByteMap; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.FillPolicy; +import net.opentsdb.core.IllegalDataException; +import net.opentsdb.storage.MockBase; +import net.opentsdb.utils.ByteSet; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ TimeSyncedIterator.class }) +public class TestIntersectionIterator extends BaseTimeSyncedIteratorTest { + + /** used for the flattenTags tests */ + private static final byte[] UID1 = new byte[] { 0, 0, 1 }; + private static final byte[] UID2 = new byte[] { 0, 0, 2 }; + private static final byte[] UID3 = new byte[] { 0, 0, 3 }; + private ByteMap<byte[]> tags; + private ByteSet agg_tags; + private ITimeSyncedIterator sub; + private ByteSet query_tags; + + @Before + public void beforeLocal() throws Exception { + tags = new ByteMap<byte[]>(); + tags.put(UID1, UID1); + tags.put(UID2, UID2); + + agg_tags = new ByteSet(); + agg_tags.add(UID3); + + sub = mock(ITimeSyncedIterator.class); + query_tags = new ByteSet(); + query_tags.add(UID1); + when(sub.getQueryTagKs()).thenReturn(query_tags); + } + + @Test (expected = NullPointerException.class) + public void ctorNullResults() { + new IntersectionIterator("it", null, true, true); + } + + @Test + public void ctorEmptyResults() { + final IntersectionIterator it = new IntersectionIterator("it", + new HashMap<String, ITimeSyncedIterator>(), true, true); + assertEquals(0, it.getSeriesSize()); + assertFalse(it.hasNext()); + } + + @Test + public void twoAndThreeSeries() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + + final IntersectionIterator it = new IntersectionIterator("it", iterators, false, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(2, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 1, 4, 11, 14 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(2, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(values[0]++, set_dps[0].toDouble(), 0.0001); + assertEquals(values[1]++, set_dps[1].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(2, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(values[2]++, set_dps[0].toDouble(), 0.0001); + assertEquals(values[3]++, set_dps[1].toDouble(), 0.0001); + + ts += 60000; + } + } + + @Test + public void twoAndThreeSeriesExtraDPinKickedSeries() throws Exception { + // in this case we want to make sure the kicked series doesn't cause us + // to dump a bunch of nulls + oneExtraSameE(); + queryAB_Dstar(); + + final HashMap<String, String> tags = new HashMap<String, String>(2); + tags.put("D", "G"); + tags.put("E", "E"); + tsdb.addPoint("B", 1431561630, 14, tags).joinUninterruptibly(); + + final IntersectionIterator it = new IntersectionIterator("it", iterators, false, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(2, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 1, 4, 11, 14 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(2, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(values[0]++, set_dps[0].toDouble(), 0.0001); + assertEquals(values[1]++, set_dps[1].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(2, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(values[2]++, set_dps[0].toDouble(), 0.0001); + assertEquals(values[3]++, set_dps[1].toDouble(), 0.0001); + + ts += 60000; + } + } + + @Test + public void threeSeriesIntersectToTwo() throws Exception { + threeDisjointSameE(); + queryAB_Dstar(); + + final IntersectionIterator it = new IntersectionIterator("it", iterators, false, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(2, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 1, 7, 11, 17 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(2, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(values[0]++, set_dps[0].toDouble(), 0.0001); + assertEquals(values[1]++, set_dps[1].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(2, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(values[2]++, set_dps[0].toDouble(), 0.0001); + assertEquals(values[3]++, set_dps[1].toDouble(), 0.0001); + + ts += 60000; + } + } + + @Test + public void threeSeriesIntersectToExtraDPsinKicked() throws Exception { + reduceToOne(); + queryAB_Dstar(); + + HashMap<String, String> tags = new HashMap<String, String>(2); + tags.put("D", "F"); + tags.put("E", "E"); + tsdb.addPoint("A", 1431561630, 1024, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "Q"); + tags.put("E", "E"); + tsdb.addPoint("B", 1431561630, 1024, tags).joinUninterruptibly(); + + final IntersectionIterator it = new IntersectionIterator("it", iterators, false, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(1, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 7, 17 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(1, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(values[0]++, set_dps[0].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(1, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(values[1]++, set_dps[0].toDouble(), 0.0001); + + ts += 60000; + } + } + + @Test + public void threeSeriesIntersectToOne() throws Exception { + reduceToOne(); + queryAB_Dstar(); + + final IntersectionIterator it = new IntersectionIterator("it", iterators, false, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(1, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 7, 17 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(1, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(values[0]++, set_dps[0].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(1, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(values[1]++, set_dps[0].toDouble(), 0.0001); + + ts += 60000; + } + } + + @Test + public void threeSeriesAggedIntoOne() throws Exception { + threeSameE(); + queryAB_AggAll(); + + final IntersectionIterator it = new IntersectionIterator("it", iterators, false, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(1, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 12, 42 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(1, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(values[0], set_dps[0].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(1, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(values[1], set_dps[0].toDouble(), 0.0001); + + values[0] += 3; + values[1] += 3; + + ts += 60000; + } + } + + @Test + public void threeSeriesFullIntersetWithNaNs() throws Exception { + threeSameEGaps(); + queryAB_Dstar(); + for (ITimeSyncedIterator iterator : iterators.values()) { + iterator.setFillPolicy(new NumericFillPolicy(FillPolicy.NOT_A_NUMBER)); + } + + final IntersectionIterator it = new IntersectionIterator("it", iterators, false, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(3, it.getSeriesSize()); + + long ts = 1431561600000L; + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(1, set_dps[0].toDouble(), 0.0001); + assertEquals(4, set_dps[1].toDouble(), 0.0001); + assertTrue(Double.isNaN(set_dps[2].toDouble())); + + // whole series is NaN'd + set_dps = dps.get("1"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertTrue(Double.isNaN(set_dps[0].toDouble())); + assertTrue(Double.isNaN(set_dps[1].toDouble())); + assertTrue(Double.isNaN(set_dps[2].toDouble())); + + ts += 60000; + it.next(); + + set_dps = dps.get("0"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertTrue(Double.isNaN(set_dps[0].toDouble())); + assertEquals(5, set_dps[1].toDouble(), 0.0001); + assertEquals(8, set_dps[2].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertTrue(Double.isNaN(set_dps[0].toDouble())); + assertEquals(15, set_dps[1].toDouble(), 0.0001); + assertTrue(Double.isNaN(set_dps[2].toDouble())); + + ts += 60000; + it.next(); + + set_dps = dps.get("0"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(3, set_dps[0].toDouble(), 0.0001); + assertTrue(Double.isNaN(set_dps[1].toDouble())); + assertEquals(9, set_dps[2].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(13, set_dps[0].toDouble(), 0.0001); + assertTrue(Double.isNaN(set_dps[1].toDouble())); + assertEquals(19, set_dps[2].toDouble(), 0.0001); + + assertFalse(it.hasNext()); + } + + @Test + public void twoSeriesTimeOffset() throws Exception { + timeOffset(); + queryAB_Dstar(); + for (ITimeSyncedIterator iterator : iterators.values()) { + iterator.setFillPolicy(new NumericFillPolicy(FillPolicy.NOT_A_NUMBER)); + } + + final IntersectionIterator it = new IntersectionIterator("it", iterators, false, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(2, it.getSeriesSize()); + + long ts = 1431561600000L; + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(2, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(1, set_dps[0].toDouble(), 0.0001); + assertEquals(4, set_dps[1].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(2, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertTrue(Double.isNaN(set_dps[0].toDouble())); + assertTrue(Double.isNaN(set_dps[1].toDouble())); + + ts += 60000; + it.next(); + + set_dps = dps.get("0"); + assertEquals(2, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(2, set_dps[0].toDouble(), 0.0001); + assertEquals(5, set_dps[1].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(2, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertTrue(Double.isNaN(set_dps[0].toDouble())); + assertTrue(Double.isNaN(set_dps[1].toDouble())); + + ts += 60000; + it.next(); + + set_dps = dps.get("0"); + assertEquals(2, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertTrue(Double.isNaN(set_dps[0].toDouble())); + assertTrue(Double.isNaN(set_dps[1].toDouble())); + + set_dps = dps.get("1"); + assertEquals(2, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(13, set_dps[0].toDouble(), 0.0001); + assertEquals(16, set_dps[1].toDouble(), 0.0001); + + ts += 60000; + it.next(); + + set_dps = dps.get("0"); + assertEquals(2, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertTrue(Double.isNaN(set_dps[0].toDouble())); + assertTrue(Double.isNaN(set_dps[1].toDouble())); + + set_dps = dps.get("1"); + assertEquals(2, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(14, set_dps[0].toDouble(), 0.0001); + assertEquals(17, set_dps[1].toDouble(), 0.0001); + + assertFalse(it.hasNext()); + } + + @Test (expected = IllegalDataException.class) + public void noIntersectionUsingResultTags() throws Exception { + threeDifE(); + queryAB_Dstar(); + new IntersectionIterator("it", iterators, false, false); + } + + @Test + public void intersectUsingQueryTags() throws Exception { + threeDifE(); + queryAB_Dstar(); + + final IntersectionIterator it = new IntersectionIterator("it", iterators, true, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(3, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 1, 4, 7, 11, 14, 17 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(values[0]++, set_dps[0].toDouble(), 0.0001); + assertEquals(values[1]++, set_dps[1].toDouble(), 0.0001); + assertEquals(values[2]++, set_dps[2].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(values[3]++, set_dps[0].toDouble(), 0.0001); + assertEquals(values[4]++, set_dps[1].toDouble(), 0.0001); + assertEquals(values[5]++, set_dps[2].toDouble(), 0.0001); + + ts += 60000; + } + } + + @Test + public void commonAggregatedTag() throws Exception { + twoSeriesAggedE(); + queryAB_Dstar(); + + final IntersectionIterator it = new IntersectionIterator("it", iterators, false, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(1, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 2, 22 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(1, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(values[0], set_dps[0].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(1, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(values[1], set_dps[0].toDouble(), 0.0001); + + values[0] += 2; + values[1] += 2; + + ts += 60000; + } + } + + @Test + public void extraAggTagIgnored() throws Exception { + twoSeriesAggedEandExtraTagK(); + queryAB_Dstar(); + + final IntersectionIterator it = new IntersectionIterator("it", iterators, false, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(1, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 2, 22 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(1, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(values[0], set_dps[0].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(1, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(values[1], set_dps[0].toDouble(), 0.0001); + + values[0] += 2; + values[1] += 2; + + ts += 60000; + } + } + + @Test (expected = IllegalDataException.class) + public void extraAggTagNoIntersection() throws Exception { + twoSeriesAggedEandExtraTagK(); + queryAB_Dstar(); + new IntersectionIterator("it", iterators, false, true); + } + + @Test (expected = IllegalDataException.class) + public void onlyOneResultSet() throws Exception { + threeSameENoB(); + queryAB_Dstar(); + new IntersectionIterator("it", iterators, false, true); + } + + @Test (expected = IllegalDataException.class) + public void oneAggedOneTaggedNoIntersection() throws Exception { + oneAggedTheOtherTagged(); + queryAB_AggAll(); + new IntersectionIterator("it", iterators, false, true); + } + + @Test + public void oneAggedOneTaggedUseQueryTagsWoutQueryTags() throws Exception { + oneAggedTheOtherTagged(); + queryAB_AggAll(); + + final IntersectionIterator it = new IntersectionIterator("it", iterators, true, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(1, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 2, 11 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(1, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(values[0], set_dps[0].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(1, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(values[1]++, set_dps[0].toDouble(), 0.0001); + + // the first set is agged + values[0] += 2; + + ts += 60000; + } + } + + @Test + public void singleSeries() throws Exception { + oneExtraSameE(); + queryA_DD(); + + final IntersectionIterator it = new IntersectionIterator("it", iterators, false, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(1, dps.size()); + assertEquals(1, it.getSeriesSize()); + + long ts = 1431561600000L; + double value = 1; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(1, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(value++, set_dps[0].toDouble(), 0.0001); + + ts += 60000; + } + } + + @Test (expected = IllegalDataException.class) + public void setAMissingE() throws Exception { + threeAMissingE(); + queryAB_Dstar(); + new IntersectionIterator("it", iterators, false, false); + } + + @Test + public void setAMissingEQueryTags() throws Exception { + threeAMissingE(); + queryAB_Dstar(); + + final IntersectionIterator it = new IntersectionIterator("it", iterators, true, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(3, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 1, 4, 7, 11, 14, 17 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(values[0]++, set_dps[0].toDouble(), 0.0001); + assertEquals(values[1]++, set_dps[1].toDouble(), 0.0001); + assertEquals(values[2]++, set_dps[2].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(values[3]++, set_dps[0].toDouble(), 0.0001); + assertEquals(values[4]++, set_dps[1].toDouble(), 0.0001); + assertEquals(values[5]++, set_dps[2].toDouble(), 0.0001); + + ts += 60000; + } + } + + @Test + public void noData() throws Exception { + setDataPointStorage(); + queryAB_Dstar(); + + final IntersectionIterator it = new IntersectionIterator("it", iterators, false, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertEquals(0, dps.size()); + assertFalse(it.hasNext()); + assertEquals(0, it.getSeriesSize()); + } + + @Test (expected = IllegalDataException.class) + public void nextException() throws Exception { + threeDisjointSameE(); + queryAB_Dstar(); + + final IntersectionIterator it = new IntersectionIterator("it", iterators, false, false); + it.next(); + it.next(); + it.next(); + it.next(); + } + + @Test + public void flattenTags() throws Exception { + final byte[] flat = IntersectionIterator.flattenTags( + false, false, tags, agg_tags, sub); + assertArrayEquals(MockBase.concatByteArrays(UID1, UID1, UID2, UID2), flat); + } + + @Test + public void flattenTagsWithAgg() throws Exception { + final byte[] flat = IntersectionIterator.flattenTags( + false, true, tags, agg_tags, sub); + assertArrayEquals( + MockBase.concatByteArrays(UID1, UID1, UID2, UID2, UID3), flat); + } + + @Test + public void flattenTagsQueryTags() throws Exception { + final byte[] flat = IntersectionIterator.flattenTags( + true, false, tags, agg_tags, sub); + assertArrayEquals(MockBase.concatByteArrays(UID1, UID1), flat); + } + + @Test + public void flattenTagsQueryTagsWithAgg() throws Exception { + final byte[] flat = IntersectionIterator.flattenTags( + true, true, tags, agg_tags, sub); + assertArrayEquals(MockBase.concatByteArrays(UID1, UID1, UID3), flat); + } + + @Test + public void flattenEmptyTags() throws Exception { + tags.clear(); + final byte[] flat = IntersectionIterator.flattenTags( + false, false, tags, agg_tags, sub); + assertArrayEquals(HBaseClient.EMPTY_ARRAY, flat); + } + + @Test + public void flattenEmptyTagsWithAggEmpty() throws Exception { + agg_tags.clear(); + final byte[] flat = IntersectionIterator.flattenTags( + false, true, tags, agg_tags, sub); + assertArrayEquals(MockBase.concatByteArrays(UID1, UID1, UID2, UID2), flat); + } + + // TODO - how will this play out if we choose a default agg of "none" and the + // user hasn't asked for any filtering? + @Test + public void flattenTagsQueryTagsEmpty() throws Exception { + query_tags.clear(); + final byte[] flat = IntersectionIterator.flattenTags( + true, false, tags, agg_tags, sub); + assertArrayEquals(HBaseClient.EMPTY_ARRAY, flat); + } + + @Test + public void flattenTagsQueryTagsEmptyWithAgg() throws Exception { + query_tags.clear(); + final byte[] flat = IntersectionIterator.flattenTags( + true, true, tags, agg_tags, sub); + assertArrayEquals(UID3, flat); + } + + @Test (expected = NullPointerException.class) + public void flattenTagsNullTags() throws Exception { + IntersectionIterator.flattenTags(false, false, null, agg_tags, sub); + } + + @Test + public void flattenTagsNullAggTagsNotRequested() throws Exception { + final byte[] flat = IntersectionIterator.flattenTags( + false, false, tags, null, sub); + assertArrayEquals(MockBase.concatByteArrays(UID1, UID1, UID2, UID2), flat); + } + + @Test (expected = NullPointerException.class) + public void flattenTagsNullAggTags() throws Exception { + IntersectionIterator.flattenTags(false, true, tags, null, sub); + } + + @Test + public void flattenTagsNullSubNotRequested() throws Exception { + final byte[] flat = IntersectionIterator.flattenTags( + false, false, tags, agg_tags, null); + assertArrayEquals(MockBase.concatByteArrays(UID1, UID1, UID2, UID2), flat); + } + + @Test (expected = NullPointerException.class) + public void flattenTagsNullSub() throws Exception { + IntersectionIterator.flattenTags(true, false, tags, agg_tags, null); + } + +} diff --git a/test/query/expression/TestMovingAverage.java b/test/query/expression/TestMovingAverage.java new file mode 100644 index 0000000000..3d2be90d99 --- /dev/null +++ b/test/query/expression/TestMovingAverage.java @@ -0,0 +1,518 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.expression; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.SeekableView; +import net.opentsdb.core.SeekableViewsForTest; +import net.opentsdb.core.TSQuery; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.stumbleupon.async.Deferred; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ TSQuery.class }) +public class TestMovingAverage { + private static long START_TIME = 1356998400000L; + private static int INTERVAL = 60000; + private static int NUM_POINTS = 5; + private static String METRIC = "sys.cpu"; + + private TSQuery data_query; + private SeekableView view; + private DataPoints dps; + private DataPoints[] group_bys; + private List<DataPoints[]> query_results; + private List<String> params; + private MovingAverage func; + + @Before + public void before() throws Exception { + view = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 1, 1); + data_query = mock(TSQuery.class); + when(data_query.startTime()).thenReturn(START_TIME); + when(data_query.endTime()).thenReturn(START_TIME + (INTERVAL * NUM_POINTS)); + + dps = PowerMockito.mock(DataPoints.class); + when(dps.iterator()).thenReturn(view); + when(dps.metricNameAsync()).thenReturn(Deferred.fromResult(METRIC)); + + group_bys = new DataPoints[] { dps }; + + query_results = new ArrayList<DataPoints[]>(1); + query_results.add(group_bys); + + params = new ArrayList<String>(1); + func = new MovingAverage(); + } + + @Test + public void evaluateWindow1dps() throws Exception { + params.add("1"); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(1, results.length); + assertEquals(METRIC, results[0].metricName()); + + long ts = START_TIME; + double v = 1; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + v += 1; + } + } + + @Test + public void evaluateWindow2dps() throws Exception { + params.add("2"); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(1, results.length); + assertEquals(METRIC, results[0].metricName()); + + long ts = START_TIME; + double v = 0; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + if (v < 1) { + v = 1.5; + } else { + v += 1; + } + } + } + + @Test + public void evaluateWindow5dps() throws Exception { + params.add("5"); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(1, results.length); + assertEquals(METRIC, results[0].metricName()); + + long ts = START_TIME; + double v = 0; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + if (ts == 1356998640000L) { + v = 3.0; + } + } + } + + @Test + public void evaluateWindow6dps() throws Exception { + params.add("6"); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(1, results.length); + assertEquals(METRIC, results[0].metricName()); + + long ts = START_TIME; + double v = 0; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + } + } + + @Test + public void evaluateWindow1min() throws Exception { + params.add("'1min'"); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(1, results.length); + assertEquals(METRIC, results[0].metricName()); + + long ts = START_TIME; + double v = 0; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + if (v < 1) { + v = 2; + } else { + v += 1; + } + } + } + + @Test + public void evaluateWindow2min() throws Exception { + params.add("'2min'"); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(1, results.length); + assertEquals(METRIC, results[0].metricName()); + + long ts = START_TIME; + double v = 0; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + if (ts == 1356998520000L) { + v = 2.5; + } else if (v > 0) { + v += 1; + } + } + } + + @Test + public void evaluateWindow3min() throws Exception { + params.add("'3min'"); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(1, results.length); + assertEquals(METRIC, results[0].metricName()); + + long ts = START_TIME; + double v = 0; + for (DataPoint dp : results[0]) { + System.out.println(dp.timestamp() + " : " + dp.doubleValue()); + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + if (ts == 1356998580000L) { + v = 3; + } else if (v > 0) { + v += 1; + } + } + } + + @Test + public void evaluateWindow4min() throws Exception { + params.add("'4min'"); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(1, results.length); + assertEquals(METRIC, results[0].metricName()); + + long ts = START_TIME; + double v = 0; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + if (ts == 1356998640000L) { + v = 3.5; + } + } + } + + @Test + public void evaluateWindow5min() throws Exception { + params.add("'5min'"); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(1, results.length); + assertEquals(METRIC, results[0].metricName()); + + long ts = START_TIME; + double v = 0; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + } + } + + @Test + public void evaluateGroupBy() throws Exception { + params.add("1"); + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals(METRIC, results[0].metricName()); + assertEquals("sys.mem", results[1].metricName()); + + long ts = START_TIME; + double v = 1; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + v += 1; + } + + ts = START_TIME; + v = 10; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + v += 1; + } + } + + @Test + public void evaluateSubQuery() throws Exception { + params.add("1"); + + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + DataPoints[] group_bys2 = new DataPoints[] { dps2 }; + query_results.add(group_bys2); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals(METRIC, results[0].metricName()); + assertEquals("sys.mem", results[1].metricName()); + + long ts = START_TIME; + double v = 1; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + v += 1; + } + + ts = START_TIME; + v = 10; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + v += 1; + } + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateNullQuery() throws Exception { + params.add("1"); + func.evaluate(null, query_results, params); + } + + @Test + public void evaluateNullResults() throws Exception { + params.add("1"); + final DataPoints[] results = func.evaluate(data_query, null, params); + assertEquals(0, results.length); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateNullParams() throws Exception { + func.evaluate(data_query, query_results, null); + } + + @Test + public void evaluateEmptyResults() throws Exception { + params.add("1"); + final DataPoints[] results = func.evaluate(data_query, + Collections.<DataPoints[]>emptyList(), params); + assertEquals(0, results.length); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateEmptyParams() throws Exception { + func.evaluate(data_query, query_results, params); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateWindowIsZeroDataPoints() throws Exception { + params.add("0"); + func.evaluate(data_query, query_results, params); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateWindowIsZeroTime() throws Exception { + params.add("'0sec'"); + func.evaluate(data_query, query_results, params); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateWindowTimedMissingQuotes() throws Exception { + params.add("60sec"); + func.evaluate(data_query, query_results, params); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateWindowNull() throws Exception { + params.add(null); + func.evaluate(data_query, query_results, params); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateWindowEmpty() throws Exception { + params.add(""); + func.evaluate(data_query, query_results, params); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateWindowUnknown() throws Exception { + params.add("somethingelse"); + func.evaluate(data_query, query_results, params); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateWindowNotFirstParam() throws Exception { + params.add("somethingelse"); + params.add("60"); + func.evaluate(data_query, query_results, params); + } + + @Test + public void writeStringField() throws Exception { + params.add("1"); + assertEquals("movingAverage(inner_expression)", + func.writeStringField(params, "inner_expression")); + assertEquals("movingAverage(null)", func.writeStringField(params, null)); + assertEquals("movingAverage()", func.writeStringField(params, "")); + assertEquals("movingAverage(inner_expression)", + func.writeStringField(null, "inner_expression")); + } + + @Test + public void parseParam() throws Exception { + // second + assertEquals(1000, func.parseParam("'1sec'")); + assertEquals(1000, func.parseParam("'1s'")); + assertEquals(5000, func.parseParam("'5sec'")); + assertEquals(5000, func.parseParam("'5s'")); + + // minute + assertEquals(60000, func.parseParam("'1min'")); + assertEquals(60000, func.parseParam("'1m'")); + assertEquals(300000, func.parseParam("'5min'")); + assertEquals(300000, func.parseParam("'5m'")); + + // hour + assertEquals(3600000, func.parseParam("'1hr")); + assertEquals(3600000, func.parseParam("'1h'")); + assertEquals(3600000, func.parseParam("'1hour'")); + assertEquals(18000000, func.parseParam("'5hr'")); + assertEquals(18000000, func.parseParam("'5h'")); + assertEquals(18000000, func.parseParam("'5hour'")); + + // day + assertEquals(86400000, func.parseParam("'1day'")); + assertEquals(86400000, func.parseParam("'1d'")); + assertEquals(432000000, func.parseParam("'5day'")); + assertEquals(432000000, func.parseParam("'5d'")); + + // TODO - fix it, closing with a 1 seems to work instead of a ' + assertEquals(1000, func.parseParam("'1sec1")); + + // missing quotes + try { + func.parseParam("'1sec"); + fail("Expected an IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + try { + func.parseParam("1sec'"); + fail("Expected an IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + try { + func.parseParam("1sec"); + fail("Expected an IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + + // no numbers or units + try { + func.parseParam("'sec'"); + fail("Expected an IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + try { + func.parseParam("'60'"); + fail("Expected an IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + + // null or empty or short + try { + func.parseParam(null); + fail("Expected an IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + try { + func.parseParam(""); + fail("Expected an IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + try { + func.parseParam("'"); + fail("Expected an IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + + // floating point + try { + func.parseParam("'1.5sec'"); + fail("Expected an IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + } +} diff --git a/test/query/expression/TestMultiplySeries.java b/test/query/expression/TestMultiplySeries.java new file mode 100644 index 0000000000..292f975f47 --- /dev/null +++ b/test/query/expression/TestMultiplySeries.java @@ -0,0 +1,196 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.expression; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.SeekableView; +import net.opentsdb.core.SeekableViewsForTest; +import net.opentsdb.core.TSQuery; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.sun.java_cup.internal.runtime.Scanner; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ TSQuery.class, Scanner.class }) +public class TestMultiplySeries extends BaseTimeSyncedIteratorTest { + + private static long START_TIME = 1356998400000L; + private static int INTERVAL = 60000; + private static int NUM_POINTS = 5; + + private TSQuery data_query; + private SeekableView view; + private DataPoints dps; + private DataPoints[] group_bys; + private List<DataPoints[]> query_results; + private List<String> params; + private MultiplySeries func; + + @Before + public void beforeLocal() throws Exception { + view = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 1, 1); + data_query = mock(TSQuery.class); + when(data_query.startTime()).thenReturn(START_TIME); + when(data_query.endTime()).thenReturn(START_TIME + (INTERVAL * NUM_POINTS)); + + dps = PowerMockito.mock(DataPoints.class); + when(dps.iterator()).thenReturn(view); + when(dps.metricName()).thenReturn(METRIC_STRING); + when(dps.metricUID()).thenReturn(new byte[] {0,0,1}); + + group_bys = new DataPoints[] { dps }; + + query_results = new ArrayList<DataPoints[]>(1); + query_results.add(group_bys); + + params = new ArrayList<String>(1); + func = new MultiplySeries(tsdb); + } + + @Test + public void multiplyOneSeriesEach() throws Exception { + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricName()).thenReturn("sys.mem"); + when(dps2.metricUID()).thenReturn(new byte[] {0,0,2}); + group_bys = new DataPoints[] { dps2 }; + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(1, results.length); + assertEquals(METRIC_STRING, results[0].metricName()); + final int[] vals = new int[] { 10, 22, 36, 52, 70 }; + int i = 0; + long ts = START_TIME; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertEquals(vals[i++], dp.toDouble(), 0.001); + ts += INTERVAL; + } + } + + @Test + public void multiplyMultipleSeriesEach() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + + query_results.clear(); + query_results.add(results.get("1").getValue()); + query_results.add(results.get("0").getValue()); + final DataPoints[] results = func.evaluate(data_query, + query_results, params); + + assertEquals(3, results.length); + + double[][] vals = new double[2][]; + vals[0] = new double[] { 11, 24, 39 }; + vals[1] = new double[] { 56, 75, 96 }; + for (int i = 0; i < results.length; i++) { + long ts = 1431561600000l; + final SeekableView it = results[i].iterator(); + int x = 0; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + if (i < 2) { + assertEquals(vals[i][x++], dp.toDouble(), 0.0001); + } else { + assertEquals(0, dp.toDouble(), 0.0001); + } + ts += INTERVAL; + } + } + } + + @Test (expected = IllegalArgumentException.class) + public void multiplyOneResultSet() throws Exception { + func.evaluate(data_query, query_results, params); + } + + @Test (expected = IllegalArgumentException.class) + public void multiplyTooManyResultSets() throws Exception { + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricName()).thenReturn("sys.mem"); + when(dps2.metricUID()).thenReturn(new byte[] {0,0,2}); + group_bys = new DataPoints[] { dps2 }; + // doesn't matter what they are + for (int i = 0; i < 100; i++) { + query_results.add(group_bys); + } + + func.evaluate(data_query, query_results, params); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateNullQuery() throws Exception { + params.add("1"); + func.evaluate(null, query_results, params); + } + + @Test + public void evaluateNullResults() throws Exception { + params.add("1"); + final DataPoints[] results = func.evaluate(data_query, null, params); + assertEquals(0, results.length); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateNullParams() throws Exception { + func.evaluate(data_query, query_results, null); + } + + @Test + public void evaluateEmptyResults() throws Exception { + params.add("1"); + final DataPoints[] results = func.evaluate(data_query, + Collections.<DataPoints[]>emptyList(), params); + assertEquals(0, results.length); + } + + @Test + public void writeStringField() throws Exception { + params.add("1"); + assertEquals("multiplySeries(inner_expression)", + func.writeStringField(params, "inner_expression")); + assertEquals("multiplySeries(null)", func.writeStringField(params, null)); + assertEquals("multiplySeries()", func.writeStringField(params, "")); + assertEquals("multiplySeries(inner_expression)", + func.writeStringField(null, "inner_expression")); + } +} diff --git a/test/query/expression/TestNumericFillPolicy.java b/test/query/expression/TestNumericFillPolicy.java new file mode 100644 index 0000000000..2b83d68b5c --- /dev/null +++ b/test/query/expression/TestNumericFillPolicy.java @@ -0,0 +1,304 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.expression; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import net.opentsdb.core.FillPolicy; +import net.opentsdb.utils.JSON; + +import org.junit.Test; + +public class TestNumericFillPolicy { + + @Test + public void builder() throws Exception { + NumericFillPolicy nfp = NumericFillPolicy.Builder() + .setPolicy(FillPolicy.NOT_A_NUMBER).build(); + assertEquals(FillPolicy.NOT_A_NUMBER, nfp.getPolicy()); + assertTrue(Double.isNaN((Double)nfp.getValue())); + + nfp = NumericFillPolicy.Builder() + .setPolicy(null).build(); + assertEquals(FillPolicy.ZERO, nfp.getPolicy()); + assertEquals(0, nfp.getValue(), 0.0001); + } + + @Test + public void policyCtor() throws Exception { + NumericFillPolicy nfp = new NumericFillPolicy(FillPolicy.NONE); + assertTrue(Double.isNaN((Double)nfp.getValue())); + assertEquals(FillPolicy.NONE, nfp.getPolicy()); + + nfp = new NumericFillPolicy(FillPolicy.NOT_A_NUMBER); + assertTrue(Double.isNaN((Double)nfp.getValue())); + assertEquals(FillPolicy.NOT_A_NUMBER, nfp.getPolicy()); + + nfp = new NumericFillPolicy(FillPolicy.NULL); + assertTrue(Double.isNaN((Double)nfp.getValue())); + assertEquals(FillPolicy.NULL, nfp.getPolicy()); + + nfp = new NumericFillPolicy(FillPolicy.ZERO); + assertEquals(0, nfp.getValue(), 0.0001); + assertEquals(FillPolicy.ZERO, nfp.getPolicy()); + + nfp = new NumericFillPolicy(null); + assertEquals(0, nfp.getValue(), 0.0001); + assertEquals(FillPolicy.ZERO, nfp.getPolicy()); + + nfp = new NumericFillPolicy(FillPolicy.SCALAR); + assertEquals(0, nfp.getValue(), 0.0001); + assertEquals(FillPolicy.SCALAR, nfp.getPolicy()); + + } + + @Test + public void policyAndValueCtor() throws Exception { + NumericFillPolicy nfp = new NumericFillPolicy(FillPolicy.NONE, Double.NaN); + assertTrue(Double.isNaN((Double)nfp.getValue())); + assertEquals(FillPolicy.NONE, nfp.getPolicy()); + + nfp = new NumericFillPolicy(FillPolicy.NOT_A_NUMBER, Double.NaN); + assertTrue(Double.isNaN((Double)nfp.getValue())); + assertEquals(FillPolicy.NOT_A_NUMBER, nfp.getPolicy()); + + nfp = new NumericFillPolicy(FillPolicy.NULL, 0); + assertTrue(Double.isNaN((Double)nfp.getValue())); + assertEquals(FillPolicy.NULL, nfp.getPolicy()); + + nfp = new NumericFillPolicy(FillPolicy.ZERO, 0); + assertEquals(0, nfp.getValue(), 0.0001); + assertEquals(FillPolicy.ZERO, nfp.getPolicy()); + + nfp = new NumericFillPolicy(null, 0); + assertEquals(0, nfp.getValue(), 0.0001); + assertEquals(FillPolicy.ZERO, nfp.getPolicy()); + + nfp = new NumericFillPolicy(FillPolicy.SCALAR, 42); + assertEquals(42, nfp.getValue(), 0.0001); + assertEquals(FillPolicy.SCALAR, nfp.getPolicy()); + + nfp = new NumericFillPolicy(FillPolicy.SCALAR, 0); + assertEquals(0, nfp.getValue(), 0.0001); + assertEquals(FillPolicy.SCALAR, nfp.getPolicy()); + + nfp = new NumericFillPolicy(FillPolicy.SCALAR, Double.NaN); + assertTrue(Double.isNaN((Double)nfp.getValue())); + assertEquals(FillPolicy.SCALAR, nfp.getPolicy()); + + nfp = new NumericFillPolicy(FillPolicy.SCALAR, 42.5); + assertEquals(42.5, (Double)nfp.getValue(), 0.0001); + assertEquals(FillPolicy.SCALAR, nfp.getPolicy()); + + // defaults from value + nfp = new NumericFillPolicy(null, Double.NaN); + assertTrue(Double.isNaN((Double)nfp.getValue())); + assertEquals(FillPolicy.NOT_A_NUMBER, nfp.getPolicy()); + + nfp = new NumericFillPolicy(null, 42); + assertEquals(42, nfp.getValue(), 0.0001); + assertEquals(FillPolicy.SCALAR, nfp.getPolicy()); + + nfp = new NumericFillPolicy(null, 42.5); + assertEquals(42.5, (Double)nfp.getValue(), 0.0001); + assertEquals(FillPolicy.SCALAR, nfp.getPolicy()); + + nfp = new NumericFillPolicy(null, -42.5); + assertEquals(-42.5, (Double)nfp.getValue(), 0.0001); + assertEquals(FillPolicy.SCALAR, nfp.getPolicy()); + + nfp = new NumericFillPolicy(null, Double.NaN); + assertTrue(Double.isNaN((Double)nfp.getValue())); + assertEquals(FillPolicy.NOT_A_NUMBER, nfp.getPolicy()); + + nfp = new NumericFillPolicy(null, 0); + assertEquals(0, nfp.getValue(), 0.0001); + assertEquals(FillPolicy.ZERO, nfp.getPolicy()); + + nfp = new NumericFillPolicy(null, 0.0); + assertEquals(0.0, (Double)nfp.getValue(), 0.0001); + assertEquals(FillPolicy.ZERO, nfp.getPolicy()); + + nfp = new NumericFillPolicy(null, -0.0); + assertEquals(-0.0, (Double)nfp.getValue(), 0.0001); + assertEquals(FillPolicy.ZERO, nfp.getPolicy()); + + nfp = new NumericFillPolicy(null, -0); + assertEquals(-0, nfp.getValue(), 0.0001); + assertEquals(FillPolicy.ZERO, nfp.getPolicy()); + + nfp = new NumericFillPolicy(FillPolicy.NOT_A_NUMBER, 0); + assertTrue(Double.isNaN((Double)nfp.getValue())); + assertEquals(FillPolicy.NOT_A_NUMBER, nfp.getPolicy()); + + nfp = new NumericFillPolicy(FillPolicy.NULL, Double.NaN); + assertTrue(Double.isNaN((Double)nfp.getValue())); + assertEquals(FillPolicy.NULL, nfp.getPolicy()); + + // inappropriate combos + try { + nfp = new NumericFillPolicy(FillPolicy.ZERO, 42); + fail("expected an IllegalArgumentException"); + } catch (IllegalArgumentException iae) { } + + try { + nfp = new NumericFillPolicy(FillPolicy.NONE, 42); + fail("expected an IllegalArgumentException"); + } catch (IllegalArgumentException iae) { } + + try { + nfp = new NumericFillPolicy(FillPolicy.NULL, 42); + fail("expected an IllegalArgumentException"); + } catch (IllegalArgumentException iae) { } + + try { + nfp = new NumericFillPolicy(FillPolicy.NOT_A_NUMBER, 42); + fail("expected an IllegalArgumentException"); + } catch (IllegalArgumentException iae) { } + + } + + @Test + public void serdes() throws Exception { + + NumericFillPolicy ser_nfp = new NumericFillPolicy(FillPolicy.NONE); + String json = JSON.serializeToString(ser_nfp); + assertTrue(json.contains("\"policy\":\"none\"")); + assertTrue(json.contains("\"value\":\"NaN\"")); + NumericFillPolicy des_nfp = JSON.parseToObject(json, NumericFillPolicy.class); + assertTrue(des_nfp != ser_nfp); + assertTrue(des_nfp.equals(ser_nfp)); + + ser_nfp = new NumericFillPolicy(FillPolicy.ZERO); + json = JSON.serializeToString(ser_nfp); + assertTrue(json.contains("\"policy\":\"zero\"")); + assertTrue(json.contains("\"value\":0")); + des_nfp = JSON.parseToObject(json, NumericFillPolicy.class); + assertTrue(des_nfp != ser_nfp); + assertTrue(des_nfp.equals(ser_nfp)); + + ser_nfp = new NumericFillPolicy(FillPolicy.NOT_A_NUMBER); + json = JSON.serializeToString(ser_nfp); + assertTrue(json.contains("\"policy\":\"nan\"")); + assertTrue(json.contains("\"value\":\"NaN\"")); + des_nfp = JSON.parseToObject(json, NumericFillPolicy.class); + assertTrue(des_nfp != ser_nfp); + assertTrue(des_nfp.equals(ser_nfp)); + + ser_nfp = new NumericFillPolicy(FillPolicy.NULL); + json = JSON.serializeToString(ser_nfp); + assertTrue(json.contains("\"policy\":\"null\"")); + assertTrue(json.contains("\"value\":\"NaN\"")); + des_nfp = JSON.parseToObject(json, NumericFillPolicy.class); + assertTrue(des_nfp != ser_nfp); + assertTrue(des_nfp.equals(ser_nfp)); + + ser_nfp = new NumericFillPolicy(FillPolicy.SCALAR, 42); + json = JSON.serializeToString(ser_nfp); + assertTrue(json.contains("\"policy\":\"scalar\"")); + assertTrue(json.contains("\"value\":42")); + des_nfp = JSON.parseToObject(json, NumericFillPolicy.class); + assertTrue(des_nfp != ser_nfp); + assertTrue(des_nfp.equals(ser_nfp)); + + ser_nfp = new NumericFillPolicy(FillPolicy.SCALAR, 42.5); + json = JSON.serializeToString(ser_nfp); + assertTrue(json.contains("\"policy\":\"scalar\"")); + assertTrue(json.contains("\"value\":42.5")); + des_nfp = JSON.parseToObject(json, NumericFillPolicy.class); + assertTrue(des_nfp != ser_nfp); + assertTrue(des_nfp.equals(ser_nfp)); + + ser_nfp = new NumericFillPolicy(FillPolicy.SCALAR, -42.5); + json = JSON.serializeToString(ser_nfp); + assertTrue(json.contains("\"policy\":\"scalar\"")); + assertTrue(json.contains("\"value\":-42.5")); + des_nfp = JSON.parseToObject(json, NumericFillPolicy.class); + assertTrue(des_nfp != ser_nfp); + assertTrue(des_nfp.equals(ser_nfp)); + + json = "{\"policy\":\"zero\"}"; + des_nfp = JSON.parseToObject(json, NumericFillPolicy.class); + assertEquals(FillPolicy.ZERO, des_nfp.getPolicy()); + assertEquals(0, des_nfp.getValue(), 0.0001); + + json = "{\"policy\":\"nan\"}"; + des_nfp = JSON.parseToObject(json, NumericFillPolicy.class); + assertEquals(FillPolicy.NOT_A_NUMBER, des_nfp.getPolicy()); + assertTrue(Double.isNaN((Double)des_nfp.getValue())); + + json = "{\"policy\":\"scalar\"}"; + des_nfp = JSON.parseToObject(json, NumericFillPolicy.class); + assertEquals(FillPolicy.SCALAR, des_nfp.getPolicy()); + assertEquals(0, des_nfp.getValue(), 0.0001); + + json = "{\"policy\":\"none\"}"; + des_nfp = JSON.parseToObject(json, NumericFillPolicy.class); + assertEquals(FillPolicy.NONE, des_nfp.getPolicy()); + assertTrue(Double.isNaN((Double)des_nfp.getValue())); + + json = "{\"policy\":\"null\"}"; + des_nfp = JSON.parseToObject(json, NumericFillPolicy.class); + assertEquals(FillPolicy.NULL, des_nfp.getPolicy()); + assertTrue(Double.isNaN((Double)des_nfp.getValue())); + + json = "{\"policy\":\"scalar\",\"value\":42}"; + des_nfp = JSON.parseToObject(json, NumericFillPolicy.class); + assertEquals(FillPolicy.SCALAR, des_nfp.getPolicy()); + assertEquals(42, des_nfp.getValue(), 0.0001); + + json = "{\"policy\":\"scalar\",\"value\":\"42\"}"; + des_nfp = JSON.parseToObject(json, NumericFillPolicy.class); + assertEquals(FillPolicy.SCALAR, des_nfp.getPolicy()); + assertEquals(42, des_nfp.getValue(), 0.0001); + + json = "{\"policy\":\"scalar\",\"value\":42.5}"; + des_nfp = JSON.parseToObject(json, NumericFillPolicy.class); + assertEquals(FillPolicy.SCALAR, des_nfp.getPolicy()); + assertEquals(42.5, (Double)des_nfp.getValue(), 0.0001); + + json = "{\"policy\":\"nan\",\"value\":NaN}"; + des_nfp = JSON.parseToObject(json, NumericFillPolicy.class); + assertEquals(FillPolicy.NOT_A_NUMBER, des_nfp.getPolicy()); + assertTrue(Double.isNaN((Double)des_nfp.getValue())); + + json = "{\"policy\":\"scalar\",\"value\":0}"; + des_nfp = JSON.parseToObject(json, NumericFillPolicy.class); + assertEquals(FillPolicy.SCALAR, des_nfp.getPolicy()); + assertEquals(0, des_nfp.getValue(), 0.0001); + + json = "{\"policy\":\"scalar\",\"value\":0.0}"; + des_nfp = JSON.parseToObject(json, NumericFillPolicy.class); + assertEquals(FillPolicy.SCALAR, des_nfp.getPolicy()); + assertEquals(0.0, (Double)des_nfp.getValue(), 0.0001); + + try { + json = "{\"policy\":\"unknown\"}"; + des_nfp = JSON.parseToObject(json, NumericFillPolicy.class); + fail("Expected a IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + + try { + json = "{\"policy\":\"scalar\",value\":\"foo\"}"; + des_nfp = JSON.parseToObject(json, NumericFillPolicy.class); + fail("Expected a IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + + try { + json = "{\"policy\":\"badjson"; + des_nfp = JSON.parseToObject(json, NumericFillPolicy.class); + fail("Expected a IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + } +} diff --git a/test/query/expression/TestPostAggregatedDataPoints.java b/test/query/expression/TestPostAggregatedDataPoints.java new file mode 100644 index 0000000000..aa940d9072 --- /dev/null +++ b/test/query/expression/TestPostAggregatedDataPoints.java @@ -0,0 +1,220 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.expression; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; + +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.MutableDataPoint; +import net.opentsdb.core.SeekableView; +import net.opentsdb.meta.Annotation; + +import org.hbase.async.Bytes.ByteMap; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.stumbleupon.async.Deferred; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ Annotation.class }) +public class TestPostAggregatedDataPoints { + private static int NUM_POINTS = 5; + private static String METRIC_NAME = "sys.cpu"; + private static long BASE_TIME = 1356998400000L; + private static int TIME_INTERVAL = 60000; + + private DataPoints base_data_points; + private DataPoint[] points; + private Map<String, String> tags; + private List<String> agg_tags; + private List<String> tsuids; + private List<Annotation> annotations; + private ByteMap<byte[]> tag_uids; + + @Before + public void before() throws Exception { + base_data_points = PowerMockito.mock(DataPoints.class); + points = new MutableDataPoint[NUM_POINTS]; + + long ts = BASE_TIME; + for (int i = 0; i < NUM_POINTS; i++) { + MutableDataPoint mdp = new MutableDataPoint(); + mdp.reset(ts, i); + points[i] = mdp; + ts += TIME_INTERVAL; + } + + tags = new HashMap<String, String>(1); + tags.put("colo", "lga"); + agg_tags = new ArrayList<String>(1); + agg_tags.add("host"); + tsuids = new ArrayList<String>(1); + tsuids.add("0101010202"); // just 1 byte UIDs for kicks + annotations = new ArrayList<Annotation>(1); + annotations.add(PowerMockito.mock(Annotation.class)); + tag_uids = new ByteMap<byte[]>(); + tag_uids.put(new byte[] { 1 }, new byte[] { 1 }); + + when(base_data_points.metricName()).thenReturn(METRIC_NAME); + when(base_data_points.metricNameAsync()).thenReturn( + Deferred.fromResult(METRIC_NAME)); + when(base_data_points.getTags()).thenReturn(tags); + when(base_data_points.getTagsAsync()).thenReturn(Deferred.fromResult(tags)); + when(base_data_points.getAggregatedTags()).thenReturn(agg_tags); + when(base_data_points.getAggregatedTagsAsync()).thenReturn( + Deferred.fromResult(agg_tags)); + when(base_data_points.getTSUIDs()).thenReturn(tsuids); + when(base_data_points.getAnnotations()).thenReturn(annotations); + when(base_data_points.getTagUids()).thenReturn(tag_uids); + when(base_data_points.getQueryIndex()).thenReturn(42); + } + + @Test + public void ctorDefaults() throws Exception { + final PostAggregatedDataPoints dps = new PostAggregatedDataPoints( + base_data_points, points); + assertEquals(METRIC_NAME, dps.metricName()); + assertEquals(METRIC_NAME, dps.metricNameAsync().join()); + assertSame(tags, dps.getTags()); + assertSame(tags, dps.getTagsAsync().join()); + assertSame(agg_tags, dps.getAggregatedTags()); + assertSame(agg_tags, dps.getAggregatedTagsAsync().join()); + assertSame(tsuids, dps.getTSUIDs()); + assertSame(annotations, dps.getAnnotations()); + assertSame(tag_uids, dps.getTagUids()); + assertEquals(42, dps.getQueryIndex()); + assertEquals(5, dps.size()); + assertEquals(5, dps.aggregatedSize()); + + // values + final SeekableView iterator = dps.iterator(); + int values = 0; + long value = 0; + long ts = BASE_TIME; + while(iterator.hasNext()) { + final DataPoint dp = iterator.next(); + assertEquals(value++, dp.longValue()); + assertEquals(ts, dp.timestamp()); + ts += TIME_INTERVAL; + values++; + } + assertEquals(5, values); + assertFalse(iterator.hasNext()); + try { + iterator.next(); + fail("Expected a NoSuchElementException"); + } catch (NoSuchElementException e) { } + + assertEquals(BASE_TIME + (4 * TIME_INTERVAL), dps.timestamp(4)); + assertEquals(4, dps.longValue(4)); + assertTrue(dps.isInteger(4)); + try { + assertEquals(4, dps.doubleValue(4), 0.00); + fail("Expected a ClassCastException"); + } catch (ClassCastException e) { } + + try { + dps.timestamp(5); + fail("Expected a ArrayIndexOutOfBoundsException"); + } catch (ArrayIndexOutOfBoundsException e) { } + try { + dps.isInteger(5); + fail("Expected a ArrayIndexOutOfBoundsException"); + } catch (ArrayIndexOutOfBoundsException e) { } + try { + dps.longValue(5); + fail("Expected a ArrayIndexOutOfBoundsException"); + } catch (ArrayIndexOutOfBoundsException e) { } + try { + dps.doubleValue(5); + fail("Expected a ArrayIndexOutOfBoundsException"); + } catch (ArrayIndexOutOfBoundsException e) { } + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNullBase() throws Exception { + new PostAggregatedDataPoints(null, points); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNullPoints() throws Exception { + new PostAggregatedDataPoints(base_data_points, null); + } + + @Test + public void alias() throws Exception { + final PostAggregatedDataPoints dps = new PostAggregatedDataPoints( + base_data_points, points); + final String alias = "ein"; + dps.setAlias(alias); + assertEquals(alias, dps.metricName()); + assertEquals(alias, dps.metricNameAsync().join()); + assertTrue(dps.getTags().isEmpty()); + assertTrue(dps.getTagsAsync().join().isEmpty()); + assertTrue(dps.getAggregatedTags().isEmpty()); + assertTrue(dps.getAggregatedTagsAsync().join().isEmpty()); + assertSame(tsuids, dps.getTSUIDs()); + assertSame(annotations, dps.getAnnotations()); + assertSame(tag_uids, dps.getTagUids()); + assertEquals(42, dps.getQueryIndex()); + assertEquals(5, dps.size()); + assertEquals(5, dps.aggregatedSize()); + } + + @Test + public void emptyPoints() throws Exception { + points = new MutableDataPoint[0]; + final PostAggregatedDataPoints dps = new PostAggregatedDataPoints( + base_data_points, points); + assertEquals(METRIC_NAME, dps.metricName()); + assertEquals(METRIC_NAME, dps.metricNameAsync().join()); + assertSame(tags, dps.getTags()); + assertSame(tags, dps.getTagsAsync().join()); + assertSame(agg_tags, dps.getAggregatedTags()); + assertSame(agg_tags, dps.getAggregatedTagsAsync().join()); + assertSame(tsuids, dps.getTSUIDs()); + assertSame(annotations, dps.getAnnotations()); + assertSame(tag_uids, dps.getTagUids()); + assertEquals(42, dps.getQueryIndex()); + assertEquals(0, dps.size()); + assertEquals(0, dps.aggregatedSize()); + + // values + final SeekableView iterator = dps.iterator(); + assertFalse(iterator.hasNext()); + try { + iterator.next(); + fail("Expected a NoSuchElementException"); + } catch (NoSuchElementException e) { } + } +} diff --git a/test/query/expression/TestScale.java b/test/query/expression/TestScale.java new file mode 100644 index 0000000000..d34469d7c0 --- /dev/null +++ b/test/query/expression/TestScale.java @@ -0,0 +1,404 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.expression; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.SeekableView; +import net.opentsdb.core.SeekableViewsForTest; +import net.opentsdb.core.TSQuery; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.stumbleupon.async.Deferred; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ TSQuery.class }) +public class TestScale { + + private static long START_TIME = 1356998400000L; + private static int INTERVAL = 60000; + private static int NUM_POINTS = 5; + private static String METRIC = "sys.cpu"; + + private TSQuery data_query; + private SeekableView view; + private DataPoints dps; + private DataPoints[] group_bys; + private List<DataPoints[]> query_results; + private List<String> params; + private Scale func; + + @Before + public void before() throws Exception { + view = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 1, 1); + data_query = mock(TSQuery.class); + when(data_query.startTime()).thenReturn(START_TIME); + when(data_query.endTime()).thenReturn(START_TIME + (INTERVAL * NUM_POINTS)); + + dps = PowerMockito.mock(DataPoints.class); + when(dps.iterator()).thenReturn(view); + when(dps.metricNameAsync()).thenReturn(Deferred.fromResult(METRIC)); + + group_bys = new DataPoints[] { dps }; + + query_results = new ArrayList<DataPoints[]>(1); + query_results.add(group_bys); + + params = new ArrayList<String>(1); + func = new Scale(); + } + + @Test + public void evaluateFactor1GroupByLong() throws Exception { + params.add("1"); + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals(METRIC, results[0].metricName()); + assertEquals("sys.mem", results[1].metricName()); + + long ts = START_TIME; + long v = 1; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + ts = START_TIME; + v = 10; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + } + + @Test + public void evaluateFactor1GroupByDouble() throws Exception { + params.add("1"); + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, false, 10, 1.5); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals(METRIC, results[0].metricName()); + assertEquals("sys.mem", results[1].metricName()); + + long ts = START_TIME; + double v = 1; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals((long)v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + ts = START_TIME; + v = 10; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + v += 1.5; + } + } + + @Test + public void evaluateFactor1point5GroupBy() throws Exception { + params.add("1.5"); + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals(METRIC, results[0].metricName()); + assertEquals("sys.mem", results[1].metricName()); + + long ts = START_TIME; + double v = 1.5; + for (DataPoint dp : results[0]) { + System.out.println(dp.timestamp() + " : " + dp.toDouble()); + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + v += 1.5; + } + ts = START_TIME; + v = 15; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(v, dp.doubleValue(), 0.001); + ts += INTERVAL; + v += 1.5; + } + } + + @Test + public void evaluateFactor1024GroupBy() throws Exception { + params.add("1024"); + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals(METRIC, results[0].metricName()); + assertEquals("sys.mem", results[1].metricName()); + + long ts = START_TIME; + long v = 1024; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1024; + } + ts = START_TIME; + v = 10240; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1024; + } + } + + @Test + public void evaluateFactor1SubQuerySeries() throws Exception { + params.add("1"); + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals(METRIC, results[0].metricName()); + assertEquals("sys.mem", results[1].metricName()); + + long ts = START_TIME; + long v = 1; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + ts = START_TIME; + v = 10; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v += 1; + } + } + + @Test + public void evaluateFactor0GroupByLong() throws Exception { + params.add("0"); + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals(METRIC, results[0].metricName()); + assertEquals("sys.mem", results[1].metricName()); + + long ts = START_TIME; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(0, dp.longValue()); + ts += INTERVAL; + } + ts = START_TIME; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(0, dp.longValue()); + ts += INTERVAL; + } + } + + @Test + public void evaluateFactorNegative1GroupByLong() throws Exception { + params.add("-1"); + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricNameAsync()).thenReturn(Deferred.fromResult("sys.mem")); + group_bys = new DataPoints[] { dps, dps2 }; + query_results.clear(); + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(2, results.length); + assertEquals(METRIC, results[0].metricName()); + assertEquals("sys.mem", results[1].metricName()); + + long ts = START_TIME; + long v = -1; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v -= 1; + } + ts = START_TIME; + v = -10; + for (DataPoint dp : results[1]) { + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(v, dp.longValue()); + ts += INTERVAL; + v -= 1; + } + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateNullQuery() throws Exception { + params.add("1"); + func.evaluate(null, query_results, params); + } + + @Test + public void evaluateNullResults() throws Exception { + params.add("1"); + final DataPoints[] results = func.evaluate(data_query, null, params); + assertEquals(0, results.length); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateNullParams() throws Exception { + func.evaluate(data_query, query_results, null); + } + + @Test + public void evaluateEmptyResults() throws Exception { + params.add("1"); + final DataPoints[] results = func.evaluate(data_query, + Collections.<DataPoints[]>emptyList(), params); + assertEquals(0, results.length); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateEmptyParams() throws Exception { + func.evaluate(data_query, query_results, params); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateScaleNull() throws Exception { + params.add(null); + func.evaluate(data_query, query_results, params); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateScaleEmpty() throws Exception { + params.add(""); + func.evaluate(data_query, query_results, params); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateScaleNotaNumber() throws Exception { + params.add("not a number"); + func.evaluate(data_query, query_results, params); + } + + @Test + public void writeStringField() throws Exception { + params.add("1"); + assertEquals("scale(inner_expression)", + func.writeStringField(params, "inner_expression")); + assertEquals("scale(null)", func.writeStringField(params, null)); + assertEquals("scale()", func.writeStringField(params, "")); + assertEquals("scale(inner_expression)", + func.writeStringField(null, "inner_expression")); + } +} diff --git a/test/query/expression/TestSumSeries.java b/test/query/expression/TestSumSeries.java new file mode 100644 index 0000000000..77a22b56a3 --- /dev/null +++ b/test/query/expression/TestSumSeries.java @@ -0,0 +1,195 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.expression; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.SeekableView; +import net.opentsdb.core.SeekableViewsForTest; +import net.opentsdb.core.TSQuery; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.sun.java_cup.internal.runtime.Scanner; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ TSQuery.class, Scanner.class }) +public class TestSumSeries extends BaseTimeSyncedIteratorTest { + + private static long START_TIME = 1356998400000L; + private static int INTERVAL = 60000; + private static int NUM_POINTS = 5; + + private TSQuery data_query; + private SeekableView view; + private DataPoints dps; + private DataPoints[] group_bys; + private List<DataPoints[]> query_results; + private List<String> params; + private SumSeries func; + + @Before + public void beforeLocal() throws Exception { + view = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 1, 1); + data_query = mock(TSQuery.class); + when(data_query.startTime()).thenReturn(START_TIME); + when(data_query.endTime()).thenReturn(START_TIME + (INTERVAL * NUM_POINTS)); + + dps = PowerMockito.mock(DataPoints.class); + when(dps.iterator()).thenReturn(view); + when(dps.metricName()).thenReturn(METRIC_STRING); + when(dps.metricUID()).thenReturn(new byte[] {0,0,1}); + + group_bys = new DataPoints[] { dps }; + + query_results = new ArrayList<DataPoints[]>(1); + query_results.add(group_bys); + + params = new ArrayList<String>(1); + func = new SumSeries(tsdb); + } + + @Test + public void sumOneSeriesEach() throws Exception { + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricName()).thenReturn("sys.mem"); + when(dps2.metricUID()).thenReturn(new byte[] {0,0,2}); + group_bys = new DataPoints[] { dps2 }; + query_results.add(group_bys); + + final DataPoints[] results = func.evaluate(data_query, query_results, params); + + assertEquals(1, results.length); + assertEquals(METRIC_STRING, results[0].metricName()); + + long ts = START_TIME; + double v = 11; + for (DataPoint dp : results[0]) { + assertEquals(ts, dp.timestamp()); + assertEquals(v, dp.toDouble(), 0.001); + v += 2; + ts += INTERVAL; + } + } + + @Test + public void sumMultipleSeriesEach() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + + query_results.clear(); + query_results.add(results.get("1").getValue()); + query_results.add(results.get("0").getValue()); + final DataPoints[] results = func.evaluate(data_query, + query_results, params); + + assertEquals(3, results.length); + + final int vals[] = new int[] { 12, 18, 17 }; + for (int i = 0; i < results.length; i++) { + long ts = 1431561600000l; + final SeekableView it = results[i].iterator(); + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(vals[i], dp.toDouble(), 0.0001); + if (i < 2) { + vals[i] += 2; + } else { + vals[i]++; + } + ts += INTERVAL; + } + } + } + + @Test (expected = IllegalArgumentException.class) + public void sumOneResultSet() throws Exception { + func.evaluate(data_query, query_results, params); + } + + @Test (expected = IllegalArgumentException.class) + public void sumTooManyResultSets() throws Exception { + SeekableView view2 = SeekableViewsForTest.generator(START_TIME, INTERVAL, + NUM_POINTS, true, 10, 1); + DataPoints dps2 = PowerMockito.mock(DataPoints.class); + when(dps2.iterator()).thenReturn(view2); + when(dps2.metricName()).thenReturn("sys.mem"); + when(dps2.metricUID()).thenReturn(new byte[] {0,0,2}); + group_bys = new DataPoints[] { dps2 }; + // doesn't matter what they are + for (int i = 0; i < 100; i++) { + query_results.add(group_bys); + } + + func.evaluate(data_query, query_results, params); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateNullQuery() throws Exception { + params.add("1"); + func.evaluate(null, query_results, params); + } + + @Test + public void evaluateNullResults() throws Exception { + params.add("1"); + final DataPoints[] results = func.evaluate(data_query, null, params); + assertEquals(0, results.length); + } + + @Test (expected = IllegalArgumentException.class) + public void evaluateNullParams() throws Exception { + func.evaluate(data_query, query_results, null); + } + + @Test + public void evaluateEmptyResults() throws Exception { + params.add("1"); + final DataPoints[] results = func.evaluate(data_query, + Collections.<DataPoints[]>emptyList(), params); + assertEquals(0, results.length); + } + + @Test + public void writeStringField() throws Exception { + params.add("1"); + assertEquals("sumSeries(inner_expression)", + func.writeStringField(params, "inner_expression")); + assertEquals("sumSeries(null)", func.writeStringField(params, null)); + assertEquals("sumSeries()", func.writeStringField(params, "")); + assertEquals("sumSeries(inner_expression)", + func.writeStringField(null, "inner_expression")); + } +} diff --git a/test/query/expression/TestTimeShift.java b/test/query/expression/TestTimeShift.java new file mode 100644 index 0000000000..c6546d34f8 --- /dev/null +++ b/test/query/expression/TestTimeShift.java @@ -0,0 +1,81 @@ +package net.opentsdb.query.expression; + +import com.stumbleupon.async.Deferred; +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.MutableDataPoint; +import net.opentsdb.core.SeekableView; +import net.opentsdb.meta.Annotation; +import org.hbase.async.Bytes; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.*; + +public class TestTimeShift { + private TimeShift timeshift; + private static final long BASE_TIME = 1356998400000L; + private static final DataPoint[] DATA_POINTS = new DataPoint[] { + // timestamp = 1,356,998,400,000 ms + MutableDataPoint.ofLongValue(BASE_TIME, 40), + // timestamp = 1,357,000,400,000 ms + MutableDataPoint.ofLongValue(BASE_TIME + 2000000, 50), + // timestamp = 1,357,002,000,000 ms + MutableDataPoint.ofLongValue(BASE_TIME + 3600000, 40), + // timestamp = 1,357,002,005,000 ms + MutableDataPoint.ofLongValue(BASE_TIME + 3605000, 50), + // timestamp = 1,357,005,600,000 ms + MutableDataPoint.ofLongValue(BASE_TIME + 7200000, 40), + // timestamp = 1,357,007,600,000 ms + MutableDataPoint.ofLongValue(BASE_TIME + 9200000, 50) + }; + + @Before + public void setUp() throws Exception { + this.timeshift = new TimeShift(); + } + + @After + public void tearDown() throws Exception { + } + + @Test + public void parseParam() throws Exception { + assertEquals(TimeUnit.DAYS.toMillis(7), this.timeshift.parseParam("+1week ")); + assertEquals(TimeUnit.DAYS.toMillis(1), this.timeshift.parseParam("+1days ")); + assertEquals(TimeUnit.HOURS.toMillis(1), this.timeshift.parseParam("+1hr ")); + assertEquals(TimeUnit.MINUTES.toMillis(1), this.timeshift.parseParam("+1min ")); + assertEquals(TimeUnit.SECONDS.toMillis(1), this.timeshift.parseParam("+1sec ")); + assertEquals(TimeUnit.DAYS.toMillis(7), this.timeshift.parseParam("+1 week ")); + assertEquals(TimeUnit.DAYS.toMillis(1), this.timeshift.parseParam("+1 days ")); + assertEquals(TimeUnit.HOURS.toMillis(1), this.timeshift.parseParam("+1 hr ")); + assertEquals(TimeUnit.MINUTES.toMillis(1), this.timeshift.parseParam("+1 min ")); + assertEquals(TimeUnit.SECONDS.toMillis(1), this.timeshift.parseParam("+1 sec ")); + assertEquals(TimeUnit.DAYS.toMillis(7), this.timeshift.parseParam("+1week")); + assertEquals(TimeUnit.DAYS.toMillis(1), this.timeshift.parseParam("+1days")); + assertEquals(TimeUnit.HOURS.toMillis(1), this.timeshift.parseParam("+1hr")); + assertEquals(TimeUnit.MINUTES.toMillis(1), this.timeshift.parseParam("+1min")); + assertEquals(TimeUnit.SECONDS.toMillis(1), this.timeshift.parseParam("+1sec")); + assertEquals(TimeUnit.DAYS.toMillis(7), this.timeshift.parseParam("+1 week")); + assertEquals(TimeUnit.DAYS.toMillis(1), this.timeshift.parseParam("+1 days")); + assertEquals(TimeUnit.HOURS.toMillis(1), this.timeshift.parseParam("+1 hr")); + assertEquals(TimeUnit.MINUTES.toMillis(1), this.timeshift.parseParam("+1 min")); + assertEquals(TimeUnit.SECONDS.toMillis(1), this.timeshift.parseParam("+1 sec")); + assertEquals(60000L, this.timeshift.parseParam("+1min")); + } + + @Test + public void shiftDataPoint() throws Exception { + DataPoint actualDp = timeshift.shift(DATA_POINTS[0], 60000L); + assertEquals(1356998460000L, actualDp.timestamp()); + DataPoint actualDp1 = timeshift.shift(DATA_POINTS[1], this.timeshift.parseParam("+1week")); + assertEquals(1357605200000L, actualDp1.timestamp()); + DataPoint actualDp2 = timeshift.shift(DATA_POINTS[1], this.timeshift.parseParam("+130days")); + assertEquals(1368232400000L, actualDp2.timestamp()); + } +} \ No newline at end of file diff --git a/test/query/expression/TestTimeSyncedIterator.java b/test/query/expression/TestTimeSyncedIterator.java new file mode 100644 index 0000000000..70491001f1 --- /dev/null +++ b/test/query/expression/TestTimeSyncedIterator.java @@ -0,0 +1,504 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.expression; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.modules.junit4.PowerMockRunner; + +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.FillPolicy; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +public class TestTimeSyncedIterator extends BaseTimeSyncedIteratorTest { + + @Test + public void ctor() throws Exception { + threeDisjointSameE(); + queryAB_Dstar(); + final TimeSyncedIterator it = new TimeSyncedIterator("0", + results.get("0").getKey().getFilterTagKs(), + results.get("0").getValue()); + + assertEquals(3, it.size()); + assertTrue(it.hasNext()); + assertEquals(0, it.getIndex()); + assertEquals("0", it.getId()); + assertEquals(results.get("0").getKey().getFilterTagKs(), it.getQueryTagKs()); + assertTrue(results.get("0").getValue() == it.getDataPoints()); + assertEquals(3, it.values().length); + assertEquals(1, it.getQueryTagKs().size()); + assertEquals(FillPolicy.ZERO, it.getFillPolicy().getPolicy()); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNullId() throws Exception { + threeDisjointSameE(); + queryAB_Dstar(); + new TimeSyncedIterator(null, + results.get("0").getKey().getFilterTagKs(), + results.get("0").getValue()); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorEmptyId() throws Exception { + threeDisjointSameE(); + queryAB_Dstar(); + new TimeSyncedIterator("", + results.get("0").getKey().getFilterTagKs(), + results.get("0").getValue()); + } + + @Test + public void ctorNullQueryTags() throws Exception { + threeDisjointSameE(); + queryAB_Dstar(); + final TimeSyncedIterator it = new TimeSyncedIterator("0", null, + results.get("0").getValue()); + + assertEquals(3, it.size()); + assertTrue(it.hasNext()); + assertEquals(0, it.getIndex()); + assertEquals("0", it.getId()); + assertNull(it.getQueryTagKs()); + assertTrue(results.get("0").getValue() == it.getDataPoints()); + assertEquals(3, it.values().length); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNullDPs() throws Exception { + threeDisjointSameE(); + queryAB_Dstar(); + new TimeSyncedIterator("0", results.get("0").getKey().getFilterTagKs(), + null); + } + + @Test + public void threeSeries() throws Exception { + threeDisjointSameE(); + queryAB_Dstar(); + final TimeSyncedIterator it = new TimeSyncedIterator("0", + results.get("0").getKey().getFilterTagKs(), + results.get("0").getValue()); + + long ts = it.nextTimestamp(); + assertEquals(1431561600000L, ts); + + double[] values = new double[] { 1, 4, 7 }; + while (it.hasNext()) { + final DataPoint[] dps = it.next(ts); + assertEquals(3, dps.length); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + assertEquals(values[0]++, dps[0].toDouble(), 0.0001); + assertEquals(values[1]++, dps[1].toDouble(), 0.0001); + assertEquals(values[2]++, dps[2].toDouble(), 0.0001); + ts = it.nextTimestamp(); + } + assertEquals(Long.MAX_VALUE, ts); + } + + @Test + public void threeSeriesEmitter() throws Exception { + threeDisjointSameE(); + queryAB_Dstar(); + final TimeSyncedIterator it = new TimeSyncedIterator("0", + results.get("0").getKey().getFilterTagKs(), + results.get("0").getValue()); + + long ts = it.nextTimestamp(); + assertEquals(1431561600000L, ts); + final DataPoint[] dps = it.values(); + double[] values = new double[] { 1, 4, 7 }; + while (it.hasNext()) { + it.next(ts); + assertEquals(3, dps.length); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + assertEquals(values[0]++, dps[0].toDouble(), 0.0001); + assertEquals(values[1]++, dps[1].toDouble(), 0.0001); + assertEquals(values[2]++, dps[2].toDouble(), 0.0001); + ts = it.nextTimestamp(); + } + assertEquals(Long.MAX_VALUE, ts); + } + + @Test + public void nullASeries() throws Exception { + threeDisjointSameE(); + queryAB_Dstar(); + final TimeSyncedIterator it = new TimeSyncedIterator("0", + results.get("0").getKey().getFilterTagKs(), + results.get("0").getValue()); + // mimic the intersector kicking out a series + it.nullIterator(0); + + long ts = it.nextTimestamp(); + assertEquals(1431561600000L, ts); + + final DataPoint[] dps = it.values(); + double[] values = new double[] { 1, 4, 7 }; + while (it.hasNext()) { + it.next(ts); + assertEquals(3, dps.length); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + assertEquals(0, dps[0].toDouble(), 0.0001); + assertEquals(values[1]++, dps[1].toDouble(), 0.0001); + assertEquals(values[2]++, dps[2].toDouble(), 0.0001); + ts = it.nextTimestamp(); + } + assertEquals(Long.MAX_VALUE, ts); + } + + @Test + public void nullAll() throws Exception { + threeDisjointSameE(); + queryAB_Dstar(); + final TimeSyncedIterator it = new TimeSyncedIterator("0", + results.get("0").getKey().getFilterTagKs(), + results.get("0").getValue()); + // mimic the intersector kicking out all series. + it.nullIterator(0); + it.nullIterator(1); + it.nullIterator(2); + + assertEquals(Long.MAX_VALUE, it.nextTimestamp()); + assertFalse(it.hasNext()); + } + + @Test + public void singleSeries() throws Exception { + threeDisjointSameE(); + queryA_DD(); + final TimeSyncedIterator it = new TimeSyncedIterator("0", + results.get("0").getKey().getFilterTagKs(), + results.get("0").getValue()); + + long ts = it.nextTimestamp(); + assertEquals(1431561600000L, ts); + + double value = 1; + while (it.hasNext()) { + final DataPoint[] dps = it.next(ts); + assertEquals(1, dps.length); + assertEquals(ts, dps[0].timestamp()); + assertEquals(value++, dps[0].toDouble(), 0.0001); + ts = it.nextTimestamp(); + } + assertEquals(Long.MAX_VALUE, ts); + } + + @Test + public void noData() throws Exception { + setDataPointStorage(); + queryA_DD(); + final TimeSyncedIterator it = new TimeSyncedIterator("0", + results.get("0").getKey().getFilterTagKs(), + results.get("0").getValue()); + + assertEquals(0, it.values().length); + assertEquals(Long.MAX_VALUE, it.nextTimestamp()); + assertFalse(it.hasNext()); + } + + @Test + public void threeSeriesMissing() throws Exception { + threeSameEGaps(); + queryAB_Dstar(); + final TimeSyncedIterator it = new TimeSyncedIterator("0", + results.get("0").getKey().getFilterTagKs(), + results.get("0").getValue()); + it.setFillPolicy(new NumericFillPolicy(FillPolicy.NOT_A_NUMBER)); + + long ts = it.nextTimestamp(); + assertEquals(1431561600000L, ts); + + DataPoint[] dps = it.next(ts); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + + assertEquals(1, dps[0].toDouble(), 0.0001); + assertEquals(4, dps[1].toDouble(), 0.0001); + assertTrue(Double.isNaN(dps[2].toDouble())); + ts = it.nextTimestamp(); + + dps = it.next(ts); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + + assertTrue(Double.isNaN(dps[0].toDouble())); + assertEquals(5, dps[1].toDouble(), 0.0001); + assertEquals(8, dps[2].toDouble(), 0.0001); + ts = it.nextTimestamp(); + + dps = it.next(ts); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + + assertEquals(3, dps[0].toDouble(), 0.0001); + assertTrue(Double.isNaN(dps[1].toDouble())); + assertEquals(9, dps[2].toDouble(), 0.0001); + ts = it.nextTimestamp(); + } + + @Test + public void threeSeriesMissingFillZero() throws Exception { + threeSameEGaps(); + queryAB_Dstar(); + final TimeSyncedIterator it = new TimeSyncedIterator("0", + results.get("0").getKey().getFilterTagKs(), + results.get("0").getValue()); + it.setFillPolicy(new NumericFillPolicy(FillPolicy.ZERO)); + + long ts = it.nextTimestamp(); + assertEquals(1431561600000L, ts); + + DataPoint[] dps = it.next(ts); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + + assertEquals(1, dps[0].toDouble(), 0.0001); + assertEquals(4, dps[1].toDouble(), 0.0001); + assertEquals(0, dps[2].toDouble(), 0.0001); + ts = it.nextTimestamp(); + + dps = it.next(ts); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + + assertEquals(0, dps[0].toDouble(), 0.0001); + assertEquals(5, dps[1].toDouble(), 0.0001); + assertEquals(8, dps[2].toDouble(), 0.0001); + ts = it.nextTimestamp(); + + dps = it.next(ts); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + + assertEquals(3, dps[0].toDouble(), 0.0001); + assertEquals(0, dps[1].toDouble(), 0.0001); + assertEquals(9, dps[2].toDouble(), 0.0001); + ts = it.nextTimestamp(); + } + + @Test + public void threeSeriesMissingNull() throws Exception { + threeSameEGaps(); + queryAB_Dstar(); + final TimeSyncedIterator it = new TimeSyncedIterator("0", + results.get("0").getKey().getFilterTagKs(), + results.get("0").getValue()); + it.setFillPolicy(new NumericFillPolicy(FillPolicy.NULL)); + + long ts = it.nextTimestamp(); + assertEquals(1431561600000L, ts); + + DataPoint[] dps = it.next(ts); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + + assertEquals(1, dps[0].toDouble(), 0.0001); + assertEquals(4, dps[1].toDouble(), 0.0001); + assertTrue(Double.isNaN(dps[2].toDouble())); + ts = it.nextTimestamp(); + + dps = it.next(ts); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + + assertTrue(Double.isNaN(dps[0].toDouble())); + assertEquals(5, dps[1].toDouble(), 0.0001); + assertEquals(8, dps[2].toDouble(), 0.0001); + ts = it.nextTimestamp(); + + dps = it.next(ts); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + + assertEquals(3, dps[0].toDouble(), 0.0001); + assertTrue(Double.isNaN(dps[1].toDouble())); + assertEquals(9, dps[2].toDouble(), 0.0001); + ts = it.nextTimestamp(); + } + + @Test + public void threeSeriesMissingScalar() throws Exception { + threeSameEGaps(); + queryAB_Dstar(); + final TimeSyncedIterator it = new TimeSyncedIterator("0", + results.get("0").getKey().getFilterTagKs(), + results.get("0").getValue()); + it.setFillPolicy(new NumericFillPolicy(FillPolicy.SCALAR, 42)); + + long ts = it.nextTimestamp(); + assertEquals(1431561600000L, ts); + + DataPoint[] dps = it.next(ts); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + + assertEquals(1, dps[0].toDouble(), 0.0001); + assertEquals(4, dps[1].toDouble(), 0.0001); + assertEquals(42, dps[2].toDouble(), 0.0001); + ts = it.nextTimestamp(); + + dps = it.next(ts); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + + assertEquals(42, dps[0].toDouble(), 0.0001); + assertEquals(5, dps[1].toDouble(), 0.0001); + assertEquals(8, dps[2].toDouble(), 0.0001); + ts = it.nextTimestamp(); + + dps = it.next(ts); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + + assertEquals(3, dps[0].toDouble(), 0.0001); + assertEquals(42, dps[1].toDouble(), 0.0001); + assertEquals(9, dps[2].toDouble(), 0.0001); + ts = it.nextTimestamp(); + } + + @Test + public void failToNextTS() throws Exception { + threeDisjointSameE(); + queryAB_Dstar(); + final TimeSyncedIterator it = new TimeSyncedIterator("0", + results.get("0").getKey().getFilterTagKs(), + results.get("0").getValue()); + + long ts = it.nextTimestamp(); + assertEquals(1431561600000L, ts); + + int i = 0; + boolean broke = false; + while (it.hasNext()) { + // in this case the caller fails to advance the TS so we keep getting the + // same data points over and over again. + it.next(ts); + ++i; + if (i > 100) { + broke = true; + break; + } + } + if (!broke) { + fail("Expected to iterate over 100 times"); + } + } + + @Test + public void nextTimestampDoesntAdvance() throws Exception { + threeDisjointSameE(); + queryAB_Dstar(); + final TimeSyncedIterator it = new TimeSyncedIterator("0", + results.get("0").getKey().getFilterTagKs(), + results.get("0").getValue()); + + assertEquals(1431561600000L, it.nextTimestamp()); + assertEquals(1431561600000L, it.nextTimestamp()); + assertEquals(1431561600000L, it.nextTimestamp()); + assertEquals(1431561600000L, it.nextTimestamp()); + assertEquals(1431561600000L, it.nextTimestamp()); + assertEquals(1431561600000L, it.nextTimestamp()); + assertEquals(1431561600000L, it.nextTimestamp()); + } + + @Test + public void nextExceptionNoException() throws Exception { + threeDisjointSameE(); + queryAB_Dstar(); + final TimeSyncedIterator it = new TimeSyncedIterator("0", + results.get("0").getKey().getFilterTagKs(), + results.get("0").getValue()); + it.next(it.nextTimestamp()); + it.next(it.nextTimestamp()); + it.next(it.nextTimestamp()); + it.next(it.nextTimestamp()); + assertEquals(Long.MAX_VALUE, it.nextTimestamp()); + } + + @Test + public void getCopy() throws Exception { + threeDisjointSameE(); + queryAB_Dstar(); + final TimeSyncedIterator it = new TimeSyncedIterator("0", + results.get("0").getKey().getFilterTagKs(), + results.get("0").getValue()); + + final ITimeSyncedIterator copy = it.getCopy(); + assertTrue(copy != it); + + long ts = it.nextTimestamp(); + assertEquals(1431561600000L, ts); + + double[] values = new double[] { 1, 4, 7 }; + while (it.hasNext()) { + final DataPoint[] dps = it.next(ts); + assertEquals(3, dps.length); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + assertEquals(values[0]++, dps[0].toDouble(), 0.0001); + assertEquals(values[1]++, dps[1].toDouble(), 0.0001); + assertEquals(values[2]++, dps[2].toDouble(), 0.0001); + ts = it.nextTimestamp(); + } + assertEquals(Long.MAX_VALUE, ts); + + ts = copy.nextTimestamp(); + assertEquals(1431561600000L, ts); + + values = new double[] { 1, 4, 7 }; + while (copy.hasNext()) { + final DataPoint[] dps = copy.next(ts); + assertEquals(3, dps.length); + assertEquals(ts, dps[0].timestamp()); + assertEquals(ts, dps[1].timestamp()); + assertEquals(ts, dps[2].timestamp()); + assertEquals(values[0]++, dps[0].toDouble(), 0.0001); + assertEquals(values[1]++, dps[1].toDouble(), 0.0001); + assertEquals(values[2]++, dps[2].toDouble(), 0.0001); + ts = copy.nextTimestamp(); + } + assertEquals(Long.MAX_VALUE, ts); + } +} diff --git a/test/query/expression/TestUnionIterator.java b/test/query/expression/TestUnionIterator.java new file mode 100644 index 0000000000..77a191e7bb --- /dev/null +++ b/test/query/expression/TestUnionIterator.java @@ -0,0 +1,1075 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.expression; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.HashMap; +import java.util.Map; + +import org.hbase.async.HBaseClient; +import org.hbase.async.Bytes.ByteMap; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.FillPolicy; +import net.opentsdb.core.IllegalDataException; +import net.opentsdb.storage.MockBase; +import net.opentsdb.utils.ByteSet; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ TimeSyncedIterator.class }) +public class TestUnionIterator extends BaseTimeSyncedIteratorTest { + + /** used for the flattenTags tests */ + private static final byte[] UID1 = new byte[] { 0, 0, 1 }; + private static final byte[] UID2 = new byte[] { 0, 0, 2 }; + private static final byte[] UID3 = new byte[] { 0, 0, 3 }; + private ByteMap<byte[]> tags; + private ByteSet agg_tags; + private ITimeSyncedIterator sub; + private ByteSet query_tags; + private NumericFillPolicy fill_policy; + + @Before + public void beforeLocal() throws Exception { + tags = new ByteMap<byte[]>(); + tags.put(UID1, UID1); + tags.put(UID2, UID2); + + agg_tags = new ByteSet(); + agg_tags.add(UID3); + + fill_policy = new NumericFillPolicy(FillPolicy.NOT_A_NUMBER); + sub = mock(ITimeSyncedIterator.class); + query_tags = new ByteSet(); + query_tags.add(UID1); + when(sub.getQueryTagKs()).thenReturn(query_tags); + when(sub.getFillPolicy()).thenReturn(fill_policy); + } + + @Test (expected = NullPointerException.class) + public void ctorNullResults() { + new UnionIterator("it", null, true, true); + } + + @Test + public void ctorEmptyResults() { + final UnionIterator it = new UnionIterator("it", + new HashMap<String, ITimeSyncedIterator>(), true, true); + assertEquals(0, it.getSeriesSize()); + assertFalse(it.hasNext()); + } + + @Test + public void twoAndThreeSeries() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + + final UnionIterator it = new UnionIterator("it", iterators, false, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(3, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 1, 4, 11, 14, 17 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(values[0]++, set_dps[0].toDouble(), 0.0001); + assertEquals(values[1]++, set_dps[1].toDouble(), 0.0001); + assertEquals(0, set_dps[2].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(values[2]++, set_dps[0].toDouble(), 0.0001); + assertEquals(values[3]++, set_dps[1].toDouble(), 0.0001); + assertEquals(values[4]++, set_dps[2].toDouble(), 0.0001); + ts += 60000; + } + } + + @Test + public void twoAndThreeSeriesExtraDP() throws Exception { + oneExtraSameE(); + queryAB_Dstar(); + + final HashMap<String, String> tags = new HashMap<String, String>(2); + tags.put("D", "G"); + tags.put("E", "E"); + tsdb.addPoint("B", 1431561630, 14, tags).joinUninterruptibly(); + + final UnionIterator it = new UnionIterator("it", iterators, false, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(3, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 1, 4, 11, 14, 17 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(values[0]++, set_dps[0].toDouble(), 0.0001); + assertEquals(values[1]++, set_dps[1].toDouble(), 0.0001); + assertEquals(0, set_dps[2].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(values[2]++, set_dps[0].toDouble(), 0.0001); + assertEquals(values[3]++, set_dps[1].toDouble(), 0.0001); + assertEquals(values[4]++, set_dps[2].toDouble(), 0.0001); + + ts += 60000; + } + } + + @Test + public void threeSeriesUnionToFour() throws Exception { + threeDisjointSameE(); + queryAB_Dstar(); + + final UnionIterator it = new UnionIterator("it", iterators, false, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(4, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 1, 4, 7, 11, 17, 14 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(4, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(ts, set_dps[3].timestamp()); + assertEquals(values[0]++, set_dps[0].toDouble(), 0.0001); + assertEquals(values[1]++, set_dps[1].toDouble(), 0.0001); + assertEquals(values[2]++, set_dps[2].toDouble(), 0.0001); + assertEquals(0, set_dps[3].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(4, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(ts, set_dps[3].timestamp()); + assertEquals(values[3]++, set_dps[0].toDouble(), 0.0001); + assertEquals(0, set_dps[1].toDouble(), 0.0001); + assertEquals(values[4]++, set_dps[2].toDouble(), 0.0001); + assertEquals(values[5]++, set_dps[3].toDouble(), 0.0001); + + ts += 60000; + } + } + + @Test + public void threeSeriesUnionToExtraDPs() throws Exception { + reduceToOne(); // though we won't :) + queryAB_Dstar(); + + HashMap<String, String> tags = new HashMap<String, String>(2); + tags.put("D", "F"); + tags.put("E", "E"); + tsdb.addPoint("A", 1431561630, 1024, tags).joinUninterruptibly(); + + tags = new HashMap<String, String>(2); + tags.put("D", "Q"); + tags.put("E", "E"); + tsdb.addPoint("B", 1431561630, 1024, tags).joinUninterruptibly(); + + final UnionIterator it = new UnionIterator("it", iterators, false, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(5, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 1, 4, 7, 17, 11, 14 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(5, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(ts, set_dps[3].timestamp()); + assertEquals(ts, set_dps[4].timestamp()); + assertEquals(values[0]++, set_dps[0].toDouble(), 0.0001); + assertEquals(values[1]++, set_dps[1].toDouble(), 0.0001); + assertEquals(values[2]++, set_dps[2].toDouble(), 0.0001); + assertEquals(0, set_dps[3].toDouble(), 0.0001); + assertEquals(0, set_dps[4].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(5, set_dps.length); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(ts, set_dps[3].timestamp()); + assertEquals(ts, set_dps[4].timestamp()); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(0, set_dps[0].toDouble(), 0.0001); + assertEquals(0, set_dps[1].toDouble(), 0.0001); + assertEquals(values[3]++, set_dps[2].toDouble(), 0.0001); + assertEquals(values[4]++, set_dps[3].toDouble(), 0.0001); + assertEquals(values[5]++, set_dps[4].toDouble(), 0.0001); + + ts += 60000; + } + } + + @Test + public void threeSeriesAgged() throws Exception { + threeSameE(); + queryAB_AggAll(); + + final UnionIterator it = new UnionIterator("it", iterators, false, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(1, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 12, 42 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(1, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(values[0], set_dps[0].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(1, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(values[1], set_dps[0].toDouble(), 0.0001); + + values[0] += 3; + values[1] += 3; + + ts += 60000; + } + } + + @Test + public void threeSeriesWithNaNs() throws Exception { + threeSameEGaps(); + queryAB_Dstar(); + for (ITimeSyncedIterator iterator : iterators.values()) { + iterator.setFillPolicy(new NumericFillPolicy(FillPolicy.NOT_A_NUMBER)); + } + + final UnionIterator it = new UnionIterator("it", iterators, false, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(3, it.getSeriesSize()); + + long ts = 1431561600000L; + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(1, set_dps[0].toDouble(), 0.0001); + assertEquals(4, set_dps[1].toDouble(), 0.0001); + assertTrue(Double.isNaN(set_dps[2].toDouble())); + + // whole series is NaN'd + set_dps = dps.get("1"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertTrue(Double.isNaN(set_dps[0].toDouble())); + assertTrue(Double.isNaN(set_dps[1].toDouble())); + assertTrue(Double.isNaN(set_dps[2].toDouble())); + + ts += 60000; + it.next(); + + set_dps = dps.get("0"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertTrue(Double.isNaN(set_dps[0].toDouble())); + assertEquals(5, set_dps[1].toDouble(), 0.0001); + assertEquals(8, set_dps[2].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertTrue(Double.isNaN(set_dps[0].toDouble())); + assertEquals(15, set_dps[1].toDouble(), 0.0001); + assertTrue(Double.isNaN(set_dps[2].toDouble())); + + ts += 60000; + it.next(); + + set_dps = dps.get("0"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(3, set_dps[0].toDouble(), 0.0001); + assertTrue(Double.isNaN(set_dps[1].toDouble())); + assertEquals(9, set_dps[2].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(13, set_dps[0].toDouble(), 0.0001); + assertTrue(Double.isNaN(set_dps[1].toDouble())); + assertEquals(19, set_dps[2].toDouble(), 0.0001); + + assertFalse(it.hasNext()); + } + + @Test + public void twoSeriesTimeOffset() throws Exception { + timeOffset(); + queryAB_Dstar(); + for (ITimeSyncedIterator iterator : iterators.values()) { + iterator.setFillPolicy(new NumericFillPolicy(FillPolicy.NOT_A_NUMBER)); + } + + final UnionIterator it = new UnionIterator("it", iterators, false, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(2, it.getSeriesSize()); + + long ts = 1431561600000L; + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(2, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(1, set_dps[0].toDouble(), 0.0001); + assertEquals(4, set_dps[1].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(2, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertTrue(Double.isNaN(set_dps[0].toDouble())); + assertTrue(Double.isNaN(set_dps[1].toDouble())); + + ts += 60000; + it.next(); + + set_dps = dps.get("0"); + assertEquals(2, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(2, set_dps[0].toDouble(), 0.0001); + assertEquals(5, set_dps[1].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(2, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertTrue(Double.isNaN(set_dps[0].toDouble())); + assertTrue(Double.isNaN(set_dps[1].toDouble())); + + ts += 60000; + it.next(); + + set_dps = dps.get("0"); + assertEquals(2, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertTrue(Double.isNaN(set_dps[0].toDouble())); + assertTrue(Double.isNaN(set_dps[1].toDouble())); + + set_dps = dps.get("1"); + assertEquals(2, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(13, set_dps[0].toDouble(), 0.0001); + assertEquals(16, set_dps[1].toDouble(), 0.0001); + + ts += 60000; + it.next(); + + set_dps = dps.get("0"); + assertEquals(2, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertTrue(Double.isNaN(set_dps[0].toDouble())); + assertTrue(Double.isNaN(set_dps[1].toDouble())); + + set_dps = dps.get("1"); + assertEquals(2, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(14, set_dps[0].toDouble(), 0.0001); + assertEquals(17, set_dps[1].toDouble(), 0.0001); + + assertFalse(it.hasNext()); + } + + @Test + public void threeSeriesUsingResultTags() throws Exception { + threeDifE(); + queryAB_Dstar(); + final UnionIterator it = new UnionIterator("it", iterators, false, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(6, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 1, 4, 7, 11, 14, 17 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(6, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(ts, set_dps[3].timestamp()); + assertEquals(ts, set_dps[4].timestamp()); + assertEquals(ts, set_dps[5].timestamp()); + assertEquals(values[0]++, set_dps[0].toDouble(), 0.0001); + assertEquals(0, set_dps[1].toDouble(), 0.0001); + assertEquals(values[1]++, set_dps[2].toDouble(), 0.0001); + assertEquals(0, set_dps[3].toDouble(), 0.0001); + assertEquals(values[2]++, set_dps[4].toDouble(), 0.0001); + assertEquals(0, set_dps[5].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(6, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(ts, set_dps[3].timestamp()); + assertEquals(ts, set_dps[4].timestamp()); + assertEquals(ts, set_dps[5].timestamp()); + assertEquals(0, set_dps[0].toDouble(), 0.0001); + assertEquals(values[3]++, set_dps[1].toDouble(), 0.0001); + assertEquals(0, set_dps[2].toDouble(), 0.0001); + assertEquals(values[4]++, set_dps[3].toDouble(), 0.0001); + assertEquals(0, set_dps[4].toDouble(), 0.0001); + assertEquals(values[5]++, set_dps[5].toDouble(), 0.0001); + + ts += 60000; + } + } + + @Test + public void threeSeriesUsingQueryTags() throws Exception { + threeDifE(); + queryAB_Dstar(); + final UnionIterator it = new UnionIterator("it", iterators, true, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(3, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 1, 4, 7, 11, 14, 17 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(values[0]++, set_dps[0].toDouble(), 0.0001); + assertEquals(values[1]++, set_dps[1].toDouble(), 0.0001); + assertEquals(values[2]++, set_dps[2].toDouble(), 0.0001); + //assertEquals(0, set_dps[3].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(values[3]++, set_dps[0].toDouble(), 0.0001); + assertEquals(values[4]++, set_dps[1].toDouble(), 0.0001); + assertEquals(values[5]++, set_dps[2].toDouble(), 0.0001); + + ts += 60000; + } + } + + @Test + public void commonAggregatedTag() throws Exception { + twoSeriesAggedE(); + queryAB_Dstar(); + + final UnionIterator it = new UnionIterator("it", iterators, false, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(1, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 2, 22 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(1, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(values[0], set_dps[0].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(1, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(values[1], set_dps[0].toDouble(), 0.0001); + + values[0] += 2; + values[1] += 2; + + ts += 60000; + } + } + + @Test + public void extraAggTagIgnored() throws Exception { + twoSeriesAggedEandExtraTagK(); + queryAB_Dstar(); + + final UnionIterator it = new UnionIterator("it", iterators, false, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(1, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 2, 22 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(1, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(values[0], set_dps[0].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(1, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(values[1], set_dps[0].toDouble(), 0.0001); + + values[0] += 2; + values[1] += 2; + + ts += 60000; + } + } + + @Test + public void extraAggTag() throws Exception { + twoSeriesAggedEandExtraTagK(); + queryAB_Dstar(); + + final UnionIterator it = new UnionIterator("it", iterators, false, true); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(2, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 2, 22 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(2, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(values[0], set_dps[0].toDouble(), 0.0001); + assertEquals(0, set_dps[1].toDouble(), 0.0001); + values[0] += 2; + + set_dps = dps.get("1"); + assertEquals(2, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(0, set_dps[0].toDouble(), 0.0001); + assertEquals(values[1], set_dps[1].toDouble(), 0.0001); + values[1] += 2; + + ts += 60000; + } + } + + @Test + public void onlyOneResultSet() throws Exception { + threeSameENoB(); + queryAB_Dstar(); + + final UnionIterator it = new UnionIterator("it", iterators, false, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(3, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 1, 4, 7 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(values[0]++, set_dps[0].toDouble(), 0.0001); + assertEquals(values[1]++, set_dps[1].toDouble(), 0.0001); + assertEquals(values[2]++, set_dps[2].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(0, set_dps[0].toDouble(), 0.0001); + assertEquals(0, set_dps[1].toDouble(), 0.0001); + assertEquals(0, set_dps[2].toDouble(), 0.0001); + + ts += 60000; + } + } + + @Test + public void onlyOneResultSetQueryTags() throws Exception { + threeSameENoB(); + queryAB_Dstar(); + + final UnionIterator it = new UnionIterator("it", iterators, true, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(3, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 1, 4, 7 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(values[0]++, set_dps[0].toDouble(), 0.0001); + assertEquals(values[1]++, set_dps[1].toDouble(), 0.0001); + assertEquals(values[2]++, set_dps[2].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(0, set_dps[0].toDouble(), 0.0001); + assertEquals(0, set_dps[1].toDouble(), 0.0001); + assertEquals(0, set_dps[2].toDouble(), 0.0001); + + ts += 60000; + } + } + + @Test + public void onlyOneResultSetAggTags() throws Exception { + threeSameENoB(); + queryAB_Dstar(); + + final UnionIterator it = new UnionIterator("it", iterators, false, true); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(3, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 1, 4, 7 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(values[0]++, set_dps[0].toDouble(), 0.0001); + assertEquals(values[1]++, set_dps[1].toDouble(), 0.0001); + assertEquals(values[2]++, set_dps[2].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(0, set_dps[0].toDouble(), 0.0001); + assertEquals(0, set_dps[1].toDouble(), 0.0001); + assertEquals(0, set_dps[2].toDouble(), 0.0001); + + ts += 60000; + } + } + + @Test + public void oneAggedOneTagged() throws Exception { + oneAggedTheOtherTagged(); + queryAB_AggAll(); + final UnionIterator it = new UnionIterator("it", iterators, false, true); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(2, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 2, 11 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(2, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(values[0], set_dps[0].toDouble(), 0.0001); + assertEquals(0, set_dps[1].toDouble(), 0.0001); + values[0] += 2; + + set_dps = dps.get("1"); + assertEquals(2, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + System.out.println(set_dps[0].toDouble()); + System.out.println(set_dps[1].toDouble()); + assertEquals(0, set_dps[0].toDouble(), 0.0001); + assertEquals(values[1]++, set_dps[1].toDouble(), 0.0001); + + ts += 60000; + } + } + + @Test + public void oneAggedOneTaggedUseQueryTagsWoutQueryTags() throws Exception { + oneAggedTheOtherTagged(); + queryAB_AggAll(); + + final UnionIterator it = new UnionIterator("it", iterators, true, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(1, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 2, 11 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(1, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(values[0], set_dps[0].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(1, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(values[1]++, set_dps[0].toDouble(), 0.0001); + + // the first set is agged + values[0] += 2; + + ts += 60000; + } + } + + @Test + public void singleSeries() throws Exception { + oneExtraSameE(); + queryA_DD(); + + final UnionIterator it = new UnionIterator("it", iterators, false, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(1, dps.size()); + assertEquals(1, it.getSeriesSize()); + + long ts = 1431561600000L; + double value = 1; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(1, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(value++, set_dps[0].toDouble(), 0.0001); + + ts += 60000; + } + } + + @Test + public void setAMissingE() throws Exception { + threeAMissingE(); + queryAB_Dstar(); + + final UnionIterator it = new UnionIterator("it", iterators, false, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(6, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 1, 4, 7, 11, 14, 17 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(6, set_dps.length); + for (int i = 0; i < set_dps.length; i++) { + assertEquals(ts, set_dps[i].timestamp()); + } + assertEquals(values[0]++, set_dps[0].toDouble(), 0.0001); + assertEquals(0, set_dps[1].toDouble(), 0.0001); + assertEquals(values[1]++, set_dps[2].toDouble(), 0.0001); + assertEquals(0, set_dps[3].toDouble(), 0.0001); + assertEquals(values[2]++, set_dps[4].toDouble(), 0.0001); + assertEquals(0, set_dps[5].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(6, set_dps.length); + for (int i = 0; i < set_dps.length; i++) { + assertEquals(ts, set_dps[i].timestamp()); + } + assertEquals(0, set_dps[0].toDouble(), 0.0001); + assertEquals(values[3]++, set_dps[1].toDouble(), 0.0001); + assertEquals(0, set_dps[2].toDouble(), 0.0001); + assertEquals(values[4]++, set_dps[3].toDouble(), 0.0001); + assertEquals(0, set_dps[4].toDouble(), 0.0001); + assertEquals(values[5]++, set_dps[5].toDouble(), 0.0001); + + ts += 60000; + } + } + + @Test + public void setAMissingEQueryTags() throws Exception { + threeAMissingE(); + queryAB_Dstar(); + + final UnionIterator it = new UnionIterator("it", iterators, true, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertTrue(it.hasNext()); + assertEquals(2, dps.size()); + assertEquals(3, it.getSeriesSize()); + + long ts = 1431561600000L; + double values[] = new double[] { 1, 4, 7, 11, 14, 17 }; + while (it.hasNext()) { + it.next(); + + DataPoint[] set_dps = dps.get("0"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(values[0]++, set_dps[0].toDouble(), 0.0001); + assertEquals(values[1]++, set_dps[1].toDouble(), 0.0001); + assertEquals(values[2]++, set_dps[2].toDouble(), 0.0001); + + set_dps = dps.get("1"); + assertEquals(3, set_dps.length); + assertEquals(ts, set_dps[0].timestamp()); + assertEquals(ts, set_dps[1].timestamp()); + assertEquals(ts, set_dps[2].timestamp()); + assertEquals(values[3]++, set_dps[0].toDouble(), 0.0001); + assertEquals(values[4]++, set_dps[1].toDouble(), 0.0001); + assertEquals(values[5]++, set_dps[2].toDouble(), 0.0001); + + ts += 60000; + } + } + + @Test + public void noData() throws Exception { + setDataPointStorage(); + queryAB_Dstar(); + + final UnionIterator it = new UnionIterator("it", iterators, false, false); + final Map<String, ExpressionDataPoint[]> dps = it.getResults(); + assertEquals(0, dps.size()); + assertFalse(it.hasNext()); + assertEquals(0, it.getSeriesSize()); + } + + @Test (expected = IllegalDataException.class) + public void nextException() throws Exception { + threeDisjointSameE(); + queryAB_Dstar(); + + final UnionIterator it = new UnionIterator("it", iterators, false, false); + it.next(); + it.next(); + it.next(); + it.next(); + } + + @Test + public void flattenTags() throws Exception { + final ExpressionDataPoint dp = getMockDB(tags, agg_tags); + final byte[] flat = UnionIterator.flattenTags(false, false, dp, sub); + assertArrayEquals(MockBase.concatByteArrays(UID1, UID1, UID2, UID2), flat); + } + + @Test + public void flattenTagsWithAgg() throws Exception { + final ExpressionDataPoint dp = getMockDB(tags, agg_tags); + final byte[] flat = UnionIterator.flattenTags(false, true, dp, sub); + assertArrayEquals( + MockBase.concatByteArrays(UID1, UID1, UID2, UID2, UID3), flat); + } + + @Test + public void flattenTagsQueryTags() throws Exception { + final ExpressionDataPoint dp = getMockDB(tags, agg_tags); + final byte[] flat = UnionIterator.flattenTags(true, false, dp, sub); + assertArrayEquals(MockBase.concatByteArrays(UID1, UID1), flat); + } + + @Test + public void flattenTagsQueryTagsWithAgg() throws Exception { + final ExpressionDataPoint dp = getMockDB(tags, agg_tags); + final byte[] flat = UnionIterator.flattenTags(true, true, dp, sub); + assertArrayEquals(MockBase.concatByteArrays(UID1, UID1, UID3), flat); + } + + @Test + public void flattenEmptyTags() throws Exception { + tags.clear(); + final ExpressionDataPoint dp = getMockDB(tags, agg_tags); + final byte[] flat = UnionIterator.flattenTags(false, false, dp, sub); + assertArrayEquals(HBaseClient.EMPTY_ARRAY, flat); + } + + @Test + public void flattenEmptyTagsWithAggEmpty() throws Exception { + agg_tags.clear(); + final ExpressionDataPoint dp = getMockDB(tags, agg_tags); + final byte[] flat = UnionIterator.flattenTags(false, true, dp, sub); + assertArrayEquals(MockBase.concatByteArrays(UID1, UID1, UID2, UID2), flat); + } + + // TODO - how will this play out if we choose a default agg of "none" and the + // user hasn't asked for any filtering? + @Test + public void flattenTagsQueryTagsEmpty() throws Exception { + query_tags.clear(); + final ExpressionDataPoint dp = getMockDB(tags, agg_tags); + final byte[] flat = UnionIterator.flattenTags(true, false, dp, sub); + assertArrayEquals(HBaseClient.EMPTY_ARRAY, flat); + } + + @Test + public void flattenTagsQueryTagsEmptyWithAgg() throws Exception { + query_tags.clear(); + final ExpressionDataPoint dp = getMockDB(tags, agg_tags); + final byte[] flat = UnionIterator.flattenTags(true, true, dp, sub); + assertArrayEquals(UID3, flat); + } + + @Test + public void flattenTagsNullTags() throws Exception { + final ExpressionDataPoint dp = getMockDB(null, agg_tags); + final byte[] flat = UnionIterator.flattenTags(true, false, dp, sub); + assertArrayEquals(HBaseClient.EMPTY_ARRAY, flat); + } + + @Test + public void flattenTagsNullAggTagsNotRequested() throws Exception { + final ExpressionDataPoint dp = getMockDB(tags, null); + final byte[] flat = UnionIterator.flattenTags(false, false, dp, sub); + assertArrayEquals(MockBase.concatByteArrays(UID1, UID1, UID2, UID2), flat); + } + + @Test (expected = NullPointerException.class) + public void flattenTagsNullAggTags() throws Exception { + final ExpressionDataPoint dp = getMockDB(tags, null); + UnionIterator.flattenTags(false, true, dp, sub); + } + + @Test + public void flattenTagsNullSubNotRequested() throws Exception { + final ExpressionDataPoint dp = getMockDB(tags, agg_tags); + final byte[] flat = UnionIterator.flattenTags(false, false, dp, null); + assertArrayEquals(MockBase.concatByteArrays(UID1, UID1, UID2, UID2), flat); + } + + @Test (expected = NullPointerException.class) + public void flattenTagsNullSub() throws Exception { + final ExpressionDataPoint dp = getMockDB(tags, agg_tags); + UnionIterator.flattenTags(true, false, dp, null); + } + + /** + * A helper to mock out the calls for flatten tags + * @param tags The tags to return + * @param agg_tags The aggregated tags to return + * @return A mocked data point + */ + private ExpressionDataPoint getMockDB(final ByteMap<byte[]> tags, + final ByteSet agg_tags) { + final ExpressionDataPoint dp = mock(ExpressionDataPoint.class); + when(dp.tags()).thenReturn(tags); + when(dp.aggregatedTags()).thenReturn(agg_tags); + return dp; + } +} diff --git a/test/query/filter/TestTagVFilter.java b/test/query/filter/TestTagVFilter.java new file mode 100644 index 0000000000..5c952c83df --- /dev/null +++ b/test/query/filter/TestTagVFilter.java @@ -0,0 +1,421 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.filter; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import net.opentsdb.core.BaseTsdbTest; +import net.opentsdb.core.TSDB; +import net.opentsdb.uid.NoSuchUniqueName; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +import com.stumbleupon.async.DeferredGroupException; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ TSDB.class }) +public class TestTagVFilter extends BaseTsdbTest { + + @Test (expected = IllegalArgumentException.class) + public void getFilterNullTagk() throws Exception { + TagVFilter.getFilter(null, "myflter"); + } + + @Test (expected = IllegalArgumentException.class) + public void getFilterEmptyTagk() throws Exception { + TagVFilter.getFilter(null, "myflter"); + } + + @Test (expected = IllegalArgumentException.class) + public void getFilterEmptyFilter() throws Exception { + TagVFilter.getFilter(TAGK_STRING, ""); + } + + @Test (expected = IllegalArgumentException.class) + public void getFilterNullFilter() throws Exception { + TagVFilter.getFilter(TAGK_STRING, null); + } + + @Test + public void getFilterGroupBy() throws Exception { + assertNull(TagVFilter.getFilter(TAGK_STRING, "*")); + } + + @Test + public void getFilterLiteral() throws Exception { + assertNull(TagVFilter.getFilter(TAGK_STRING, TAGV_STRING)); + } + + @Test + public void getFilterGroupByPiped() throws Exception { + assertNull(TagVFilter.getFilter(TAGK_STRING, "web01|web02")); + } + + @Test + public void getFilterWildcard() throws Exception { + final TagVFilter filter = TagVFilter.getFilter(TAGK_STRING, + TagVWildcardFilter.FILTER_NAME + "(*bonk.com)"); + assertEquals(TAGK_STRING, filter.getTagk()); + assertTrue(filter instanceof TagVWildcardFilter); + assertFalse(((TagVWildcardFilter)filter).isCaseInsensitive()); + } + + @Test + public void getFilterWildcardInsensitive() throws Exception { + final TagVFilter filter = TagVFilter.getFilter(TAGK_STRING, + TagVWildcardFilter.TagVIWildcardFilter.FILTER_NAME + "(*bonk.com)"); + assertEquals(TAGK_STRING, filter.getTagk()); + assertTrue(filter instanceof TagVWildcardFilter); + assertTrue(((TagVWildcardFilter)filter).isCaseInsensitive()); + } + + @Test + public void getFilterWildcardFatfinger() throws Exception { + // falls through to the shortcut + final TagVFilter filter = TagVFilter.getFilter(TAGK_STRING, + "wil@*sugarbean"); + assertEquals(TAGK_STRING, filter.getTagk()); + assertTrue(filter instanceof TagVWildcardFilter); + assertTrue(((TagVWildcardFilter)filter).isCaseInsensitive()); + } + + @Test + public void getFilterWildcardImplicit() throws Exception { + final TagVFilter filter = TagVFilter.getFilter(TAGK_STRING, "*bonk.com"); + assertEquals(TAGK_STRING, filter.getTagk()); + assertTrue(filter instanceof TagVWildcardFilter); + assertTrue(((TagVWildcardFilter)filter).isCaseInsensitive()); + } + + @Test + public void getFilterPipe() throws Exception { + final TagVFilter filter = TagVFilter.getFilter(TAGK_STRING, + TagVLiteralOrFilter.FILTER_NAME + "(quirm|bonk)"); + assertEquals(TAGK_STRING, filter.getTagk()); + assertTrue(filter instanceof TagVLiteralOrFilter); + assertFalse(((TagVLiteralOrFilter)filter).isCaseInsensitive()); + } + + @Test + public void getFilterPipeInsensitive() throws Exception { + final TagVFilter filter = TagVFilter.getFilter(TAGK_STRING, + TagVLiteralOrFilter.TagVILiteralOrFilter.FILTER_NAME + "(quirm|bonk)"); + assertEquals(TAGK_STRING, filter.getTagk()); + assertTrue(filter instanceof TagVLiteralOrFilter); + assertTrue(((TagVLiteralOrFilter)filter).isCaseInsensitive()); + } + + @Test + public void getFilterPipeFatfinger() throws Exception { + assertNull(TagVFilter.getFilter(TAGK_STRING, "lite@sugarbean|granny")); + } + + @Test + public void getFilterRegex() throws Exception { + final TagVFilter filter = TagVFilter.getFilter(TAGK_STRING, + TagVRegexFilter.FILTER_NAME + "(.*sugarbean)"); + assertEquals(TAGK_STRING, filter.getTagk()); + assertTrue(filter instanceof TagVRegexFilter); + } + + @Test + public void getFilterRegexFatFinger() throws Exception { + // falls through to the implicity + final TagVFilter filter = TagVFilter.getFilter(TAGK_STRING, "rexp@.*sugarbean"); + assertEquals(TAGK_STRING, filter.getTagk()); + assertTrue(filter instanceof TagVWildcardFilter); + assertTrue(((TagVWildcardFilter)filter).isCaseInsensitive()); + } + + @Test + public void getFilterRegexCase() throws Exception { + final TagVFilter filter = TagVFilter.getFilter(TAGK_STRING, + TagVRegexFilter.FILTER_NAME.toUpperCase() + "(.*sugarbean)"); + assertEquals(TAGK_STRING, filter.getTagk()); + assertTrue(filter instanceof TagVRegexFilter); + } + + @Test (expected = IllegalArgumentException.class) + public void getFilterMissingClosingParens() throws Exception { + TagVFilter.getFilter(TAGK_STRING, TagVRegexFilter.FILTER_NAME + "(.*sugarbean"); + } + + @Test (expected = IllegalArgumentException.class) + public void getFilterEmptyParens() throws Exception { + TagVFilter.getFilter(TAGK_STRING, TagVRegexFilter.FILTER_NAME + "()"); + } + + @Test (expected = IllegalArgumentException.class) + public void getFilterUnknownType() throws Exception { + TagVFilter.getFilter(TAGK_STRING, "dummyfilter(nothere)"); + } + + @Test + public void resolveName() throws Exception { + final TagVFilter filter = new TagVWildcardFilter(TAGK_STRING, "*omnia"); + filter.resolveTagkName(tsdb).join(); + assertArrayEquals(TAGK_BYTES, filter.getTagkBytes()); + assertTrue(filter.getTagVUids().isEmpty()); + } + + @Test + public void resolveNameLiteral() throws Exception { + final TagVFilter filter = new TagVLiteralOrFilter(TAGK_STRING, TAGV_STRING); + filter.resolveTagkName(tsdb).join(); + assertArrayEquals(TAGK_BYTES, filter.getTagkBytes()); + assertEquals(1, filter.getTagVUids().size()); + assertArrayEquals(TAGV_BYTES, filter.getTagVUids().get(0)); + } + + @Test + public void resolveNameLiterals() throws Exception { + final TagVFilter filter = new TagVLiteralOrFilter(TAGK_STRING, "web01|web02"); + filter.resolveTagkName(tsdb).join(); + assertArrayEquals(TAGK_BYTES, filter.getTagkBytes()); + assertEquals(2, filter.getTagVUids().size()); + assertArrayEquals(TAGV_BYTES, filter.getTagVUids().get(0)); + assertArrayEquals(TAGV_B_BYTES, filter.getTagVUids().get(1)); + } + + @Test (expected = DeferredGroupException.class) + public void resolveNameLiteralsNSUNTagV() throws Exception { + final TagVFilter filter = new TagVLiteralOrFilter(TAGK_STRING, "web01|web03"); + filter.resolveTagkName(tsdb).join(); + } + + @Test + public void resolveNameLiteralsNSUNTagvSkipped() throws Exception { + config.overrideConfig("tsd.query.skip_unresolved_tagvs", "true"); + final TagVFilter filter = new TagVLiteralOrFilter(TAGK_STRING, "web01|web03"); + filter.resolveTagkName(tsdb).join(); + assertArrayEquals(TAGK_BYTES, filter.getTagkBytes()); + assertEquals(1, filter.getTagVUids().size()); + assertArrayEquals(TAGV_BYTES, filter.getTagVUids().get(0)); + } + + @Test + public void resolveNameLiteralsTooMany() throws Exception { + config.overrideConfig("tsd.query.filter.expansion_limit", "1"); + final TagVFilter filter = new TagVLiteralOrFilter(TAGK_STRING, "web01|web02"); + filter.resolveTagkName(tsdb).join(); + assertArrayEquals(TAGK_BYTES, filter.getTagkBytes()); + assertTrue(filter.getTagVUids().isEmpty()); + } + + @Test + public void resolveNameLiteralsCaseInsensitive() throws Exception { + final TagVFilter filter = new TagVLiteralOrFilter(TAGK_STRING, "web01|web02", + true); + filter.resolveTagkName(tsdb).join(); + assertArrayEquals(TAGK_BYTES, filter.getTagkBytes()); + assertTrue(filter.getTagVUids().isEmpty()); + } + + @Test (expected = NoSuchUniqueName.class) + public void resolveNameNSUN() throws Exception { + final TagVFilter filter = new TagVWildcardFilter(NSUN_TAGK, "*omnia"); + filter.resolveTagkName(tsdb).join(); + } + + @Test (expected = NullPointerException.class) + public void resolveNameNullTSDB() throws Exception { + new TagVWildcardFilter("host", "*omnia").resolveTagkName(null); + } + + @Test + public void comparableTest() throws Exception { + final TagVFilter filter_a = new TagVWildcardFilter("host", "*omnia"); + Whitebox.setInternalState(filter_a, "tagk_bytes", new byte[] { 0, 0, 0, 1 }); + final TagVFilter filter_b = new TagVRegexFilter("dc", ".*katch"); + Whitebox.setInternalState(filter_b, "tagk_bytes", new byte[] { 0, 0, 0, 2 }); + + assertEquals(0, filter_a.compareTo(filter_a)); + assertEquals(-1, filter_a.compareTo(filter_b)); + assertEquals(1, filter_b.compareTo(filter_a)); + + Whitebox.setInternalState(filter_a, "tagk_bytes", (byte[])null); + assertEquals(0, filter_a.compareTo(filter_a)); + assertEquals(-1, filter_a.compareTo(filter_b)); + assertEquals(1, filter_b.compareTo(filter_a)); + + Whitebox.setInternalState(filter_b, "tagk_bytes", (byte[])null); + assertEquals(0, filter_a.compareTo(filter_a)); + assertEquals(0, filter_a.compareTo(filter_b)); + assertEquals(0, filter_b.compareTo(filter_a)); + + } + + @Test + public void stripParentheses() throws Exception { + assertEquals(".*sugarbean", TagVFilter.stripParentheses( + TagVRegexFilter.FILTER_NAME + "(.*sugarbean)")); + } + + @Test + public void stripParenthesesEmptyParentheses() throws Exception { + // let the filter's ctor handle this case + assertEquals("", TagVFilter.stripParentheses( + TagVRegexFilter.FILTER_NAME + "()")); + } + + @Test (expected = IllegalArgumentException.class) + public void stripParenthesesMissingClosing() throws Exception { + TagVFilter.stripParentheses(TagVRegexFilter.FILTER_NAME + "(.*sugarbean"); + } + + @Test (expected = IllegalArgumentException.class) + public void stripParenthesesMissingOpening() throws Exception { + TagVFilter.stripParentheses("regexp.*sugarbean)"); + } + + @Test (expected = IllegalArgumentException.class) + public void stripParenthesesNull() throws Exception { + TagVFilter.stripParentheses(null); + } + + @Test (expected = IllegalArgumentException.class) + public void stripParenthesesEmpty() throws Exception { + TagVFilter.stripParentheses(""); + } + + @SuppressWarnings("unchecked") + @Test + public void tagsToFiltersOldGroupBy() throws Exception { + final Map<String, String> tags = new HashMap<String, String>(3); + tags.put("host", "quirm"); // literal + tags.put("owner", "vimes|vetinary"); // pipe + tags.put("colo", "*"); // group by all + final List<TagVFilter> filters = new ArrayList<TagVFilter>(3); + TagVFilter.tagsToFilters(tags, filters); + + assertEquals(3, filters.size()); + for (final TagVFilter filter : filters) { + if (filter.getTagk().equals("host")) { + assertTrue(filter instanceof TagVLiteralOrFilter); + assertFalse(((TagVLiteralOrFilter)filter).isCaseInsensitive()); + assertEquals(1, ((Set<String>)Whitebox + .getInternalState(filter, "literals")).size()); + } else if (filter.getTagk().equals("owner")) { + assertTrue(filter instanceof TagVLiteralOrFilter); + assertFalse(((TagVLiteralOrFilter)filter).isCaseInsensitive()); + assertEquals(2, ((Set<String>)Whitebox + .getInternalState(filter, "literals")).size()); + } else if (filter.getTagk().equals("colo")) { + assertTrue(filter instanceof TagVWildcardFilter); + assertTrue(((TagVWildcardFilter)filter).isCaseInsensitive()); + } else { + fail("Unexpected filter type: " + filter); + } + assertTrue(filter.isGroupBy()); + } + } + + @Test + public void tagsToFiltersNewFunctions() throws Exception { + final Map<String, String> tags = new HashMap<String, String>(4); + tags.put("host", "*beybi"); + tags.put("owner", "wildcard(*snapcase*)"); + tags.put("colo", "regexp(.*opolis)"); + tags.put("geo", "literal_or(tsort|chalk)"); + final List<TagVFilter> filters = new ArrayList<TagVFilter>(3); + TagVFilter.tagsToFilters(tags, filters); + + assertEquals(4, filters.size()); + for (final TagVFilter filter : filters) { + if (filter.getTagk().equals("host")) { + assertTrue(filter instanceof TagVWildcardFilter); + assertTrue(((TagVWildcardFilter)filter).isCaseInsensitive()); + } else if (filter.getTagk().equals("owner")) { + assertTrue(filter instanceof TagVWildcardFilter); + assertFalse(((TagVWildcardFilter)filter).isCaseInsensitive()); + } else if (filter.getTagk().equals("colo")) { + assertTrue(filter instanceof TagVRegexFilter); + } else if (filter.getTagk().equals("geo")) { + assertTrue(filter instanceof TagVLiteralOrFilter); + assertFalse(((TagVLiteralOrFilter)filter).isCaseInsensitive()); + } else { + fail("Unexpected filter type: " + filter); + } + assertTrue(filter.isGroupBy()); + } + } + + @Test (expected = IllegalArgumentException.class) + public void tagsToFiltersNoSuchFunction() throws Exception { + final Map<String, String> tags = new HashMap<String, String>(1); + tags.put("host", "doesnotexist(*beybi)"); + final List<TagVFilter> filters = new ArrayList<TagVFilter>(1); + TagVFilter.tagsToFilters(tags, filters); + } + + @Test + public void tagsToFiltersDuplicate() throws Exception { + final Map<String, String> tags = new HashMap<String, String>(1); + tags.put("host", "*beybi"); + final List<TagVFilter> filters = new ArrayList<TagVFilter>(1); + filters.add(new TagVWildcardFilter("host", "*beybi", true)); + assertFalse(filters.get(0).isGroupBy()); + TagVFilter.tagsToFilters(tags, filters); + assertEquals(1, filters.size()); + assertTrue(filters.get(0).isGroupBy()); + } + + @Test + public void tagsToFiltersSameTagDiffValues() throws Exception { + final Map<String, String> tags = new HashMap<String, String>(1); + tags.put("host", "*beybi"); + final List<TagVFilter> filters = new ArrayList<TagVFilter>(1); + filters.add(new TagVWildcardFilter("host", "*helit", true)); + assertFalse(filters.get(0).isGroupBy()); + TagVFilter.tagsToFilters(tags, filters); + assertEquals(2, filters.size()); + } + + @Test + public void getCopy() { + final TagVFilter filter = TagVFilter.Builder() + .setFilter("*") + .setTagk(TAGK_STRING) + .setType("wildcard") + .setGroupBy(true) + .build(); + final TagVFilter copy = filter.getCopy(); + assertNotSame(filter, copy); + assertEquals(filter.filter, copy.filter); + assertEquals(filter.tagk, copy.tagk); + assertEquals(filter.getType(), copy.getType()); + assertEquals(filter.group_by, copy.group_by); + } + + // TODO - test the plugin loader similar to the other plugins +} diff --git a/test/query/filter/TestTagVLiteralOrFilter.java b/test/query/filter/TestTagVLiteralOrFilter.java new file mode 100644 index 0000000000..fbc14bfc23 --- /dev/null +++ b/test/query/filter/TestTagVLiteralOrFilter.java @@ -0,0 +1,181 @@ +package net.opentsdb.query.filter; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.Before; +import org.junit.Test; + +public class TestTagVLiteralOrFilter { + private static final String TAGK = "host"; + private Map<String, String> tags; + + @Before + public void before() throws Exception { + tags = new HashMap<String, String>(1); + tags.put(TAGK, "CMTDibbler"); + } + + @Test + public void matchMiddle() throws Exception { + TagVFilter filter = new TagVLiteralOrFilter(TAGK, + "LutZe|CMTDibbler|Slant"); + assertTrue(filter.match(tags).join()); + assertFalse(((TagVLiteralOrFilter)filter).isCaseInsensitive()); + } + + @Test + public void matchStart() throws Exception { + TagVFilter filter = new TagVLiteralOrFilter(TAGK, + "CMTDibbler|LutZe|Slant"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchEnd() throws Exception { + TagVFilter filter = new TagVLiteralOrFilter(TAGK, + "LutZe|Slant|CMTDibbler"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchNoPipes() throws Exception { + TagVFilter filter = new TagVLiteralOrFilter(TAGK, + "CMTDibbler"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchPipeNoValueAfter() throws Exception { + TagVFilter filter = new TagVLiteralOrFilter(TAGK, + "CMTDibbler|"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchPipeNoValueBefore() throws Exception { + TagVFilter filter = new TagVLiteralOrFilter(TAGK, + "|CMTDibbler"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchFail() throws Exception { + TagVFilter filter = new TagVLiteralOrFilter(TAGK, + "LutZe|Keli|Slant"); + assertFalse(filter.match(tags).join()); + } + + @Test + public void matchFailCase() throws Exception { + TagVFilter filter = new TagVLiteralOrFilter(TAGK, + "LutZe|CMtDibbler|Slant"); + assertFalse(filter.match(tags).join()); + } + + @Test + public void matchCaseInsensitive() throws Exception { + TagVFilter filter = new TagVLiteralOrFilter(TAGK, + "LutZe|CMtDibbler|Slant", true); + assertTrue(filter.match(tags).join()); + assertTrue(((TagVLiteralOrFilter)filter).isCaseInsensitive()); + } + + @Test + public void matchCaseInsensitiveValue() throws Exception { + tags.put(TAGK, "CMTDIBBLER"); + TagVFilter filter = new TagVLiteralOrFilter(TAGK, + "LutZe|CMtDibbler|Slant", true); + assertTrue(filter.match(tags).join()); + assertTrue(((TagVLiteralOrFilter)filter).isCaseInsensitive()); + } + + @Test + public void matchCaseInsensitiveFail() throws Exception { + TagVFilter filter = new TagVLiteralOrFilter(TAGK, + "LutZe|CMtDibble|Slant", true); + assertFalse(filter.match(tags).join()); + assertTrue(((TagVLiteralOrFilter)filter).isCaseInsensitive()); + } + + @Test + public void matchNoSuchTagk() throws Exception { + TagVFilter filter = new TagVLiteralOrFilter(TAGK, "LutZe|Keli|Slant"); + tags.clear(); + tags.put("colo", "lga"); + assertFalse(filter.match(tags).join()); + } + + @Test + public void matchNoSuchTagkCaseInsensitive() throws Exception { + TagVFilter filter = new TagVLiteralOrFilter(TAGK, "LutZe|Keli|Slant", true); + tags.clear(); + tags.put("colo", "lga"); + assertFalse(filter.match(tags).join()); + } + + @Test + public void matchSingle() throws Exception { + TagVFilter filter = new TagVLiteralOrFilter(TAGK, "CMTDibbler"); + assertTrue(filter.match(tags).join()); + assertFalse(((TagVLiteralOrFilter)filter).isCaseInsensitive()); + } + + @Test + public void matchSingleCaseInsensitive() throws Exception { + TagVFilter filter = new TagVLiteralOrFilter(TAGK, "cmtDibbler", true); + assertTrue(filter.match(tags).join()); + assertTrue(((TagVLiteralOrFilter)filter).isCaseInsensitive()); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNullTagk() throws Exception { + new TagVLiteralOrFilter(null, "LutZe|Keli|Slant"); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorEmptyTagk() throws Exception { + new TagVLiteralOrFilter("", "LutZe|Keli|Slant"); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNullFilter() throws Exception { + new TagVLiteralOrFilter(TAGK, null); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorEmptyFilter() throws Exception { + new TagVLiteralOrFilter(TAGK, ""); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorJustAPipe() throws Exception { + new TagVLiteralOrFilter(TAGK, "|"); + } + + @Test + public void toStringTest() throws Exception { + TagVFilter filter = new TagVLiteralOrFilter(TAGK, "LutZe|CMTDibbler|Slant"); + assertTrue(filter.toString().contains("literal_or")); + } + + @Test + public void hashCodeAndEqualsTest() throws Exception { + TagVFilter filter_a = new TagVLiteralOrFilter(TAGK, "LutZe|CMTDibbler|Slant"); + TagVFilter filter_b = new TagVLiteralOrFilter(TAGK, "LutZe|CMTDibbler|Slant"); + TagVFilter filter_c = new TagVLiteralOrFilter(TAGK, "LutZe|Slant"); + TagVFilter filter_d = new TagVLiteralOrFilter(TAGK, "LutZe|cmtdibbler|Slant"); + + assertEquals(filter_a.hashCode(), filter_b.hashCode()); + assertFalse(filter_a.hashCode() == filter_c.hashCode()); + assertFalse(filter_a.hashCode() == filter_d.hashCode()); + + assertEquals(filter_a, filter_b); + assertFalse(filter_a.equals(filter_c)); + assertFalse(filter_a.equals(filter_d)); + } +} diff --git a/test/query/filter/TestTagVNotKeyFilter.java b/test/query/filter/TestTagVNotKeyFilter.java new file mode 100644 index 0000000000..75d34670cd --- /dev/null +++ b/test/query/filter/TestTagVNotKeyFilter.java @@ -0,0 +1,47 @@ +package net.opentsdb.query.filter; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.Before; +import org.junit.Test; + +public class TestTagVNotKeyFilter { + private static final String TAGK = "host"; + private static final String TAGK2 = "owner"; + private Map<String, String> tags; + + @Before + public void before() throws Exception { + tags = new HashMap<String, String>(1); + tags.put(TAGK, "ogg-01.ops.ankh.morpork.com"); + tags.put(TAGK2, "Hrun"); + } + + @Test + public void matchHasKey() throws Exception { + TagVFilter filter = new TagVNotKeyFilter(TAGK, ""); + assertFalse(filter.match(tags).join()); + } + + @Test + public void matchDoesNotHaveKey() throws Exception { + TagVFilter filter = new TagVNotKeyFilter("colo", ""); + assertTrue(filter.match(tags).join()); + } + + @Test + public void ctorNullFilter() throws Exception { + TagVFilter filter = new TagVNotKeyFilter(TAGK, null); + assertTrue(filter.postScan()); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorFilterHasValue() throws Exception { + assertNotNull(new TagVNotKeyFilter(TAGK, "Evadne")); + } +} diff --git a/test/query/filter/TestTagVNotLiteralOrFilter.java b/test/query/filter/TestTagVNotLiteralOrFilter.java new file mode 100644 index 0000000000..1cb1c5ba29 --- /dev/null +++ b/test/query/filter/TestTagVNotLiteralOrFilter.java @@ -0,0 +1,179 @@ +package net.opentsdb.query.filter; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.Before; +import org.junit.Test; + +public class TestTagVNotLiteralOrFilter { + private static final String TAGK = "host"; + private Map<String, String> tags; + + @Before + public void before() throws Exception { + tags = new HashMap<String, String>(1); + tags.put(TAGK, "CMTDibbler"); + } + + @Test + public void matchMiddle() throws Exception { + TagVFilter filter = new TagVNotLiteralOrFilter(TAGK, + "LutZe|CMTDibbler|Slant"); + assertFalse(filter.match(tags).join()); + assertFalse(((TagVNotLiteralOrFilter)filter).isCaseInsensitive()); + } + + @Test + public void matchStart() throws Exception { + TagVFilter filter = new TagVNotLiteralOrFilter(TAGK, + "CMTDibbler|LutZe|Slant"); + assertFalse(filter.match(tags).join()); + } + + @Test + public void matchEnd() throws Exception { + TagVFilter filter = new TagVNotLiteralOrFilter(TAGK, + "LutZe|Slant|CMTDibbler"); + assertFalse(filter.match(tags).join()); + } + + @Test + public void matchNoPipes() throws Exception { + TagVFilter filter = new TagVNotLiteralOrFilter(TAGK, + "CMTDibbler"); + assertFalse(filter.match(tags).join()); + } + + @Test + public void matchPipeNoValueAfter() throws Exception { + TagVFilter filter = new TagVNotLiteralOrFilter(TAGK, + "CMTDibbler|"); + assertFalse(filter.match(tags).join()); + } + + @Test + public void matchPipeNoValueBefore() throws Exception { + TagVFilter filter = new TagVNotLiteralOrFilter(TAGK, + "|CMTDibbler"); + assertFalse(filter.match(tags).join()); + } + + @Test + public void matchFail() throws Exception { + TagVFilter filter = new TagVNotLiteralOrFilter(TAGK, + "LutZe|Keli|Slant"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchFailCase() throws Exception { + TagVFilter filter = new TagVNotLiteralOrFilter(TAGK, + "LutZe|CMtDibbler|Slant"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchCaseInsensitive() throws Exception { + TagVFilter filter = new TagVNotLiteralOrFilter(TAGK, + "LutZe|CMtDibbler|Slant", true); + assertFalse(filter.match(tags).join()); + assertTrue(((TagVNotLiteralOrFilter)filter).isCaseInsensitive()); + } + + @Test + public void matchCaseInsensitiveFail() throws Exception { + TagVFilter filter = new TagVNotLiteralOrFilter(TAGK, + "LutZe|CMtDibble|Slant", true); + assertTrue(filter.match(tags).join()); + assertTrue(((TagVNotLiteralOrFilter)filter).isCaseInsensitive()); + } + + @Test + public void matchNoSuchTagk() throws Exception { + TagVFilter filter = new TagVNotLiteralOrFilter(TAGK, "LutZe|Keli|Slant"); + tags.clear(); + tags.put("colo", "lga"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchNoSuchTagkCaseInsensitive() throws Exception { + TagVFilter filter = new TagVNotLiteralOrFilter(TAGK, "LutZe|Keli|Slant", true); + tags.clear(); + tags.put("colo", "lga"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchSingle() throws Exception { + TagVFilter filter = new TagVNotLiteralOrFilter(TAGK, "CMTDibbler"); + assertFalse(filter.match(tags).join()); + assertFalse(((TagVNotLiteralOrFilter)filter).isCaseInsensitive()); + } + + @Test + public void matchSingleCaseInsensitive() throws Exception { + TagVFilter filter = new TagVNotLiteralOrFilter(TAGK, "cmtDibbler", true); + assertFalse(filter.match(tags).join()); + assertTrue(((TagVNotLiteralOrFilter)filter).isCaseInsensitive()); + } + + @Test + public void matchSingleCharacterTagValue() throws Exception { + TagVFilter filter = new TagVNotLiteralOrFilter(TAGK, "c"); + tags.put(TAGK, "c"); + assertFalse(filter.match(tags).join()); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNullTagk() throws Exception { + new TagVNotLiteralOrFilter(null, "LutZe|Keli|Slant"); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorEmptyTagk() throws Exception { + new TagVNotLiteralOrFilter("", "LutZe|Keli|Slant"); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNullFilter() throws Exception { + new TagVNotLiteralOrFilter(TAGK, null); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorEmptyFilter() throws Exception { + new TagVNotLiteralOrFilter(TAGK, ""); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorJustAPipe() throws Exception { + new TagVNotLiteralOrFilter(TAGK, "|"); + } + + @Test + public void toStringTest() throws Exception { + TagVFilter filter = new TagVNotLiteralOrFilter(TAGK, "LutZe|CMTDibbler|Slant"); + assertTrue(filter.toString().contains("literal_or")); + } + + @Test + public void hashCodeAndEqualsTest() throws Exception { + TagVFilter filter_a = new TagVNotLiteralOrFilter(TAGK, "LutZe|CMTDibbler|Slant"); + TagVFilter filter_b = new TagVNotLiteralOrFilter(TAGK, "LutZe|CMTDibbler|Slant"); + TagVFilter filter_c = new TagVNotLiteralOrFilter(TAGK, "LutZe|Slant"); + TagVFilter filter_d = new TagVNotLiteralOrFilter(TAGK, "LutZe|cmtdibbler|Slant"); + + assertEquals(filter_a.hashCode(), filter_b.hashCode()); + assertFalse(filter_a.hashCode() == filter_c.hashCode()); + assertFalse(filter_a.hashCode() == filter_d.hashCode()); + + assertEquals(filter_a, filter_b); + assertFalse(filter_a.equals(filter_c)); + assertFalse(filter_a.equals(filter_d)); + } +} diff --git a/test/query/filter/TestTagVRegexFilter.java b/test/query/filter/TestTagVRegexFilter.java new file mode 100644 index 0000000000..8464eb032a --- /dev/null +++ b/test/query/filter/TestTagVRegexFilter.java @@ -0,0 +1,126 @@ +package net.opentsdb.query.filter; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.HashMap; +import java.util.Map; +import java.util.regex.PatternSyntaxException; + +import org.junit.Before; +import org.junit.Test; + +public class TestTagVRegexFilter { + private static final String TAGK = "host"; + private Map<String, String> tags; + + @Before + public void before() throws Exception { + tags = new HashMap<String, String>(1); + tags.put(TAGK, "ogg-01.ops.ankh.morpork.com"); + } + + @Test + public void matchExact() throws Exception { + TagVFilter filter = new TagVRegexFilter(TAGK, "ogg-01.ops.ankh.morpork.com"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchPostfix() throws Exception { + TagVFilter filter = new TagVRegexFilter(TAGK, ".*.ops.ankh.morpork.com"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchPrefix() throws Exception { + TagVFilter filter = new TagVRegexFilter(TAGK, "ogg-01.ops.ankh.*"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchAnything() throws Exception { + TagVFilter filter = new TagVRegexFilter(TAGK, ".*"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchFailed() throws Exception { + TagVFilter filter = new TagVRegexFilter(TAGK, "ogg-01.ops.qurim.*"); + assertFalse(filter.match(tags).join()); + } + + @Test + public void matchGrouping() throws Exception { + TagVFilter filter = new TagVRegexFilter(TAGK, + "ogg-01.ops.(ankh|quirm|tsort).morpork.com"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchNumbers() throws Exception { + TagVFilter filter = new TagVRegexFilter(TAGK, + "ogg-\\d+.ops.ankh.morpork.com"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchNotEnoughNumbers() throws Exception { + TagVFilter filter = new TagVRegexFilter(TAGK, + "ogg-\\d(3).ops.ankh.morpork.com"); + assertFalse(filter.match(tags).join()); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNullTagk() throws Exception { + new TagVRegexFilter(null, "ogg-01.ops.qurim.*"); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorEmptyTagk() throws Exception { + new TagVRegexFilter("", "ogg-01.ops.qurim.*"); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNullFilter() throws Exception { + new TagVRegexFilter(TAGK, null); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorEmptyFilter() throws Exception { + new TagVRegexFilter(TAGK, ""); + } + + @Test (expected = PatternSyntaxException.class) + public void ctorBadRegex() throws Exception { + new TagVRegexFilter(TAGK, "ogg-\\d(3.ops.ankh.morpork.com"); + } + + @Test + public void toStringTest() throws Exception { + TagVFilter filter = new TagVRegexFilter(TAGK, + "ogg-\\d+.ops.ankh.morpork.com"); + assertTrue(filter.toString().contains("regex")); + } + + @Test + public void hashCodeAndEqualsTest() throws Exception { + TagVFilter filter_a = new TagVRegexFilter(TAGK, + "ogg-\\d+.ops.ankh.morpork.com"); + TagVFilter filter_b = new TagVRegexFilter(TAGK, + "ogg-\\d+.ops.ankh.morpork.com"); + TagVFilter filter_c = new TagVRegexFilter(TAGK, + "ogg-\\d.ops.ankh.morpork.com"); + TagVFilter filter_d = new TagVRegexFilter(TAGK, + "ogg-\\d+.ops.ankh.morpork.co"); + + assertEquals(filter_a.hashCode(), filter_b.hashCode()); + assertFalse(filter_a.hashCode() == filter_c.hashCode()); + assertFalse(filter_a.hashCode() == filter_d.hashCode()); + + assertEquals(filter_a, filter_b); + assertFalse(filter_a.equals(filter_c)); + assertFalse(filter_a.equals(filter_d)); + } +} diff --git a/test/query/filter/TestTagVWildcardFilter.java b/test/query/filter/TestTagVWildcardFilter.java new file mode 100644 index 0000000000..2c1daf1c52 --- /dev/null +++ b/test/query/filter/TestTagVWildcardFilter.java @@ -0,0 +1,328 @@ +package net.opentsdb.query.filter; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.Before; +import org.junit.Test; + +public class TestTagVWildcardFilter { + private static final String TAGK = "host"; + private Map<String, String> tags; + + @Before + public void before() throws Exception { + tags = new HashMap<String, String>(1); + tags.put(TAGK, "ogg-01.ops.ankh.morpork.com"); + } + + @Test + public void matchAll() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, "*"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchAllNoSuchKey() throws Exception { + TagVFilter filter = new TagVWildcardFilter("hobbes", "*"); + assertFalse(filter.match(tags).join()); + } + + @Test + public void matchPostfix() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, "*.morpork.com"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchPrefix() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, + "ogg*"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchInfix() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, + "ogg*com"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchDoubleInfix() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, + "ogg*ops*ank*com"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchTripleInfix() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, + "ogg*ops*com"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchPreAndPostfix() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, + "*morpork*"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchPostAndInfix() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, + "*ops*com"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchPostAndDoubleInfix() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, + "*ops*mor*com"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchPreAndInfix() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, + "ogg*ops*"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchPreAndDoubleInfix() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, + "ogg*ops*mor*"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchMultiWildcardInfix() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, + "ogg***com"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchMultiWildcardPrefix() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, + "ogg*****"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchMultiWildcardPostfix() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, + "****com"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchWildcardsEverywhere() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, + "****ogg*****mor****com****"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchExactPostfix() throws Exception { + tags.put(TAGK, "*ops*mor"); + TagVFilter filter = new TagVWildcardFilter(TAGK, + "*ops*mor"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchExactPretfix() throws Exception { + tags.put(TAGK, "ogg*ops*"); + TagVFilter filter = new TagVWildcardFilter(TAGK, + "ogg*ops*"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchExactInfix() throws Exception { + tags.put(TAGK, "ogg*ops*mor"); + TagVFilter filter = new TagVWildcardFilter(TAGK, + "ogg*ops*mor"); + assertTrue(filter.match(tags).join()); + } + + // Make sure this file is encoded in UTF-8 of the following will fail + @Test + public void matchUTF8Postfix() throws Exception { + tags.put(TAGK, "Здравей'_хора"); + TagVFilter filter = new TagVWildcardFilter(TAGK, + "*хора"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchUTF8Prefix() throws Exception { + tags.put(TAGK, "Здравей'_хора"); + TagVFilter filter = new TagVWildcardFilter(TAGK, + "Здр*"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchUTF8Infix() throws Exception { + tags.put(TAGK, "Здравей'_хора"); + TagVFilter filter = new TagVWildcardFilter(TAGK, + "Здр*ра"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchPostfixFail() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, + "*.morpork.org"); + assertFalse(filter.match(tags).join()); + } + + @Test + public void matchPrefixFail() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, + "magrat*"); + assertFalse(filter.match(tags).join()); + } + + @Test + public void matchInfixFail() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, + "magrat*com"); + assertFalse(filter.match(tags).join()); + } + + @Test + public void matchPreAndPostfixFail() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, + "*quirm*"); + assertFalse(filter.match(tags).join()); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNullFilter() throws Exception { + new TagVWildcardFilter(TAGK, null); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorEmptyFilter() throws Exception { + new TagVWildcardFilter(TAGK, ""); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNoWildcard() throws Exception { + new TagVWildcardFilter(TAGK, "someliteral"); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNullTagk() throws Exception { + new TagVWildcardFilter(null, "*quirm*"); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorEmptyTagk() throws Exception { + new TagVWildcardFilter("", "*quirm*"); + } + + @Test + public void matchPostfixCaseFail() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, + "*.MorPork.com"); + assertFalse(filter.match(tags).join()); + } + + @Test + public void matchPrefixCaseFail() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, + "Ogg*"); + assertFalse(filter.match(tags).join()); + } + + @Test + public void matchInfixCaseFail() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, + "ogG*Com"); + assertFalse(filter.match(tags).join()); + } + + @Test + public void matchPostfixCaseInsensitive() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, + "*.MorPork.com", true); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchPostfixCaseInsensitiveValue() throws Exception { + tags.put(TAGK, "ogg-01.ops.ankh.MORPORK.com"); + TagVFilter filter = new TagVWildcardFilter(TAGK, + "*.MorPork.com", true); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchPrefixCaseInsensitive() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, + "Ogg*", true); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchInfixCaseInsensitive() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, + "ogG*Com", true); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchNothingButStars() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, "****"); + assertTrue(filter.match(tags).join()); + } + + @Test + public void matchNoSuchTagk() throws Exception { + final TagVFilter filter = new TagVWildcardFilter(TAGK, "*.morpork.com"); + tags.remove("host"); + tags.put("colo", "lga"); + assertFalse(filter.match(tags).join()); + } + + @Test + public void matchNoSuchTagkCaseInsensitive() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, "*.morpork.com", true); + tags.remove("host"); + tags.put("colo", "lga"); + assertFalse(filter.match(tags).join()); + } + + @Test + public void toStringTest() throws Exception { + TagVFilter filter = new TagVWildcardFilter(TAGK, "ogg*com"); + assertTrue(filter.toString().contains("wild")); + } + + @Test + public void hashCodeAndEqualsTest() throws Exception { + TagVFilter filter_a = new TagVWildcardFilter(TAGK, "ogg*com"); + TagVFilter filter_b = new TagVWildcardFilter(TAGK, "ogg*com"); + TagVFilter filter_c = new TagVWildcardFilter(TAGK, "*com"); + TagVFilter filter_d = new TagVWildcardFilter(TAGK, "Ogg*com"); + + assertEquals(filter_a.hashCode(), filter_b.hashCode()); + assertFalse(filter_a.hashCode() == filter_c.hashCode()); + assertFalse(filter_a.hashCode() == filter_d.hashCode()); + + assertEquals(filter_a, filter_b); + assertFalse(filter_a.equals(filter_c)); + assertFalse(filter_a.equals(filter_d)); + } + +} diff --git a/test/query/pojo/TestDownsampler.java b/test/query/pojo/TestDownsampler.java new file mode 100644 index 0000000000..9613999238 --- /dev/null +++ b/test/query/pojo/TestDownsampler.java @@ -0,0 +1,104 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.pojo; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import net.opentsdb.core.FillPolicy; +import net.opentsdb.query.expression.NumericFillPolicy; +import net.opentsdb.utils.JSON; + +import org.junit.Test; + +public class TestDownsampler { + + @Test(expected = IllegalArgumentException.class) + public void validationErrorWhenIntervalIsNull() throws Exception { + String json = "{\"aggregator\":\"sum\"}"; + Downsampler downsampler = JSON.parseToObject(json, Downsampler.class); + downsampler.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void validationErrorWhenIntervalIsEmpty() throws Exception { + String json = "{\"interval\":\"\",\"aggregator\":\"sum\"}"; + Downsampler downsampler = JSON.parseToObject(json, Downsampler.class); + downsampler.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void validationErrorWhenIntervalIsInvalid() throws Exception { + String json = "{\"interval\":\"45foo\",\"aggregator\":\"sum\"}"; + Downsampler downsampler = JSON.parseToObject(json, Downsampler.class); + downsampler.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void validationErrorWhenAggregatorIsNull() throws Exception { + String json = "{\"interval\":\"1h\"}"; + Downsampler downsampler = JSON.parseToObject(json, Downsampler.class); + downsampler.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void validationErrorWhenAggregatorIsEmpty() throws Exception { + String json = "{\"interval\":\"1h\",\"aggregator\":\"\"}"; + Downsampler downsampler = JSON.parseToObject(json, Downsampler.class); + downsampler.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void validationErrorWhenAggregatorIsInvalid() throws Exception { + String json = "{\"interval\":\"1h\",\"aggregator\":\"no such agg\"}"; + Downsampler downsampler = JSON.parseToObject(json, Downsampler.class); + downsampler.validate(); + } + + @Test + public void deserialize() throws Exception { + String json = "{\"interval\":\"1h\",\"aggregator\":\"zimsum\"}"; + Downsampler downsampler = JSON.parseToObject(json, Downsampler.class); + downsampler.validate(); + Downsampler expected = Downsampler.Builder() + .setInterval("1h").setAggregator("zimsum").build(); + assertEquals(expected, downsampler); + + json = "{\"interval\":\"1h\",\"aggregator\":\"zimsum\"," + + "\"fillPolicy\":{\"policy\":\"nan\"},\"junkfield\":true}"; + downsampler = JSON.parseToObject(json, Downsampler.class); + downsampler.validate(); + expected = Downsampler.Builder() + .setInterval("1h").setAggregator("zimsum") + .setFillPolicy(new NumericFillPolicy(FillPolicy.NOT_A_NUMBER)).build(); + assertEquals(expected, downsampler); + } + + @Test + public void serialize() throws Exception { + Downsampler downsampler = Downsampler.Builder() + .setInterval("1h").setAggregator("zimsum").build(); + String json = JSON.serializeToString(downsampler); + assertTrue(json.contains("\"interval\":\"1h\"")); + assertTrue(json.contains("\"aggregator\":\"zimsum\"")); + assertTrue(json.contains("\"fillPolicy\":null")); + + downsampler = Downsampler.Builder() + .setInterval("15m").setAggregator("max") + .setFillPolicy(new NumericFillPolicy(FillPolicy.NOT_A_NUMBER)).build(); + json = JSON.serializeToString(downsampler); + assertTrue(json.contains("\"interval\":\"15m\"")); + assertTrue(json.contains("\"aggregator\":\"max\"")); + assertTrue(json.contains("\"fillPolicy\":{")); + assertTrue(json.contains("\"policy\":\"nan\"")); + } +} diff --git a/test/query/pojo/TestExpression.java b/test/query/pojo/TestExpression.java new file mode 100644 index 0000000000..6d1f5afd4e --- /dev/null +++ b/test/query/pojo/TestExpression.java @@ -0,0 +1,96 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.pojo; + +import net.opentsdb.query.expression.VariableIterator.SetOperator; +import net.opentsdb.utils.JSON; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestExpression { + @Test(expected = IllegalArgumentException.class) + public void validationErrorWhenIdIsNull() throws Exception { + String json = "{\"expr\":\"a + b + c\"}"; + Expression expression = JSON.parseToObject(json, Expression.class); + expression.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void validationErrorWhenIdIsEmpty() throws Exception { + String json = "{\"expr\":\"a + b + c\",\"id\":\"\"}"; + Expression expression = JSON.parseToObject(json, Expression.class); + expression.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void validationErrorWhenIdIsInvalid() throws Exception { + String json = "{\"expr\":\"a + b + c\",\"id\":\"system.busy\"}"; + Expression expression = JSON.parseToObject(json, Expression.class); + expression.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void validationErrorWhenExprIsNull() throws Exception { + String json = "{\"id\":\"1\"}"; + Expression expression = JSON.parseToObject(json, Expression.class); + expression.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void validationErrorWhenExprIsEmpty() throws Exception { + String json = "{\"id\":\"1\",\"expr\":\"\"}"; + Expression expression = JSON.parseToObject(json, Expression.class); + expression.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void validationErrorWhenJoinIsInvalid() throws Exception { + String json = "{\"expr\":\"a + b + c\",\"id\":\"system.busy\"," + + "\"join\":{\"operator\":\"nosuchjoin\"}}"; + Expression expression = JSON.parseToObject(json, Expression.class); + expression.validate(); + } + + @Test + public void deserialize() throws Exception { + String json = "{\"id\":\"e\",\"expr\":\"a + b + c\"}"; + Expression expression = JSON.parseToObject(json, Expression.class); + expression.validate(); + Expression expected = Expression.Builder().setId("e") + .setExpression("a + b + c").setJoin( + Join.Builder().setOperator(SetOperator.UNION).build()).build(); + assertEquals(expected, expression); + } + + @Test + public void serialize() throws Exception { + Expression expression = Expression.Builder().setId("e1") + .setJoin(Join.Builder().setOperator(SetOperator.UNION).build()) + .setExpression("a + b + c").build(); + String actual = JSON.serializeToString(expression); + assertTrue(actual.contains("\"id\":\"e1\"")); + assertTrue(actual.contains("\"expr\":\"a + b + c\"")); + assertTrue(actual.contains("\"join\":{\"operator\":\"union\"")); + + } + + @Test + public void unknownShouldBeIgnored() throws Exception { + String json = "{\"id\":\"1\",\"expr\":\"a + b + c\",\"unknown\":\"yo\"}"; + JSON.parseToObject(json, Expression.class); + // pass if no unexpected exception + } +} diff --git a/test/query/pojo/TestFilter.java b/test/query/pojo/TestFilter.java new file mode 100644 index 0000000000..1d1f9ea0a5 --- /dev/null +++ b/test/query/pojo/TestFilter.java @@ -0,0 +1,99 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.pojo; + +import net.opentsdb.query.filter.TagVFilter; +import net.opentsdb.utils.JSON; + +import org.junit.Test; + +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestFilter { + + @Test(expected = IllegalArgumentException.class) + public void validationErrorWhenIdIsNull() throws Exception { + String json = "{\"id\":null}"; + Filter filter = JSON.parseToObject(json, Filter.class); + filter.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void validationBadId() throws Exception { + String json = "{\"id\":\"bad.Id\",\"tags\":[]}"; + Filter filter = JSON.parseToObject(json, Filter.class); + filter.validate(); + } + + @Test + public void deserialize() throws Exception { + String json = "{\"id\":\"f1\",\"tags\":[{\"tagk\":\"host\"," + + "\"filter\":\"*\",\"type\":\"iwildcard\",\"groupBy\":false}]," + + "\"explicitTags\":\"true\"}"; + + TagVFilter tag = new TagVFilter.Builder().setFilter("*").setGroupBy( + false) + .setTagk("host").setType("iwildcard").build(); + + Filter expectedFilter = Filter.Builder().setId("f1") + .setTags(Arrays.asList(tag)).setExplicitTags(true).build(); + + Filter filter = JSON.parseToObject(json, Filter.class); + filter.validate(); + assertEquals(expectedFilter, filter); + } + + @Test + public void serialize() throws Exception { + TagVFilter tag = new TagVFilter.Builder().setFilter("*").setGroupBy(false) + .setTagk("host").setType("iwildcard").build(); + + Filter filter = Filter.Builder().setId("f1") + .setTags(Arrays.asList(tag)).setExplicitTags(true).build(); + + String actual = JSON.serializeToString(filter); + assertTrue(actual.contains("\"id\":\"f1\"")); + assertTrue(actual.contains("\"tags\":[")); + assertTrue(actual.contains("\"tagk\":\"host\"")); + assertTrue(actual.contains("\"explicitTags\":true")); + } + + @Test + public void unknownShouldBeIgnored() throws Exception { + String json = "{\"id\":\"1\",\"unknown\":\"yo\"}"; + JSON.parseToObject(json, Filter.class); + // pass if no unexpected exception + } + + @Test(expected = IllegalArgumentException.class) + public void invalidTags() throws Exception { + String json = "{\"id\":\"1\",\"tags\":[{\"tagk\":\"\"," + + "\"filter\":\"*\",\"type\":\"iwildcard\",\"group_by\":false}]," + + "\"aggregation\":{\"tags\":[\"appid\"],\"aggregator\":\"sum\"}}"; + + Filter filter = JSON.parseToObject(json, Filter.class); + filter.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void invalidAggregation() throws Exception { + String json = "{\"id\":\"1\",\"tags\":[{\"tagk\":\"\"," + + "\"filter\":\"*\",\"type\":\"iwildcard\",\"group_by\":false}]," + + "\"aggregator\":\"what\"}"; + Filter filter = JSON.parseToObject(json, Filter.class); + filter.validate(); + } +} diff --git a/test/query/pojo/TestJoin.java b/test/query/pojo/TestJoin.java new file mode 100644 index 0000000000..8b27a5d3ec --- /dev/null +++ b/test/query/pojo/TestJoin.java @@ -0,0 +1,66 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.pojo; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import net.opentsdb.query.expression.VariableIterator.SetOperator; +import net.opentsdb.utils.JSON; + +import org.junit.Test; + +public class TestJoin { + + @Test + public void deserialize() throws Exception { + final String json = "{\"operator\":\"union\"}"; + final Join join = Join.Builder().setOperator(SetOperator.UNION).build(); + final Join deserialized = JSON.parseToObject(json, Join.class); + assertEquals(join, deserialized); + } + + @Test + public void serialize() throws Exception { + final Join join = Join.Builder().setOperator(SetOperator.UNION).build(); + final String json = JSON.serializeToString(join); + assertTrue(json.contains("\"operator\":\"union\"")); + } + + @Test(expected = IllegalArgumentException.class) + public void validationErrorWhenOperatorIsNull() throws Exception { + final String json = "{\"operator\":null}"; + final Join join = JSON.parseToObject(json, Join.class); + join.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void validationErrorWhenOperatorIsEmpty() throws Exception { + final String json = "{\"operator\":\"\"}"; + final Join join = JSON.parseToObject(json, Join.class); + join.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void validationErrorWhenOperatorIsInvalid() throws Exception { + final String json = "{\"operator\":\"nosuchop\"}"; + final Join join = JSON.parseToObject(json, Join.class); + join.validate(); + } + + @Test + public void unknownShouldBeIgnored() throws Exception { + String json = "{\"operator\":\"intersection\",\"unknown\":\"yo\"}"; + JSON.parseToObject(json, Filter.class); + // pass if no unexpected exception + } +} diff --git a/test/query/pojo/TestMetric.java b/test/query/pojo/TestMetric.java new file mode 100644 index 0000000000..c312fc1b97 --- /dev/null +++ b/test/query/pojo/TestMetric.java @@ -0,0 +1,123 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.pojo; + +import net.opentsdb.core.FillPolicy; +import net.opentsdb.query.expression.NumericFillPolicy; +import net.opentsdb.utils.JSON; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestMetric { + @Test(expected = IllegalArgumentException.class) + public void validationErrorWhenMetricIsNull() throws Exception { + String json = "{\"id\":\"1\",\"filter\":\"2\"," + + "\"timeOffset\":\"1h-ago\",\"aggregator\":\"sum\"," + + "\"fillPolicy\":{\"policy\":\"nan\"}}"; + Metric metric = JSON.parseToObject(json, Metric.class); + metric.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void validationErrorWhenMetricIsEmpty() throws Exception { + String json = "{\"metric\":\"\",\"id\":\"1\",\"filter\":\"2\"," + + "\"timeOffset\":\"1h-ago\",\"aggregator\":\"sum\"," + + "\"fillPolicy\":{\"policy\":\"nan\"}}"; + Metric metric = JSON.parseToObject(json, Metric.class); + metric.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void validationErrorWhenIDIsNull() throws Exception { + String json = "{\"metric\":\"system.cpu\",\"id\":null,\"filter\":\"2\"," + + "\"timeOffset\":\"1h-ago\",\"aggregator\":\"sum\"," + + "\"fillPolicy\":{\"policy\":\"nan\"}}"; + Metric metric = JSON.parseToObject(json, Metric.class); + metric.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void validationErrorWhenIDIsEmpty() throws Exception { + String json = "{\"metric\":\"system.cpu\",\"id\":\"\",\"filter\":\"2\"," + + "\"timeOffset\":\"1h-ago\",\"aggregator\":\"sum\"," + + "\"fillPolicy\":{\"policy\":\"nan\"}}"; + Metric metric = JSON.parseToObject(json, Metric.class); + metric.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void validationErrorWhenIDIsInvalid() throws Exception { + String json = "{\"metric\":\"system.cpu\",\"id\":\"system.cpu\",\"filter\":\"2\"," + + "\"timeOffset\":\"1h-ago\",\"aggregator\":\"sum\"," + + "\"fillPolicy\":{\"policy\":\"nan\"}}"; + Metric metric = JSON.parseToObject(json, Metric.class); + metric.validate(); + } + + @Test + public void deserializeAllFields() throws Exception { + String json = "{\"metric\":\"YAMAS.cpu.idle\",\"id\":\"e1\",\"filter\":\"f2\"," + + "\"timeOffset\":\"1h-ago\",\"aggregator\":\"sum\"," + + "\"fillPolicy\":{\"policy\":\"nan\"}}"; + Metric metric = JSON.parseToObject(json, Metric.class); + metric.validate(); + Metric expectedMetric = Metric.Builder().setMetric("YAMAS.cpu.idle") + .setId("e1").setFilter("f2").setTimeOffset("1h-ago") + .setAggregator("sum") + .setFillPolicy(new NumericFillPolicy(FillPolicy.NOT_A_NUMBER)) + .build(); + + assertEquals(expectedMetric, metric); + } + + @Test + public void serialize() throws Exception { + Metric metric = Metric.Builder().setMetric("YAMAS.cpu.idle") + .setId("e1").setFilter("f2").setTimeOffset("1h-ago") + .setFillPolicy(new NumericFillPolicy(FillPolicy.NOT_A_NUMBER)) + .build(); + + String actual = JSON.serializeToString(metric); + assertTrue(actual.contains("\"metric\":\"YAMAS.cpu.idle\"")); + assertTrue(actual.contains("\"id\":\"e1\"")); + assertTrue(actual.contains("\"filter\":\"f2\"")); + assertTrue(actual.contains("\"timeOffset\":\"1h-ago\"")); + assertTrue(actual.contains("\"fillPolicy\":{")); + } + + @Test + public void unknownShouldBeIgnored() throws Exception { + String json = "{\"aggregator\":\"sum\",\"tags\":[\"foo\",\"bar\"],\"unknown\":\"garbage\"}"; + JSON.parseToObject(json, Metric.class); + // pass if no unexpected exception + } + + @Test(expected = IllegalArgumentException.class) + public void validationtErrorWhenTimeOffsetIsInvalid() throws Exception { + String json = "{\"metric\":\"YAMAS.cpu.idle\",\"id\":\"1\",\"filter\":\"2\"," + + "\"timeOffset\":\"what?\"}"; + Metric metric = JSON.parseToObject(json, Metric.class); + metric.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void validationtErrorBadFill() throws Exception { + String json = "{\"metric\":\"YAMAS.cpu.idle\",\"id\":\"1\",\"filter\":\"2\"," + + "\"fillPolicy\":{\"policy\":\"zero\",\"value\":42}}"; + Metric metric = JSON.parseToObject(json, Metric.class); + metric.validate(); + } +} diff --git a/test/query/pojo/TestOutput.java b/test/query/pojo/TestOutput.java new file mode 100644 index 0000000000..b4084e4b3e --- /dev/null +++ b/test/query/pojo/TestOutput.java @@ -0,0 +1,45 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.pojo; + +import net.opentsdb.utils.JSON; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class TestOutput { + @Test + public void deserializeAllFields() throws Exception { + String json = "{\"id\":\"m1\",\"alias\":\"CPU OK\"}"; + Output output = JSON.parseToObject(json, Output.class); + Output expectedOutput = Output.Builder().setId("m1").setAlias("CPU OK") + .build(); + assertEquals(expectedOutput, output); + } + + @Test + public void serialize() throws Exception { + Output output = Output.Builder().setId("m1").setAlias("CPU OK") + .build(); + String actual = JSON.serializeToString(output); + String expected = "{\"id\":\"m1\",\"alias\":\"CPU OK\"}"; + assertEquals(expected, actual); + } + + @Test + public void unknownFieldShouldBeIgnored() throws Exception { + String json = "{\"id\":\"m1\",\"unknown\":\"yo\"}"; + JSON.parseToObject(json, Filter.class); + // pass if no unexpected exception + } +} diff --git a/test/query/pojo/TestQuery.java b/test/query/pojo/TestQuery.java new file mode 100644 index 0000000000..fb00681a0a --- /dev/null +++ b/test/query/pojo/TestQuery.java @@ -0,0 +1,235 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.pojo; + +import net.opentsdb.core.FillPolicy; +import net.opentsdb.query.expression.NumericFillPolicy; +import net.opentsdb.query.expression.VariableIterator.SetOperator; +import net.opentsdb.query.filter.TagVFilter; +import net.opentsdb.utils.JSON; + +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestQuery { + Timespan time; + TagVFilter tag; + Filter filter; + Metric metric; + Expression expression; + Output output; + + String json = "{" + + " \"time\":{" + + " \"start\":\"3h-ago\"," + + " \"end\":\"1h-ago\"," + + " \"timezone\":\"UTC\"," + + " \"aggregator\":\"avg\"," + + " \"downsampler\":{\"interval\":\"15m\"," + + " \"aggregator\":\"avg\"," + + " \"fillPolicy\":{\"policy\":\"nan\"}}" + + " }," + + " \"filters\":[" + + " {" + + " \"id\":\"f1\"," + + " \"tags\":[" + + " {" + + " \"tagk\":\"host\"," + + " \"filter\":\"*\"," + + " \"type\":\"iwildcard\"," + + " \"groupBy\":false" + + " }" + + " ]" + + " }" + + " ]," + + " \"metrics\":[" + + " {" + + " \"metric\":\"YAMAS.cpu.idle\"," + + " \"id\":\"m1\"," + + " \"filter\":\"f1\"," + + " \"aggregator\":\"sum\"," + + " \"timeOffset\":\"0\"" + + " }" + + " ]," + + " \"expressions\":[" + + " {" + + " \"id\":\"e1\"," + + " \"expr\":\"m1 * 1024\"" + + " }" + + " ]," + + " \"outputs\":[" + + " {" + + " \"id\":\"m1\"," + + " \"alias\":\"CPU Idle EAST DC\"" + + " }" + + " ]" + + "}"; + + @Before + public void setup() { + time = Timespan.Builder().setStart("3h-ago").setAggregator("avg") + .setEnd("1h-ago").setTimezone("UTC").setDownsampler( + Downsampler.Builder().setInterval("15m").setAggregator("avg") + .setFillPolicy(new NumericFillPolicy(FillPolicy.NOT_A_NUMBER)).build()) + .build(); + TagVFilter tag = new TagVFilter.Builder().setFilter("*").setGroupBy( + false) + .setTagk("host").setType("iwildcard").build(); + filter = Filter.Builder().setId("f1").setTags(Arrays.asList(tag)).build(); + metric = Metric.Builder().setMetric("YAMAS.cpu.idle") + .setId("m1").setFilter("f1").setTimeOffset("0") + .setAggregator("sum").build(); + expression = Expression.Builder().setId("e1") + .setExpression("m1 * 1024").setJoin( + Join.Builder().setOperator(SetOperator.UNION).build()).build(); + output = Output.Builder().setId("m1").setAlias("CPU Idle EAST DC") + .build(); + } + + @Test(expected = IllegalArgumentException.class) + public void validationErrorWhenTimeIsNull() throws Exception { + Query query = getDefaultQueryBuilder().setTime(null).build(); + query.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void invalidTime() throws Exception { + Timespan invalidTime = Timespan.Builder().build(); + Query query = getDefaultQueryBuilder().setTime(invalidTime).build(); + query.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void metricsIsNull() throws Exception { + Query query = getDefaultQueryBuilder().setMetrics(null).build(); + query.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void metricsIsEmpty() throws Exception { + Query query = getDefaultQueryBuilder().setMetrics( + Collections.<Metric>emptyList()).build(); + query.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void invalidMetric() throws Exception { + Metric invalidMetric = Metric.Builder().build(); + Query query = getDefaultQueryBuilder() + .setMetrics(Arrays.asList(invalidMetric)).build(); + query.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void invalidFilter() throws Exception { + Filter invalidFilter = Filter.Builder().build(); + Query query = getDefaultQueryBuilder() + .setFilters(Arrays.asList(invalidFilter)).build(); + query.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void invalidExpression() throws Exception { + Expression invalidExpression = Expression.Builder().build(); + Query query = getDefaultQueryBuilder() + .setExpressions(Arrays.asList(invalidExpression)).build(); + query.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void noSuchFilterIdInMetric() throws Exception { + Metric invalid_metric = Metric.Builder().setMetric("YAMAS.cpu.idle") + .setId("m2").setFilter("f2").setTimeOffset("0").build(); + Query query = getDefaultQueryBuilder().setMetrics( + Arrays.asList(invalid_metric, metric)).build(); + query.validate(); + } + + @Test + public void deserialize() throws Exception { + Query query = JSON.parseToObject(json, Query.class); + query.validate(); + Query expected = Query.Builder().setExpressions(Arrays.asList(expression)) + .setFilters(Arrays.asList(filter)).setMetrics(Arrays.asList(metric)) + .setTime(time).setOutputs(Arrays.asList(output)).build(); + assertEquals(expected, query); + } + + @Test(expected = IllegalArgumentException.class) + public void duplicatedFilterId() throws Exception { + Query query = getDefaultQueryBuilder().setFilters( + Arrays.asList(filter, filter)).build(); + query.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void duplicatedExpressionId() throws Exception { + Query query = getDefaultQueryBuilder().setExpressions( + Arrays.asList(expression, expression)).build(); + query.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void duplicatedMetricId() throws Exception { + Query query = getDefaultQueryBuilder().setMetrics( + Arrays.asList(metric, metric)).build(); + query.validate(); + } + + @Test + public void serialize() throws Exception { + Query query = Query.Builder().setExpressions(Arrays.asList(expression)) + .setFilters(Arrays.asList(filter)).setMetrics(Arrays.asList(metric)) + .setName("q1").setTime(time).setOutputs(Arrays.asList(output)).build(); + + String actual = JSON.serializeToString(query); +// String expected = "{\"name\":\"q1\",\"time\":{\"start\":\"3h-ago\"," +// + "\"end\":\"1h-ago\",\"timezone\":\"UTC\",\"downsample\":\"15m-avg-nan\"," +// + "\"interpolation\":\"LERP\"},\"filters\":[{\"id\":\"f1\"," +// + "\"tags\":[{\"tagk\":\"host\",\"filter\":\"*\",\"group_by\":false," +// + "\"type\":\"iwildcard\"}],\"aggregator\":\"sum\"}]," +// + "\"metrics\":[{\"metric\":\"YAMAS.cpu.idle\"," +// + "\"id\":\"m1\",\"filter\":\"f1\",\"time_offset\":\"0\"}]," +// + "\"expressions\":[{\"id\":\"e1\",\"expr\":\"a + b + c\"}]," +// + "\"outputs\":[{\"var\":\"q1.m1\",\"alias\":\"CPU Idle EAST DC\"}]}"; + assertTrue(actual.contains("\"name\":\"q1\"")); + assertTrue(actual.contains("\"start\":\"3h-ago\"")); + assertTrue(actual.contains("\"end\":\"1h-ago\"")); + assertTrue(actual.contains("\"timezone\":\"UTC\"")); + // TODO - finish the assertions + } + + @Test + public void justMetrics() throws Exception { + Query query = Query.Builder() + .setMetrics(Arrays.asList(metric)) + .setName("q1") + .setTime(time) + .build(); + query.validate(); + assertEquals(metric.getMetric(), query.getMetrics().get(0).getMetric()); + } + + + private Query.Builder getDefaultQueryBuilder() { + return Query.Builder().setExpressions(Arrays.asList(expression)) + .setFilters(Arrays.asList(filter)).setMetrics(Arrays.asList(metric)) + .setName("q1").setTime(time).setOutputs(Arrays.asList(output)); + } +} diff --git a/test/query/pojo/TestTimeSpan.java b/test/query/pojo/TestTimeSpan.java new file mode 100644 index 0000000000..ecd1450d72 --- /dev/null +++ b/test/query/pojo/TestTimeSpan.java @@ -0,0 +1,137 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.query.pojo; + +import net.opentsdb.core.FillPolicy; +import net.opentsdb.query.expression.NumericFillPolicy; +import net.opentsdb.utils.JSON; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestTimeSpan { + @Test(expected = IllegalArgumentException.class) + public void startIsNull() { + String json = "{\"start\":null,\"end\":\"2015/05/05\"," + + "\"timezone\":\"UTC\",\"downsample\":\"15m-avg-nan\"," + + ",\"aggregator\":\"sum\"}"; + Timespan timespan = JSON.parseToObject(json, Timespan.class); + timespan.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void startIsEmpty() { + String json = "{\"start\":\"\",\"end\":\"2015/05/05\"," + + "\"timezone\":\"UTC\",\"downsample\":\"15m-avg-nan\"" + + ",\"aggregator\":\"sum\"}"; + Timespan timespan = JSON.parseToObject(json, Timespan.class); + timespan.validate(); + } + + @Test + public void endIsNull() { + String json = "{\"start\":\"2015/05/05\",\"end\":null," + + "\"timezone\":\"UTC\",\"downsample\":\"15m-avg-nan\"" + + ",\"aggregator\":\"sum\"}"; + Timespan timespan = JSON.parseToObject(json, Timespan.class); + timespan.validate(); + } + + @Test + public void endIsEmpty() { + String json = "{\"start\":\"1h-ago\",\"end\":\"\"," + + "\"timezone\":\"UTC\",\"downsample\":\"15m-avg-nan\"" + + ",\"aggregator\":\"sum\"}"; + Timespan timespan = JSON.parseToObject(json, Timespan.class); + timespan.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void aggregatorIsNull() { + String json = "{\"start\":\"1h-ago\",\"end\":\"2015/05/05\"," + + "\"timezone\":\"UTC\",\"downsample\":\"15m-avg-nan\"," + + "}"; + Timespan timespan = JSON.parseToObject(json, Timespan.class); + timespan.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void aggregatorIsEmpty() { + String json = "{\"start\":\"1h-ago\",\"end\":\"2015/05/05\"," + + "\"timezone\":\"UTC\",\"downsample\":\"15m-avg-nan\"" + + ",\"aggregator\":\"\"}"; + Timespan timespan = JSON.parseToObject(json, Timespan.class); + timespan.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void idIsNull() { + String json = "{\"start\":\"-1h\",\"end\":\"2015/05/05\"," + + "\"timezone\":\"UTC\",\"downsample\":\"15m-avg-nan\"" + + ",\"interpolation\":\"LERP\"}"; + Timespan timespan = JSON.parseToObject(json, Timespan.class); + timespan.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void idIsEmpty() { + String json = "{\"start\":\"-1h\",\"end\":\"2015/05/05\"," + + "\"timezone\":\"UTC\",\"downsample\":\"15m-avg-nan\"" + + ",\"interpolation\":\"LERP\"}"; + Timespan timespan = JSON.parseToObject(json, Timespan.class); + timespan.validate(); + } + + @Test(expected = IllegalArgumentException.class) + public void invalidDownsample() { + String json = "{\"start\":\"1h-ago\",\"end\":\"2015/05/05\",\"timezone\":\"UTC\"," + + "\"downsampler\":\"xxx\"}"; + Timespan timespan = JSON.parseToObject(json, Timespan.class); + timespan.validate(); + } + + @Test + public void deserialize() { + String json = "{\"start\":\"1h-ago\",\"end\":\"2015/05/05\",\"timezone\":\"UTC\"," + + "\"downsampler\":{\"interval\":\"15m\",\"aggregator\":\"avg\"," + + "\"fillPolicy\":{\"policy\":\"nan\"}},\"aggregator\":\"sum\"," + + "\"unknownfield\":\"boo\"}"; + Timespan timespan = JSON.parseToObject(json, Timespan.class); + Timespan expected = Timespan.Builder().setStart("1h-ago") + .setEnd("2015/05/05").setTimezone("UTC").setAggregator("sum") + .setDownsampler( + Downsampler.Builder().setInterval("15m").setAggregator("avg") + .setFillPolicy(new NumericFillPolicy(FillPolicy.NOT_A_NUMBER)).build()) + .build(); + timespan.validate(); + assertEquals(expected, timespan); + } + + @Test + public void serialize() { + Timespan timespan = Timespan.Builder().setStart("1h-ago") + .setEnd("2015/05/05").setTimezone("UTC").setAggregator("sum").setDownsampler( + Downsampler.Builder().setInterval("15m").setAggregator("avg") + .setFillPolicy(new NumericFillPolicy(FillPolicy.NOT_A_NUMBER)).build()) + .build(); + String actual = JSON.serializeToString(timespan); + assertTrue(actual.contains("\"start\":\"1h-ago\"")); + assertTrue(actual.contains("\"end\":\"2015/05/05\"")); + assertTrue(actual.contains("\"aggregator\":\"sum\"")); + assertTrue(actual.contains("\"timezone\":\"UTC\"")); + assertTrue(actual.contains("\"downsampler\":{")); + assertTrue(actual.contains("\"interval\":\"15m\"")); + } +} diff --git a/test/resources/logback-test.xml b/test/resources/logback-test.xml new file mode 100644 index 0000000000..9afb833523 --- /dev/null +++ b/test/resources/logback-test.xml @@ -0,0 +1 @@ +<configuration /> diff --git a/test/rollup/TestRollupConfig.java b/test/rollup/TestRollupConfig.java new file mode 100644 index 0000000000..612c28f010 --- /dev/null +++ b/test/rollup/TestRollupConfig.java @@ -0,0 +1,297 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.rollup; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import org.hbase.async.HBaseClient; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.stumbleupon.async.Deferred; + +import net.opentsdb.core.TSDB; +import net.opentsdb.utils.JSON; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({ TSDB.class, HBaseClient.class }) +public class TestRollupConfig { + private final static String tsdb_table = "tsdb"; + private final static String rollup_table = "tsdb-rollup-10m"; + private final static String preagg_table = "tsdb-rollup-agg-10m"; + private final static String rollup_table_1h = "tsdb-rollup-1h"; + private final static String preagg_table_1h = "tsdb-rollup-agg-1h"; + + private TSDB tsdb; + private HBaseClient client; + private RollupConfig.Builder builder; + private RollupInterval raw; + private RollupInterval tenmin; + private RollupInterval oneHourWithDelay; + + @Before + public void before() throws Exception { + tsdb = PowerMockito.mock(TSDB.class); + client = PowerMockito.mock(HBaseClient.class); + when(tsdb.getClient()).thenReturn(client); + + raw = RollupInterval.builder() + .setTable(tsdb_table) + .setPreAggregationTable(tsdb_table) + .setInterval("1m") + .setRowSpan("1h") + .setDefaultInterval(true) + .build(); + + tenmin = RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("10m") + .setRowSpan("1d") + .build(); + + oneHourWithDelay = RollupInterval.builder() + .setTable(rollup_table_1h) + .setPreAggregationTable(preagg_table_1h) + .setInterval("1h") + .setRowSpan("1d") + .setDelaySla("2d") + .build(); + + builder = RollupConfig.builder() + .addAggregationId("Sum", 0) + .addAggregationId("Max", 1) + .addInterval(raw) + .addInterval(tenmin) + .addInterval(oneHourWithDelay); + } + + @Test + public void ctor() throws Exception { + RollupConfig config = builder.build(); + assertEquals(3, config.forward_intervals.size()); + assertSame(raw, config.forward_intervals.get("1m")); + assertSame(tenmin, config.forward_intervals.get("10m")); + assertSame(oneHourWithDelay, config.forward_intervals.get("1h")); + + assertEquals(5, config.reverse_intervals.size()); + assertSame(raw, config.reverse_intervals.get(tsdb_table)); + assertSame(tenmin, config.reverse_intervals.get(rollup_table)); + assertSame(tenmin, config.reverse_intervals.get(preagg_table)); + assertSame(oneHourWithDelay, config.reverse_intervals.get(rollup_table_1h)); + assertSame(oneHourWithDelay, config.reverse_intervals.get(preagg_table_1h)); + + assertEquals(2, config.aggregations_to_ids.size()); + assertEquals(2, config.ids_to_aggregations.size()); + + assertEquals(0, (int) config.aggregations_to_ids.get("sum")); + assertEquals(1, (int) config.aggregations_to_ids.get("max")); + + assertEquals("sum", config.ids_to_aggregations.get(0)); + assertEquals("max", config.ids_to_aggregations.get(1)); + + // missing aggregations + builder = RollupConfig.builder() + .addInterval(raw) + .addInterval(tenmin) + .addInterval(oneHourWithDelay); + try { + builder.build(); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + + // duplicate aggregation id + builder = RollupConfig.builder() + .addAggregationId("Sum", 1) + .addAggregationId("Max", 1) + .addInterval(raw) + .addInterval(tenmin) + .addInterval(oneHourWithDelay); + try { + builder.build(); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + + // invalid ID + builder = RollupConfig.builder() + .addAggregationId("Sum", 0) + .addAggregationId("Max", 128) + .addInterval(raw) + .addInterval(tenmin) + .addInterval(oneHourWithDelay); + try { + builder.build(); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + + // empty intervals + builder = RollupConfig.builder() + .addAggregationId("Sum", 0) + .addAggregationId("Max", 1); + try { + builder.build(); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + + // dupe intervals + builder = RollupConfig.builder() + .addAggregationId("Sum", 0) + .addAggregationId("Max", 1) + .addInterval(raw) + .addInterval(raw); + try { + builder.build(); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + + // two defaults + tenmin = RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("10m") + .setRowSpan("1d") + .setDefaultInterval(true) + .build(); + builder = RollupConfig.builder() + .addAggregationId("Sum", 0) + .addAggregationId("Max", 1) + .addInterval(raw) + .addInterval(raw); + try { + builder.build(); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + } + + @Test + public void getRollupIntervalString() throws Exception { + final RollupConfig config = builder.build(); + + assertSame(raw, config.getRollupInterval("1m")); + assertSame(tenmin, config.getRollupInterval("10m")); + assertSame(oneHourWithDelay, config.getRollupInterval("1h")); + + try { + config.getRollupInterval("5m"); + fail("Expected NoSuchRollupForIntervalException"); + } catch (NoSuchRollupForIntervalException e) { } + + try { + config.getRollupInterval(null); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + + try { + config.getRollupInterval(""); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + } + + @Test + public void getRollupIntervalForTable() throws Exception { + final RollupConfig config = builder.build(); + + assertSame(raw, config.getRollupIntervalForTable(tsdb_table)); + assertSame(tenmin, config.getRollupIntervalForTable(rollup_table)); + assertSame(tenmin, config.getRollupIntervalForTable(preagg_table)); + assertSame(oneHourWithDelay, config.getRollupIntervalForTable(rollup_table_1h)); + assertSame(oneHourWithDelay, config.getRollupIntervalForTable(preagg_table_1h)); + + try { + config.getRollupIntervalForTable("nosuchtable"); + fail("Expected NoSuchRollupForTableException"); + } catch (NoSuchRollupForTableException e) { } + + try { + config.getRollupIntervalForTable(null); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + + try { + config.getRollupIntervalForTable(""); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + } + + @Test + public void serdes() throws Exception { + RollupConfig config = builder.build(); + String json = JSON.serializeToString(config); + + assertTrue(json.contains("\"intervals\":[")); + assertTrue(json.contains("\"interval\":\"1m\"")); + assertTrue(json.contains("interval\":\"10m\"")); + assertTrue(json.contains("\"aggregationIds\":{")); + assertTrue(json.contains("\"sum\":0")); + assertTrue(json.contains("\"max\":1")); + + json = "{\"intervals\":[{\"interval\":\"1m\",\"table\":\"tsdb\"," + + "\"preAggregationTable\":\"tsdb\",\"defaultInterval\":true," + + "\"rowSpan\":\"1h\"},{\"interval\":\"10m\",\"table\":" + + "\"tsdb-rollup-10m\",\"preAggregationTable\":\"tsdb-rollup-agg-10m\"," + + "\"defaultInterval\":false,\"rowSpan\":\"1d\"}],\"aggregationIds\":" + + "{\"sum\":0,\"max\":1}}"; + config = JSON.parseToObject(json, RollupConfig.class); + assertEquals(2, config.forward_intervals.size()); + assertNotNull(config.forward_intervals.get("1m")); + assertNotNull(config.forward_intervals.get("10m")); + + assertEquals(3, config.reverse_intervals.size()); + assertNotNull(config.reverse_intervals.get(tsdb_table)); + assertNotNull(config.reverse_intervals.get(rollup_table)); + assertNotNull(config.reverse_intervals.get(preagg_table)); + + assertEquals(2, config.aggregations_to_ids.size()); + assertEquals(2, config.ids_to_aggregations.size()); + + assertEquals(0, (int) config.aggregations_to_ids.get("sum")); + assertEquals(1, (int) config.aggregations_to_ids.get("max")); + + assertEquals("sum", config.ids_to_aggregations.get(0)); + assertEquals("max", config.ids_to_aggregations.get(1)); + } + + @Test + public void ensureTablesExist() throws Exception { + when(client.ensureTableExists(any(byte[].class))) + .thenAnswer(new Answer<Deferred<Object>>() { + @Override + public Deferred<Object> answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult(null); + } + }); + + final RollupConfig config = builder.build(); + config.ensureTablesExist(tsdb); + verify(client, times(2)).ensureTableExists(tsdb_table.getBytes()); + verify(client, times(1)).ensureTableExists(rollup_table.getBytes()); + verify(client, times(1)).ensureTableExists(preagg_table.getBytes()); + verify(client, times(1)).ensureTableExists(rollup_table_1h.getBytes()); + verify(client, times(1)).ensureTableExists(preagg_table_1h.getBytes()); + } +} diff --git a/test/rollup/TestRollupInterval.java b/test/rollup/TestRollupInterval.java new file mode 100644 index 0000000000..d82674d61c --- /dev/null +++ b/test/rollup/TestRollupInterval.java @@ -0,0 +1,621 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.rollup; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +import java.nio.charset.Charset; + +import org.hbase.async.Bytes; +import org.junit.Test; + +import net.opentsdb.utils.JSON; + +public class TestRollupInterval { + private final static Charset CHARSET = Charset.forName("ISO-8859-1"); + private final static String rollup_table = "tsdb-rollup-10m"; + private final static String preagg_table = "tsdb-rollup-agg-10m"; + private final static byte[] table = rollup_table.getBytes(CHARSET); + private final static byte[] agg_table = preagg_table.getBytes(CHARSET); + + @Test + public void ctor1SecondHourNoSla() throws Exception { + final RollupInterval interval = RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("1s") + .setRowSpan("1h") + .build(); + assertEquals('h', interval.getUnits()); + assertEquals("1s", interval.getInterval()); + assertEquals('s', interval.getIntervalUnits()); + assertEquals(3600, interval.getIntervals()); + assertEquals(1, interval.getIntervalSeconds()); + assertEquals(rollup_table, interval.getTable()); + assertEquals(preagg_table, interval.getPreAggregationTable()); + assertEquals(0, Bytes.memcmp(table, interval.getTemporalTable())); + assertEquals(0, Bytes.memcmp(agg_table, interval.getGroupbyTable())); + assertEquals(0, interval.getMaximumLag()); + } + + // test odd boundaries + @Test + public void ctor7SecondHourTwoHoursDelay() throws Exception { + final RollupInterval interval = RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("7s") + .setRowSpan("1h") + .setDelaySla("2h") + .build(); + assertEquals('h', interval.getUnits()); + assertEquals("7s", interval.getInterval()); + assertEquals('s', interval.getIntervalUnits()); + assertEquals(514, interval.getIntervals()); + assertEquals(7, interval.getIntervalSeconds()); + assertEquals(rollup_table, interval.getTable()); + assertEquals(preagg_table, interval.getPreAggregationTable()); + assertEquals(0, Bytes.memcmp(table, interval.getTemporalTable())); + assertEquals(0, Bytes.memcmp(agg_table, interval.getGroupbyTable())); + assertEquals(7200, interval.getMaximumLag()); + } + + @Test + public void ctor15SecondsHour() throws Exception { + final RollupInterval interval = RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("15s") + .setRowSpan("1h") + .build(); + assertEquals('h', interval.getUnits()); + assertEquals("15s", interval.getInterval()); + assertEquals('s', interval.getIntervalUnits()); + assertEquals(240, interval.getIntervals()); + assertEquals(15, interval.getIntervalSeconds()); + assertEquals(rollup_table, interval.getTable()); + assertEquals(preagg_table, interval.getPreAggregationTable()); + assertEquals(0, Bytes.memcmp(table, interval.getTemporalTable())); + assertEquals(0, Bytes.memcmp(agg_table, interval.getGroupbyTable())); + } + + @Test + public void ctor30SecondsHour() throws Exception { + final RollupInterval interval = RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("30s") + .setRowSpan("1h") + .build(); + assertEquals('h', interval.getUnits()); + assertEquals("30s", interval.getInterval()); + assertEquals('s', interval.getIntervalUnits()); + assertEquals(120, interval.getIntervals()); + assertEquals(30, interval.getIntervalSeconds()); + assertEquals(rollup_table, interval.getTable()); + assertEquals(preagg_table, interval.getPreAggregationTable()); + assertEquals(0, Bytes.memcmp(table, interval.getTemporalTable())); + assertEquals(0, Bytes.memcmp(agg_table, interval.getGroupbyTable())); + } + + @Test + public void ctor1MinuteDay() throws Exception { + final RollupInterval interval = RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("1m") + .setRowSpan("1d") + .build(); + assertEquals('d', interval.getUnits()); + assertEquals("1m", interval.getInterval()); + assertEquals('m', interval.getIntervalUnits()); + assertEquals(1440, interval.getIntervals()); + assertEquals(60, interval.getIntervalSeconds()); + assertEquals(rollup_table, interval.getTable()); + assertEquals(preagg_table, interval.getPreAggregationTable()); + assertEquals(0, Bytes.memcmp(table, interval.getTemporalTable())); + assertEquals(0, Bytes.memcmp(agg_table, interval.getGroupbyTable())); + } + + @Test + public void ctor10MinuteDay() throws Exception { + final RollupInterval interval = RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("10m") + .setRowSpan("1d") + .build(); + assertEquals('d', interval.getUnits()); + assertEquals("10m", interval.getInterval()); + assertEquals('m', interval.getIntervalUnits()); + assertEquals(144, interval.getIntervals()); + assertEquals(600, interval.getIntervalSeconds()); + assertEquals(rollup_table, interval.getTable()); + assertEquals(preagg_table, interval.getPreAggregationTable()); + assertEquals(0, Bytes.memcmp(table, interval.getTemporalTable())); + assertEquals(0, Bytes.memcmp(agg_table, interval.getGroupbyTable())); + } + + @Test + public void ctor10Minute6Hours() throws Exception { + final RollupInterval interval = RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("10m") + .setRowSpan("6h") + .build(); + assertEquals('h', interval.getUnits()); + assertEquals("10m", interval.getInterval()); + assertEquals('m', interval.getIntervalUnits()); + assertEquals(36, interval.getIntervals()); + assertEquals(600, interval.getIntervalSeconds()); + assertEquals(rollup_table, interval.getTable()); + assertEquals(preagg_table, interval.getPreAggregationTable()); + assertEquals(0, Bytes.memcmp(table, interval.getTemporalTable())); + assertEquals(0, Bytes.memcmp(agg_table, interval.getGroupbyTable())); + } + + @Test + public void ctor10Minute12Hours() throws Exception { + final RollupInterval interval = RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("10m") + .setRowSpan("12h") + .build(); + assertEquals('h', interval.getUnits()); + assertEquals("10m", interval.getInterval()); + assertEquals('m', interval.getIntervalUnits()); + assertEquals(72, interval.getIntervals()); + assertEquals(600, interval.getIntervalSeconds()); + assertEquals(rollup_table, interval.getTable()); + assertEquals(preagg_table, interval.getPreAggregationTable()); + assertEquals(0, Bytes.memcmp(table, interval.getTemporalTable())); + assertEquals(0, Bytes.memcmp(agg_table, interval.getGroupbyTable())); + } + + @Test + public void ctor15MinuteDay() throws Exception { + final RollupInterval interval = RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("15m") + .setRowSpan("1d") + .build(); + assertEquals('d', interval.getUnits()); + assertEquals("15m", interval.getInterval()); + assertEquals('m', interval.getIntervalUnits()); + assertEquals(96, interval.getIntervals()); + assertEquals(900, interval.getIntervalSeconds()); + assertEquals(rollup_table, interval.getTable()); + assertEquals(preagg_table, interval.getPreAggregationTable()); + assertEquals(0, Bytes.memcmp(table, interval.getTemporalTable())); + assertEquals(0, Bytes.memcmp(agg_table, interval.getGroupbyTable())); + } + + @Test + public void ctor30MinuteDay() throws Exception { + final RollupInterval interval = RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("30m") + .setRowSpan("1d") + .build(); + assertEquals('d', interval.getUnits()); + assertEquals("30m", interval.getInterval()); + assertEquals('m', interval.getIntervalUnits()); + assertEquals(48, interval.getIntervals()); + assertEquals(1800, interval.getIntervalSeconds()); + assertEquals(rollup_table, interval.getTable()); + assertEquals(preagg_table, interval.getPreAggregationTable()); + assertEquals(0, Bytes.memcmp(table, interval.getTemporalTable())); + assertEquals(0, Bytes.memcmp(agg_table, interval.getGroupbyTable())); + } + + @Test + public void ctor1HourDay() throws Exception { + final RollupInterval interval = RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("1h") + .setRowSpan("1d") + .build(); + assertEquals('d', interval.getUnits()); + assertEquals("1h", interval.getInterval()); + assertEquals('h', interval.getIntervalUnits()); + assertEquals(24, interval.getIntervals()); + assertEquals(3600, interval.getIntervalSeconds()); + assertEquals(rollup_table, interval.getTable()); + assertEquals(preagg_table, interval.getPreAggregationTable()); + assertEquals(0, Bytes.memcmp(table, interval.getTemporalTable())); + assertEquals(0, Bytes.memcmp(agg_table, interval.getGroupbyTable())); + } + + @Test + public void ctor1HourMonth() throws Exception { + final RollupInterval interval = RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("1h") + .setRowSpan("1n") + .build(); + assertEquals('n', interval.getUnits()); + assertEquals("1h", interval.getInterval()); + assertEquals('h', interval.getIntervalUnits()); + assertEquals(768, interval.getIntervals()); + assertEquals(3600, interval.getIntervalSeconds()); + assertEquals(rollup_table, interval.getTable()); + assertEquals(preagg_table, interval.getPreAggregationTable()); + assertEquals(0, Bytes.memcmp(table, interval.getTemporalTable())); + assertEquals(0, Bytes.memcmp(agg_table, interval.getGroupbyTable())); + } + + @Test + public void ctor3HourMonth() throws Exception { + final RollupInterval interval = RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("3h") + .setRowSpan("1n") + .build(); + assertEquals('n', interval.getUnits()); + assertEquals("3h", interval.getInterval()); + assertEquals('h', interval.getIntervalUnits()); + assertEquals(256, interval.getIntervals()); + assertEquals(10800, interval.getIntervalSeconds()); + assertEquals(rollup_table, interval.getTable()); + assertEquals(preagg_table, interval.getPreAggregationTable()); + assertEquals(0, Bytes.memcmp(table, interval.getTemporalTable())); + assertEquals(0, Bytes.memcmp(agg_table, interval.getGroupbyTable())); + } + + @Test + public void ctor6HourMonth() throws Exception { + final RollupInterval interval = RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("6h") + .setRowSpan("1n") + .build(); + assertEquals('n', interval.getUnits()); + assertEquals("6h", interval.getInterval()); + assertEquals('h', interval.getIntervalUnits()); + assertEquals(128, interval.getIntervals()); + assertEquals(21600, interval.getIntervalSeconds()); + assertEquals(rollup_table, interval.getTable()); + assertEquals(preagg_table, interval.getPreAggregationTable()); + assertEquals(0, Bytes.memcmp(table, interval.getTemporalTable())); + assertEquals(0, Bytes.memcmp(agg_table, interval.getGroupbyTable())); + } + + @Test + public void ctor6HourYear() throws Exception { + final RollupInterval interval = RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("6h") + .setRowSpan("1y") + .build(); + assertEquals('y', interval.getUnits()); + assertEquals("6h", interval.getInterval()); + assertEquals('h', interval.getIntervalUnits()); + assertEquals(1464, interval.getIntervals()); + assertEquals(21600, interval.getIntervalSeconds()); + assertEquals(rollup_table, interval.getTable()); + assertEquals(preagg_table, interval.getPreAggregationTable()); + assertEquals(0, Bytes.memcmp(table, interval.getTemporalTable())); + assertEquals(0, Bytes.memcmp(agg_table, interval.getGroupbyTable())); + } + + @Test + public void ctor12HourYear() throws Exception { + final RollupInterval interval = RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("12h") + .setRowSpan("1y") + .build(); + assertEquals('y', interval.getUnits()); + assertEquals("12h", interval.getInterval()); + assertEquals('h', interval.getIntervalUnits()); + assertEquals(732, interval.getIntervals()); + assertEquals(43200, interval.getIntervalSeconds()); + assertEquals(rollup_table, interval.getTable()); + assertEquals(preagg_table, interval.getPreAggregationTable()); + assertEquals(0, Bytes.memcmp(table, interval.getTemporalTable())); + assertEquals(0, Bytes.memcmp(agg_table, interval.getGroupbyTable())); + } + + @Test + public void ctor1DayYear() throws Exception { + final RollupInterval interval = RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("1d") + .setRowSpan("1y") + .build(); + assertEquals('y', interval.getUnits()); + assertEquals("1d", interval.getInterval()); + assertEquals('d', interval.getIntervalUnits()); + assertEquals(366, interval.getIntervals()); + assertEquals(86400, interval.getIntervalSeconds()); + assertEquals(rollup_table, interval.getTable()); + assertEquals(preagg_table, interval.getPreAggregationTable()); + assertEquals(0, Bytes.memcmp(table, interval.getTemporalTable())); + assertEquals(0, Bytes.memcmp(agg_table, interval.getGroupbyTable())); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorUnknownNullRollupTable() throws Exception { + RollupInterval.builder() + .setTable(null) + .setPreAggregationTable(preagg_table) + .setInterval("1h") + .setRowSpan("1d") + .build(); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorUnknownEmptyRollupTable() throws Exception { + RollupInterval.builder() + .setTable("") + .setPreAggregationTable(preagg_table) + .setInterval("1h") + .setRowSpan("1d") + .build(); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorUnknownNullPreAggTable() throws Exception { + RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(null) + .setInterval("1h") + .setRowSpan("1d") + .build(); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorUnknownEmptyPreAggTable() throws Exception { + RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable("") + .setInterval("1h") + .setRowSpan("1d") + .build(); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorUnknownSpan() throws Exception { + RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("1h") + .setRowSpan("1s") + .build(); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNullInterval() throws Exception { + RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval(null) + .setRowSpan("1d") + .build(); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorEmptyInterval() throws Exception { + RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("") + .setRowSpan("1d") + .build(); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorBigDuration() throws Exception { + RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("365y") + .setRowSpan("1d") + .build(); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorTooManyIntervals() throws Exception { + RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("1s") + .setRowSpan("17") + .build(); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorDurationTooBigForSpan() throws Exception { + RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("36500s") + .setRowSpan("1h") + .build(); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorDurationEqualToSpan() throws Exception { + RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("3600s") + .setRowSpan("1h") + .build(); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorTooFewIntervals() throws Exception { + RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("3000s") + .setRowSpan("1h") + .build(); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNoUnitsInSpan() throws Exception { + RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("365y") + .setRowSpan("1") + .build(); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNoIntervalInSpan() throws Exception { + RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("365y") + .setRowSpan("d") + .build(); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNoMs() throws Exception { + RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("365y") + .setRowSpan("1000ms") + .build(); + } + + @Test (expected = IllegalArgumentException.class) + public void ctor15Minute7Days() throws Exception { + RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("15m") + .setRowSpan("7d") + .build(); + } + + @Test + public void serdes() throws Exception { + RollupInterval interval = RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("1s") + .setRowSpan("1h") + .setDefaultInterval(true) + .build(); + + String json = JSON.serializeToString(interval); + assertTrue(json.contains("\"interval\":\"1s\"")); + assertTrue(json.contains("\"table\":\"tsdb-rollup-10m\"")); + assertTrue(json.contains("\"defaultInterval\":true")); + assertTrue(json.contains("\"rowSpan\":\"1h\"")); + assertTrue(json.contains("\"preAggregationTable\":\"tsdb-rollup-agg-10m\"")); + + json = "{\"interval\":\"1s\",\"table\":\"tsdb-rollup-10m\"," + + "\"defaultRollupInterval\":true,\"rowSpan\":\"1h\"," + + "\"preAggregationTable\":\"tsdb-rollup-agg-10m\"}"; + interval = JSON.parseToObject(json, RollupInterval.class); + assertEquals('h', interval.getUnits()); + assertEquals("1s", interval.getInterval()); + assertEquals('s', interval.getIntervalUnits()); + assertEquals(3600, interval.getIntervals()); + assertEquals(1, interval.getIntervalSeconds()); + assertEquals(rollup_table, interval.getTable()); + assertEquals(preagg_table, interval.getPreAggregationTable()); + assertEquals(0, Bytes.memcmp(table, interval.getTemporalTable())); + assertEquals(0, Bytes.memcmp(agg_table, interval.getGroupbyTable())); + } + + @Test + public void testHashCodeAndEquals() throws Exception { + final RollupInterval interval_a = RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("1s") + .setRowSpan("1h") + .setDefaultInterval(true) + .build(); + RollupInterval interval_b = RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("1s") + .setRowSpan("1h") + .setDefaultInterval(true) + .build(); + assertEquals(interval_a.hashCode(), interval_b.hashCode()); + assertEquals(interval_a, interval_b); + + interval_b = RollupInterval.builder() + .setTable("nothertable") // <-- DIFF + .setPreAggregationTable(preagg_table) + .setInterval("1s") + .setRowSpan("1h") + .setDefaultInterval(true) + .build(); + assertNotEquals(interval_a.hashCode(), interval_b.hashCode()); + assertNotEquals(interval_a, interval_b); + + interval_b = RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable("nothertable") // <-- DIFF + .setInterval("1s") + .setRowSpan("1h") + .setDefaultInterval(true) + .build(); + assertNotEquals(interval_a.hashCode(), interval_b.hashCode()); + assertNotEquals(interval_a, interval_b); + + interval_b = RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("30s") // <-- DIFF + .setRowSpan("1h") + .setDefaultInterval(true) + .build(); + assertNotEquals(interval_a.hashCode(), interval_b.hashCode()); + assertNotEquals(interval_a, interval_b); + + interval_b = RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("1s") + .setRowSpan("2h") // <-- DIFF + .setDefaultInterval(true) + .build(); + assertNotEquals(interval_a.hashCode(), interval_b.hashCode()); + assertNotEquals(interval_a, interval_b); + + interval_b = RollupInterval.builder() + .setTable(rollup_table) + .setPreAggregationTable(preagg_table) + .setInterval("1s") + .setRowSpan("1h") + //.setIsDefault(true) // <-- DIFF + .build(); + assertNotEquals(interval_a.hashCode(), interval_b.hashCode()); + assertNotEquals(interval_a, interval_b); + } +} diff --git a/test/rollup/TestRollupQuery.java b/test/rollup/TestRollupQuery.java new file mode 100644 index 0000000000..31c0232dae --- /dev/null +++ b/test/rollup/TestRollupQuery.java @@ -0,0 +1,75 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015-2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.rollup; + +import net.opentsdb.core.Aggregators; +import net.opentsdb.utils.DateTime; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import static org.junit.Assert.*; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({ + DateTime.class +}) +public class TestRollupQuery { + + private static final long MOCK_TIMESTAMP = 1554117071000L; + private static final int ONE_HOUR_SECONDS = 60 * 60; + private static final int ONE_DAY_SECONDS = 24 * ONE_HOUR_SECONDS; + private static final int TWO_DAYS_SECONDS = 2 * ONE_DAY_SECONDS; + + private RollupQuery query; + + @Before + public void before() { + final RollupInterval oneHourWithDelay = RollupInterval.builder() + .setTable("fake-rollup-table") + .setPreAggregationTable("fake-preagg-table") + .setInterval("1h") + .setRowSpan("1d") + .setDelaySla("2d") + .build(); + query = new RollupQuery( + oneHourWithDelay, + Aggregators.SUM, + 3600000, + Aggregators.SUM + ); + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(MOCK_TIMESTAMP); + } + + + @Test + public void testGetLastRollupTimestamp() { + long nowSeconds = MOCK_TIMESTAMP / 1000; + long twoDaysAgo = nowSeconds - TWO_DAYS_SECONDS; + + assertEquals(twoDaysAgo, query.getLastRollupTimestampSeconds()); + } + + @Test + public void testIsInBlackoutPeriod() { + long oneHourAgo = MOCK_TIMESTAMP - ONE_HOUR_SECONDS * 1000; + assertTrue(query.isInBlackoutPeriod(oneHourAgo)); + + long threeDaysAgo = MOCK_TIMESTAMP - 3 * ONE_DAY_SECONDS * 1000; + assertFalse(query.isInBlackoutPeriod(threeDaysAgo)); + } +} diff --git a/test/rollup/TestRollupSeq.java b/test/rollup/TestRollupSeq.java new file mode 100644 index 0000000000..536f7fe639 --- /dev/null +++ b/test/rollup/TestRollupSeq.java @@ -0,0 +1,1920 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2013 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.rollup; + +import net.opentsdb.core.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.when; +import static org.powermock.api.mockito.PowerMockito.mock; +import net.opentsdb.uid.UniqueId; +import net.opentsdb.uid.UniqueId.UniqueIdType; +import net.opentsdb.utils.Config; + +import org.hbase.async.Bytes; +import org.hbase.async.KeyValue; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +import com.stumbleupon.async.Deferred; + +import java.util.Arrays; + +@RunWith(PowerMockRunner.class) +//"Classloader hell"... It's real. Tell PowerMock to ignore these classes +//because they fiddle with the class loader. We don't test them anyway. +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ RollupSeq.class, TSDB.class, UniqueId.class, KeyValue.class, + Config.class, RowKey.class, Const.class }) +public class TestRollupSeq { + protected TSDB tsdb = mock(TSDB.class); + protected Config config = mock(Config.class); + protected UniqueId metrics = mock(UniqueId.class); + protected RollupConfig rollup_config; + protected static final byte[] TABLE = { 't', 'a', 'b', 'l', 'e' }; + public static final byte[] FAMILY = { 't' }; + protected byte[] key = null; + + protected static final RollupQuery rollup_query_sum = + new RollupQuery(RollupInterval.builder() + .setTable("tsdb") + .setPreAggregationTable("tsdb-agg") + .setInterval("1s") + .setRowSpan("1h") + .build(), + Aggregators.SUM, + 1000, + Aggregators.SUM); + protected static final RollupQuery rollup_query_avg = + new RollupQuery(RollupInterval.builder() + .setTable("tsdb") + .setPreAggregationTable("tsdb-agg") + .setInterval("1s") + .setRowSpan("1h") + .build(), + Aggregators.AVG, + 1000, + Aggregators.AVG); + protected static final RollupQuery rollup_query_sum_mimmax = + new RollupQuery(RollupInterval.builder() + .setTable("tsdb") + .setPreAggregationTable("tsdb-agg") + .setInterval("1s") + .setRowSpan("1h") + .build(), + Aggregators.MIMMAX, + 1000, + Aggregators.MIMMAX); + protected static final RollupQuery rollup_query_10m_sum = + new RollupQuery(RollupInterval.builder() + .setTable("tsdb") + .setPreAggregationTable("tsdb-agg") + .setInterval("10m") + .setRowSpan("6h") + .build(), + Aggregators.SUM, + 600000, + Aggregators.SUM); + protected static final RollupQuery rollup_query_10m_avg = + new RollupQuery(RollupInterval.builder() + .setTable("tsdb") + .setPreAggregationTable("tsdb-agg") + .setInterval("10m") + .setRowSpan("6h") + .build(), + Aggregators.AVG, + 600000, + Aggregators.AVG); + protected static final RollupQuery rollup_query_10m_count = + new RollupQuery(RollupInterval.builder() + .setTable("tsdb") + .setPreAggregationTable("tsdb-agg") + .setInterval("10m") + .setRowSpan("6h") + .build(), + Aggregators.COUNT, + 600000, + Aggregators.COUNT); + protected static final RollupQuery rollup_query_1h_sum = + new RollupQuery(RollupInterval.builder() + .setTable("tsdb") + .setPreAggregationTable("tsdb-agg") + .setInterval("1h") + .setRowSpan("1d") + .build(), + Aggregators.SUM, + 3600000, + Aggregators.SUM); + protected static final RollupQuery rollup_query_1h_avg = + new RollupQuery(RollupInterval.builder() + .setTable("tsdb") + .setPreAggregationTable("tsdb-agg") + .setInterval("1h") + .setRowSpan("1d") + .build(), + Aggregators.AVG, + 3600000, + Aggregators.AVG); + protected static final RollupQuery rollup_query_1h_count = + new RollupQuery(RollupInterval.builder() + .setTable("tsdb") + .setPreAggregationTable("tsdb-agg") + .setInterval("1h") + .setRowSpan("1d") + .build(), + Aggregators.COUNT, + 3600000, + Aggregators.COUNT); + protected static final RollupQuery rollup_query_1h_avg_group_by_sum = + new RollupQuery(RollupInterval.builder() + .setTable("tsdb") + .setPreAggregationTable("tsdb-agg") + .setInterval("1h") + .setRowSpan("1d") + .build(), + Aggregators.AVG, + 3600000, + Aggregators.SUM); + + @Before + public void before() throws Exception { + // Inject the attributes we need into the "tsdb" object. + Whitebox.setInternalState(tsdb, "metrics", metrics); + Whitebox.setInternalState(tsdb, "table", TABLE); + Whitebox.setInternalState(tsdb, "config", config); + when(tsdb.getConfig()).thenReturn(config); + key = BaseTsdbTest.getRowKey( + BaseTsdbTest.generateUID(UniqueIdType.METRIC, (byte) 1), + 1356998400, + BaseTsdbTest.generateUID(UniqueIdType.TAGK, (byte) 1), + BaseTsdbTest.generateUID(UniqueIdType.TAGV, (byte) 1)); + when(RowKey.metricNameAsync(tsdb, key)) + .thenReturn(Deferred.fromResult("sys.cpu.user")); + + rollup_config = RollupConfig.builder() + .addAggregationId("sum", 0) + .addAggregationId("count", 1) + .addAggregationId("max", 2) + .addAggregationId("min", 3) + .addAggregationId("avg", 4) + .addInterval(RollupInterval.builder() + .setTable("tsdb-rollup-10m") + .setPreAggregationTable("tsdb-rollup-agg-10m") + .setInterval("10m") + .setRowSpan("6h")) + .addInterval(RollupInterval.builder() + .setTable("tsdb-rollup-1h") + .setPreAggregationTable("tsdb-rollup-agg-1h") + .setInterval("1h") + .setRowSpan("1d")) + .addInterval(RollupInterval.builder() + .setTable("tsdb-rollup-1d") + .setPreAggregationTable("tsdb-rollup-agg-1d") + .setInterval("1d") + .setRowSpan("1n")) + .build(); + when(tsdb.getRollupConfig()).thenReturn(rollup_config); + } + + @Test + public void setRow() throws Exception { + final KeyValue kv = getRollupKeyValue(key, 1356998400000L, 4L, + rollup_config.getIdForAggregator("SUM"), rollup_query_sum); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_sum); + rs.setRow(kv); + assertEquals(1, rs.size()); + final SeekableView it = rs.iterator(); + assertTrue(it.hasNext()); + DataPoint dp = it.next(); + assertTrue(dp.isInteger()); + assertEquals(4, dp.longValue()); + assertEquals(1356998400000L, dp.timestamp()); + assertEquals(1, dp.valueCount()); + assertFalse(it.hasNext()); + } + + @Test (expected = IllegalStateException.class) + public void setRowAlreadySet() throws Exception { + final KeyValue kv = getRollupKeyValue(key, 1356998400000L, 4L, + rollup_config.getIdForAggregator("SUM"), rollup_query_sum); + + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_sum); + rs.setRow(kv); + assertEquals(1, rs.size()); + //Expects an IllegalStateException + final KeyValue kv1 = getRollupKeyValue(key, 1356998500000L, 5L, + rollup_config.getIdForAggregator("SUM"), rollup_query_sum); + rs.setRow(kv1); + } + + @Test + public void addRow() throws Exception { + final KeyValue kv1 = getRollupKeyValue(key, 1356998400000L, 4L, + rollup_config.getIdForAggregator("SUM"), rollup_query_sum); + final KeyValue kv2 = getRollupKeyValue(key, 1356998500000L, 5L, + rollup_config.getIdForAggregator("SUM"), rollup_query_sum); + + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_sum); + rs.setRow(kv1); + assertEquals(1, rs.size()); + + rs.addRow(kv2); + assertEquals(2, rs.size()); + + final SeekableView it = rs.iterator(); + assertTrue(it.hasNext()); + DataPoint dp = it.next(); + assertTrue(dp.isInteger()); + assertEquals(1356998400000L, dp.timestamp()); + assertEquals(4, dp.longValue()); + assertEquals(1, dp.valueCount()); + + assertTrue(it.hasNext()); + dp = it.next(); + assertTrue(dp.isInteger()); + assertEquals(1356998500000L, dp.timestamp()); + assertEquals(5, dp.longValue()); + assertEquals(1, dp.valueCount()); + assertFalse(it.hasNext()); + } + + @Test + public void addRowMergeLater() throws Exception { + // this happens if the same row key is used for the addRow call + final byte[] qual1 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x17 }; + final byte[] val2 = Bytes.fromLong(5L); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_sum); + rs.setRow(TestRowSeq.makekv(key, qual1, val1)); + rs.addRow(TestRowSeq.makekv(key, qual2, val2)); + assertEquals(2, rs.size()); + + final byte[] qual3 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x27 }; + final byte[] val3 = Bytes.fromLong(6L); + final byte[] qual4 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x37 }; + final byte[] val4 = Bytes.fromLong(7L); + rs.addRow(TestRowSeq.makekv(key, qual3, val3)); + rs.addRow(TestRowSeq.makekv(key, qual4, val4)); + + assertEquals(4, rs.size()); + final SeekableView it = rs.iterator(); + long value = 4; + long ts = 1356998400000L; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(value, dp.longValue()); + assertEquals(1, dp.valueCount()); + ++value; + ts += 1000; + } + } + + @Test (expected = IllegalDataException.class) + public void addRowMergeEarlier() throws Exception { + // this happens if the same row key is used for the addRow call + final byte[] qual1 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x27 }; + final byte[] val1 = Bytes.fromLong(6L); + final byte[] qual2 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x37 }; + final byte[] val2 = Bytes.fromLong(7L); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_sum); + rs.setRow(TestRowSeq.makekv(key, qual1, val1)); + rs.addRow(TestRowSeq.makekv(key, qual2, val2)); + assertEquals(2, rs.size()); + + final byte[] qual3 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }; + final byte[] val3 = Bytes.fromLong(4L); + rs.addRow(TestRowSeq.makekv(key, qual3, val3)); + } + + @Test (expected = IllegalDataException.class) + public void addRowMergeMiddle() throws Exception { + // this happens if the same row key is used for the addRow call + final byte[] qual1 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x17 }; + final byte[] val2 = Bytes.fromLong(5L); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_sum); + rs.setRow(TestRowSeq.makekv(key, qual1, val1)); + rs.addRow(TestRowSeq.makekv(key, qual2, val2)); + assertEquals(2, rs.size()); + + final byte[] qual3 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x47 }; + final byte[] val3 = Bytes.fromLong(8L); + final byte[] qual4 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x57 }; + final byte[] val4 = Bytes.fromLong(9L); + rs.addRow(TestRowSeq.makekv(key, qual3, val3)); + rs.addRow(TestRowSeq.makekv(key, qual4, val4)); + assertEquals(4, rs.size()); + + final byte[] qual5 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x27 }; + final byte[] val5 = Bytes.fromLong(6L); + rs.addRow(TestRowSeq.makekv(key, qual5, val5)); + } + + @Test (expected = IllegalDataException.class) + public void addRowMergeDuplicateLater() throws Exception { + // this happens if the same row key is used for the addRow call + final byte[] qual1 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x17 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual3 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x27 }; + final byte[] val3 = Bytes.fromLong(6L); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_sum); + rs.setRow(TestRowSeq.makekv(key, qual1, val1)); + rs.addRow(TestRowSeq.makekv(key, qual2, val2)); + rs.addRow(TestRowSeq.makekv(key, qual3, val3)); + assertEquals(3, rs.size()); + rs.addRow(TestRowSeq.makekv(key, qual3, val3)); + } + + @Test + public void addRowMergeDuplicateLaterRepair() throws Exception { + when(config.fix_duplicates()).thenReturn(true); + // this happens if the same row key is used for the addRow call + final byte[] qual1 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x17 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual3 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x27 }; + final byte[] val3 = Bytes.fromLong(6L); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_sum); + rs.setRow(new KeyValue(key, FAMILY, qual1, 1, val1)); + rs.addRow(new KeyValue(key, FAMILY, qual2, 2, val2)); + rs.addRow(new KeyValue(key, FAMILY, qual3, 3, val3)); + assertEquals(3, rs.size()); + SeekableView it = rs.iterator(); + long ts = 1356998400000L; + double value = 4.0; + while(it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.toDouble(), 0.0001); + ts += 1000; + ++value; + } + rs.addRow(new KeyValue(key, FAMILY, qual3, 8, Bytes.fromLong(7L))); + assertEquals(3, rs.size()); + + it = rs.iterator(); + ts = 1356998400000L; + value = 4.0; + while(it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.toDouble(), 0.0001); + ts += 1000; + if (value >= 5) { + value = 7.0; + } else { + ++value; + } + } + } + + @Test (expected = IllegalDataException.class) + public void addRowMergeDuplicateEarlier() throws Exception { + // this happens if the same row key is used for the addRow call + final byte[] qual4 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x17 }; + final byte[] val4 = Bytes.fromLong(5L); + final byte[] qual1 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x27 }; + final byte[] val1 = Bytes.fromLong(6L); + final byte[] qual2 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x37 }; + final byte[] val2 = Bytes.fromLong(7L); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_sum); + rs.setRow(TestRowSeq.makekv(key, qual4, val4)); + rs.addRow(TestRowSeq.makekv(key, qual1, val1)); + rs.addRow(TestRowSeq.makekv(key, qual2, val2)); + assertEquals(3, rs.size()); + + final byte[] qual3 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }; + final byte[] val3 = Bytes.fromLong(4L); + rs.addRow(TestRowSeq.makekv(key, qual3, val3)); + } + + @Test + public void addRowMergeDuplicateEarlierRepair() throws Exception { + when(config.fix_duplicates()).thenReturn(true); + // this happens if the same row key is used for the addRow call + final byte[] qual1 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x17 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual3 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x27 }; + final byte[] val3 = Bytes.fromLong(6L); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_sum); + rs.setRow(new KeyValue(key, FAMILY, qual1, 1, val1)); + rs.addRow(new KeyValue(key, FAMILY, qual2, 2, val2)); + rs.addRow(new KeyValue(key, FAMILY, qual3, 3, val3)); + assertEquals(3, rs.size()); + SeekableView it = rs.iterator(); + long ts = 1356998400000L; + double value = 4.0; + while(it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.toDouble(), 0.0001); + ts += 1000; + ++value; + } + + rs.addRow(new KeyValue(key, FAMILY, qual3, 1, Bytes.fromLong(7L))); + assertEquals(3, rs.size()); + + it = rs.iterator(); + ts = 1356998400000L; + value = 4.0; + while(it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.toDouble(), 0.0001); + ts += 1000; + ++value; + } + } + + @Test (expected = IllegalArgumentException.class) + public void addRowMergeDuplicateCountLater() throws Exception { + // this happens if the same row key is used for the addRow call + final byte[] qual1 = { 0x63, 0x6F, 0x75, 0x6E, 0x74, 0x3A, 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x63, 0x6F, 0x75, 0x6E, 0x74, 0x3A, 0x00, 0x17 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual3 = { 0x63, 0x6F, 0x75, 0x6E, 0x74, 0x3A, 0x00, 0x27 }; + final byte[] val3 = Bytes.fromLong(6L); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_avg); + rs.setRow(TestRowSeq.makekv(key, qual1, val1)); + rs.addRow(TestRowSeq.makekv(key, qual2, val2)); + rs.addRow(TestRowSeq.makekv(key, qual3, val3)); + assertEquals(0, rs.size()); + rs.addRow(TestRowSeq.makekv(key, qual3, val3)); + } + + @Test + public void addRowMergeDuplicateCountLaterRepair() throws Exception { + when(config.fix_duplicates()).thenReturn(true); + // this happens if the same row key is used for the addRow call + final byte[] qual1 = { 0x63, 0x6F, 0x75, 0x6E, 0x74, 0x3A, 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x63, 0x6F, 0x75, 0x6E, 0x74, 0x3A, 0x00, 0x17 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual3 = { 0x63, 0x6F, 0x75, 0x6E, 0x74, 0x3A, 0x00, 0x27 }; + final byte[] val3 = Bytes.fromLong(6L); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_avg); + rs.setRow(new KeyValue(key, FAMILY, qual1, 1, val1)); + rs.addRow(new KeyValue(key, FAMILY, qual2, 2, val2)); + rs.addRow(new KeyValue(key, FAMILY, qual3, 3, val3)); + assertEquals(6, rs.count_values[23]); + assertEquals(0, rs.size()); + + rs.addRow(new KeyValue(key, FAMILY, qual3, 8, Bytes.fromLong(7L))); + assertEquals(7, rs.count_values[23]); + } + + @Test (expected = IllegalArgumentException.class) + public void addRowMergeDuplicateCountEarlier() throws Exception { + // this happens if the same row key is used for the addRow call + final byte[] qual4 = { 0x63, 0x6F, 0x75, 0x6E, 0x74, 0x3A, 0x00, 0x17 }; + final byte[] val4 = Bytes.fromLong(5L); + final byte[] qual1 = { 0x63, 0x6F, 0x75, 0x6E, 0x74, 0x3A, 0x00, 0x27 }; + final byte[] val1 = Bytes.fromLong(6L); + final byte[] qual2 = { 0x63, 0x6F, 0x75, 0x6E, 0x74, 0x3A, 0x00, 0x37 }; + final byte[] val2 = Bytes.fromLong(7L); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_avg); + rs.setRow(TestRowSeq.makekv(key, qual4, val4)); + rs.addRow(TestRowSeq.makekv(key, qual1, val1)); + rs.addRow(TestRowSeq.makekv(key, qual2, val2)); + assertEquals(0, rs.size()); + + final byte[] qual3 = { 0x63, 0x6F, 0x75, 0x6E, 0x74, 0x3A, 0x00, 0x07 }; + final byte[] val3 = Bytes.fromLong(4L); + rs.addRow(TestRowSeq.makekv(key, qual3, val3)); + } + + @Test + public void addRowMergeDuplicateCountEarlierRepair() throws Exception { + when(config.fix_duplicates()).thenReturn(true); + // this happens if the same row key is used for the addRow call + final byte[] qual1 = { 0x63, 0x6F, 0x75, 0x6E, 0x74, 0x3A, 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x63, 0x6F, 0x75, 0x6E, 0x74, 0x3A, 0x00, 0x17 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] qual3 = { 0x63, 0x6F, 0x75, 0x6E, 0x74, 0x3A, 0x00, 0x27 }; + final byte[] val3 = Bytes.fromLong(6L); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_avg); + rs.setRow(new KeyValue(key, FAMILY, qual1, 1, val1)); + rs.addRow(new KeyValue(key, FAMILY, qual2, 2, val2)); + rs.addRow(new KeyValue(key, FAMILY, qual3, 3, val3)); + assertEquals(0, rs.size()); + assertEquals(6, rs.count_values[23]); + + rs.addRow(new KeyValue(key, FAMILY, qual3, 1, Bytes.fromLong(7L))); + assertEquals(6, rs.count_values[23]); + } + + @Test (expected = IllegalDataException.class) + public void addRowDiffBaseTime() throws Exception { + final byte[] qual1 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_sum); + rs.setRow(TestRowSeq.makekv(key, qual1, val1)); + rs.addRow(TestRowSeq.makekv(key, qual2, val2)); + assertEquals(2, rs.size()); + + final byte[] qual3 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x37 }; + final byte[] val3 = Bytes.fromLong(6L); + final byte[] qual4 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x47 }; + final byte[] val4 = Bytes.fromLong(7L); + final byte[] row2 = { 0, 0, 1, 0x50, (byte)0xE2, 0x35, 0x10, 0, 0, 1, 0, 0, 2 }; + rs.addRow(new KeyValue(row2, TestRowSeq.FAMILY, qual3, val3)); + rs.addRow(new KeyValue(row2, TestRowSeq.FAMILY, qual4, val4)); + } + + @Test (expected = IllegalStateException.class) + public void addRowNotSet() throws Exception { + final byte[] qual1 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_sum); + rs.addRow(TestRowSeq.makekv(key, qual1, val1)); + rs.addRow(TestRowSeq.makekv(key, qual2, val2)); + + } + + @Test (expected = IllegalDataException.class) + public void addRowMergeDifferentKey() throws Exception { + final byte[] qual1 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_sum); + rs.setRow(TestRowSeq.makekv(key, qual1, val1)); + rs.addRow(TestRowSeq.makekv(key, qual2, val2)); + assertEquals(2, rs.size()); + + final byte[] qual3 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x37 }; + final byte[] val3 = Bytes.fromLong(6L); + final byte[] qual4 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x47 }; + final byte[] val4 = Bytes.fromLong(7L); + final byte[] key2 = Arrays.copyOf(key, key.length); + key2[key2.length - 1] = 3; + rs.addRow(TestRowSeq.makekv(key2, qual3, val3)); + rs.addRow(TestRowSeq.makekv(key2, qual4, val4)); + } + + @Test (expected = IllegalDataException.class) + public void addRowMergeDifferentKeyAndSalt() throws Exception { + final byte[] qual1 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_sum); + rs.setRow(TestRowSeq.makekv(key, qual1, val1)); + rs.addRow(TestRowSeq.makekv(key, qual2, val2)); + assertEquals(2, rs.size()); + + final byte[] qual3 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x37 }; + final byte[] val3 = Bytes.fromLong(6L); + final byte[] qual4 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x47 }; + final byte[] val4 = Bytes.fromLong(7L); + final byte[] key2 = Arrays.copyOf(key, key.length); + key2[0] = 2; + key2[key2.length - 1] = 3; + rs.addRow(TestRowSeq.makekv(key2, qual3, val3)); + rs.addRow(TestRowSeq.makekv(key2, qual4, val4)); + } + + @Test (expected = IllegalDataException.class) + public void addRowMergeDifferentTime() throws Exception { + final byte[] qual1 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_sum); + rs.setRow(TestRowSeq.makekv(key, qual1, val1)); + rs.addRow(TestRowSeq.makekv(key, qual2, val2)); + assertEquals(2, rs.size()); + + final byte[] qual3 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x37 }; + final byte[] val3 = Bytes.fromLong(6L); + final byte[] qual4 = { 0x73, 0x75, 0x6D, 0x3A, 0x00, 0x47 }; + final byte[] val4 = Bytes.fromLong(7L); + final byte[] key2 = Arrays.copyOf(key, key.length); + key2[7] = 3; + rs.addRow(TestRowSeq.makekv(key2, qual3, val3)); + rs.addRow(TestRowSeq.makekv(key2, qual4, val4)); + } + + @Test + public void timestamp() throws Exception { + final KeyValue kv = getRollupKeyValue(key,1356998400000L, 7L, + rollup_config.getIdForAggregator("max"), rollup_query_sum_mimmax); + + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_sum_mimmax); + rs.setRow(kv); + + assertEquals(1, rs.size()); + final SeekableView it = rs.iterator(); + assertTrue(it.hasNext()); + DataPoint dp = it.next(); + assertTrue(dp.isInteger()); + assertEquals(7, dp.longValue()); + assertEquals(1356998400000L, dp.timestamp()); + assertEquals(1, dp.valueCount()); + assertFalse(it.hasNext()); + } + + // NOTE: many of the tests below also test RollupSeq.size() + @Test + public void rollup10m() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_sum); + rs.setRow(getRollupKeyValue(key, 1420070400, 1L, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 2L, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071600, 3L, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 4L, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072800, 5L, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + + assertEquals(5, rs.size()); + final SeekableView it = rs.iterator(); + long value = 1; + long ts = 1420070400000L; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(value, dp.longValue()); + assertEquals(1, dp.valueCount()); + ++value; + ts += 600000; + } + } + + @Test + public void rollup10mDouble() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_sum); + rs.setRow(getRollupKeyValue(key, 1420070400, 0.25, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 0.50, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071600, 0.75, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 1.00, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072800, 1.25, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + + assertEquals(5, rs.size()); + final SeekableView it = rs.iterator(); + double value = 0.25; + long ts = 1420070400000L; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(value, dp.doubleValue(), 0.0001); + assertEquals(1, dp.valueCount()); + value += 0.25; + ts += 600000; + } + } + + @Test + public void rollup10mFloat() throws Exception { + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_sum); + rs.setRow(getRollupKeyValue(key, 1420070400, 10.25F, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 10.75F, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071600, 11.25F, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 11.75F, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072800, 12.25F, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + + assertEquals(5, rs.size()); + final SeekableView it = rs.iterator(); + double value = 10.25; + long ts = 1420070400000L; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(value, dp.doubleValue(), 0.0001); + assertEquals(1, dp.valueCount()); + value += 0.50; + ts += 600000; + } + } + + @Test + public void rollup10mMixFloatAndLong() throws Exception { + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_sum); + rs.setRow(getRollupKeyValue(key, 1420070400, 20, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 10.50F, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071600, 21, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 11.50F, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072800, 22, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420073400, 12.50F, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + + assertEquals(6, rs.size()); + final SeekableView it = rs.iterator(); + double dvalue = 10.50; + long lvalue = 20; + long ts = 1420070400000L; + boolean toggle = false; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + if (dp.isInteger()) { + assertEquals(lvalue, dp.longValue()); + } else { + assertEquals(dvalue, dp.doubleValue(), 0.0001); + } + assertEquals(1, dp.valueCount()); + if (toggle) { + dvalue += 1; + } else { + ++lvalue; + } + toggle = !toggle; + ts += 600000; + } + } + + @Test + public void rollupAvg10mWithCount() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_avg); + rs.setRow(getRollupKeyValue(key, 1420070400, 20, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420070400, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071000, 21, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071600, 22, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071600, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420072200, 23, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + + assertEquals(4, rs.size()); + final SeekableView it = rs.iterator(); + long value = 20; + long ts = 1420070400000L; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.longValue()); + assertEquals(2, dp.valueCount()); + ++value; + ts += 600000; + } + } + + @Test + public void rollupAvg10mWithCountFirst() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_avg); + rs.setRow(getRollupKeyValue(key, 1420070400, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420070400, 20, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071000, 21, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071600, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071600, 22, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420072200, 23, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + + assertEquals(4, rs.size()); + final SeekableView it = rs.iterator(); + long value = 20; + long ts = 1420070400000L; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.longValue()); + assertEquals(2, dp.valueCount()); + ++value; + ts += 600000; + } + } + + @Test + public void rollupAvg10mMissingCount() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_avg); + rs.setRow(getRollupKeyValue(key, 1420070400, 20, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 21, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071600, 22, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 23, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + + assertEquals(0, rs.size()); + final SeekableView it = rs.iterator(); + assertFalse(it.hasNext()); + } + + @Test + public void rollupAvg10mSkipFirstCount() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_avg); + rs.setRow(getRollupKeyValue(key, 1420070400, 20, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + //rs.addRow(getRollupKeyValue(key, 1420070400, 2, + // rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071000, 21, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071600, 22, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071600, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420072200, 23, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + + assertEquals(3, rs.size()); + final SeekableView it = rs.iterator(); + long value = 21; + long ts = 1420071000000L; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.longValue()); + assertEquals(2, dp.valueCount()); + ++value; + ts += 600000; + } + } + + @Test + public void rollupAvg10mSkipLastCount() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_avg); + rs.setRow(getRollupKeyValue(key, 1420070400, 20, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420070400, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071000, 21, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071600, 22, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071600, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420072200, 23, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + //rs.addRow(getRollupKeyValue(key, 1420072200, 2, + // rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + + assertEquals(3, rs.size()); + final SeekableView it = rs.iterator(); + long value = 20; + long ts = 1420070400000L; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.longValue()); + assertEquals(2, dp.valueCount()); + ++value; + ts += 600000; + } + } + + @Test + public void rollupAvg10mSkipMiddleCount() throws Exception { + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_avg); + rs.setRow(getRollupKeyValue(key, 1420070400, 20, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420070400, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071000, 21, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + //rs.addRow(getRollupKeyValue(key, 1420071000, 2, + // rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071600, 22, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071600, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420072200, 23, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + + assertEquals(3, rs.size()); + final SeekableView it = rs.iterator(); + long value = 20; + long ts = 1420070400000L; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.longValue()); + assertEquals(2, dp.valueCount()); + if (ts == 1420070400000L) { + ts = 1420071600000L; + value += 2; + } else { + ts += 600000; + ++value; + } + } + } + + @Test + public void rollupAvg10mMissingSum() throws Exception { + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_avg); + rs.setRow(getRollupKeyValue(key, 1420070400, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071000, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071600, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420072200, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + + assertEquals(0, rs.size()); + final SeekableView it = rs.iterator(); + assertFalse(it.hasNext()); + } + + public void rollupAvg10mSkipFirstSum() throws Exception { + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_avg); + //rs.setRow(getRollupKeyValue(key, 1420070400, 20, + // rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420070400, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071000, 21, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071600, 22, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071600, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420072200, 23, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + + assertEquals(3, rs.size()); + final SeekableView it = rs.iterator(); + long value = 21; + long ts = 1420071000000L; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.longValue()); + assertEquals(2, dp.valueCount()); + ++value; + ts += 600000; + } + } + + @Test + public void rollupAvg10mSkipLastSum() throws Exception { + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_avg); + rs.setRow(getRollupKeyValue(key, 1420070400, 20, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420070400, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071000, 21, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071600, 22, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071600, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + //rs.addRow(getRollupKeyValue(key, 1420072200, 23, + // rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + + assertEquals(3, rs.size()); + final SeekableView it = rs.iterator(); + long value = 20; + long ts = 1420070400000L; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.longValue()); + assertEquals(2, dp.valueCount()); + ++value; + ts += 600000; + } + } + + @Test + public void rollupAvg10mSkipMiddleSum() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_avg); + rs.setRow(getRollupKeyValue(key, 1420070400, 20, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420070400, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + //rs.addRow(getRollupKeyValue(key, 1420071000, 21, + // rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071600, 22, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071600, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420072200, 23, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + + assertEquals(3, rs.size()); + final SeekableView it = rs.iterator(); + long value = 20; + long ts = 1420070400000L; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertEquals(value, dp.longValue()); + assertEquals(2, dp.valueCount()); + if (ts == 1420070400000L) { + ts = 1420071600000L; + value += 2; + } else { + ts += 600000; + ++value; + } + } + } + + @Test + public void rollupAvg10mUnaligned() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_avg); + rs.setRow(getRollupKeyValue(key, 1420070400, 20, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + //rs.addRow(getRollupKeyValue(key, 1420070400, 2, + // rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + //rs.addRow(getRollupKeyValue(key, 1420071000, 21, + // rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071600, 22, + rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + //rs.addRow(getRollupKeyValue(key, 1420071600, 2, + // rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + //rs.addRow(getRollupKeyValue(key, 1420072200, 23, + // rollup_config.getIdForAggregator("SUM"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + + assertEquals(0, rs.size()); + final SeekableView it = rs.iterator(); + assertFalse(it.hasNext()); + } + + @Test + public void endOfArrayDivergence() throws Exception { + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_avg); + + Internal.setBaseTime(key, 1420070400); + + int[]indices = new int[4]; + byte[] b = new byte[] { 0, 11, 0, 27, 0, 43, 0, 59, 0, 75, 0, 91, 0, 107, 0, 123, 0, -117, 0, -101, 0, -85, 0, -69, 0, -53, 0, -37, 0, -21, 0, -5, 1, 11, 1, 27, 1, 43, 1, 59, 1, 75, 1, 91, 1, 123, 0, 0 }; + Whitebox.setInternalState(rs, "key", key); + Whitebox.setInternalState(rs, "qualifiers", b); + Whitebox.setInternalState(rs, "indices", indices); + indices[0] = b.length; + Whitebox.setInternalState(rs, "values", new byte[] { 67, 10, 71, -82, 66, -10, -108, 123, 66, -52, -52, -51, 66, -56, 97, 72, 66, -78, 5, 31, 66, -79, -31, 72, 66, 101, 0, 0, 67, 1, 99, -41, 66, -17, -77, 51, 66, -48, 10, 61, 66, -48, -118, 61, 66, -77, -47, -20, 66, -79, 81, -20, 66, -33, -118, 61, 67, 6, -31, 72, 66, -22, -26, 102, 66, -51, -103, -102, 66, -65, 51, 51, 66, -73, 112, -92, 66, -71, -47, -20, 66, -25, -6, -31, 67, 10, 10, 61, 65, -115, 92, 41, 0, 0, 0, 0 }); + b = new byte[] { 0, 11, 0, 27, 0, 43, 0, 59, 0, 75, 0, 91, 0, 107, 0, 123, 0, -117, 0, -101, 0, -85, 0, -69, 0, -53, 0, -37, 0, -21, 0, -5, 1, 11, 1, 27, 1, 43, 1, 59, 1, 75, 1, 91, 1, 107, 0, 0 }; + indices[2] = b.length; + Whitebox.setInternalState(rs, "count_qualifiers", b); + Whitebox.setInternalState(rs, "count_values", new byte[] { 66, 112, 0, 0, 66, 112, 0, 0, 66, 112, 0, 0, 66, 112, 0, 0, 66, 112, 0, 0, 66, 112, 0, 0, 65, -16, 0, 0, 66, 100, 0, 0, 66, 112, 0, 0, 66, 112, 0, 0, 66, 112, 0, 0, 66, 112, 0, 0, 66, 112, 0, 0, 66, 112, 0, 0, 66, 112, 0, 0, 66, 112, 0, 0, 66, 112, 0, 0, 66, 112, 0, 0, 66, 112, 0, 0, 66, 112, 0, 0, 66, 112, 0, 0, 66, 112, 0, 0, 65, 32, 0, 0, 0, 0, 0, 0 }); + + final SeekableView it = rs.iterator(); + int count = 0; + while (it.hasNext()) { + ++count; + it.next(); + } + assertEquals(22, count); + } + + @Test + public void arrayOverflowAvoidance() throws Exception { + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_avg); + + Internal.setBaseTime(key, 1420070400); + + final int[]indices = new int[4]; + byte[] b = new byte[] { 0, 11, 0, 27, 0, 43, 0, 59, 0, 75, 0, 91, 0, 107, 0, 123, 0, -117, 0, -101, 0, -85, 0, -69, 0, -53, 0, -37, 0, -21, 0, -5, 1, 11, 1, 27, 1, 43, 1, 59, 1, 75, 1, 91, 1, 107, 1, 123 }; + Whitebox.setInternalState(rs, "key", key); + Whitebox.setInternalState(rs, "qualifiers", b); + Whitebox.setInternalState(rs, "indices", indices); + indices[0] = b.length; + Whitebox.setInternalState(rs, "values", new byte[] { 75, 29, -83, 11, 75, 28, -40, -13, 75, 23, -115, 8, 75, 9, -84, 94, 75, 11, 55, -77, 75, 12, -115, 111, 75, 8, -21, -48, 75, 12, 66, 76, 75, 8, 118, -48, 75, 11, -65, 121, 75, 26, 64, -100, 75, 69, -15, -114, 75, 95, 85, -64, 75, 109, 28, 40, 75, 118, 100, -32, 75, 110, -119, 20, 75, 100, -71, -94, 75, 100, -94, 50, 75, 90, -53, 44, 75, 90, 47, 70, 75, 82, 80, 40, 75, 66, 106, 121, 75, 57, -102, -34, 75, 53, 69, 4 }); + indices[2] = b.length; + Whitebox.setInternalState(rs, "count_qualifiers", b); + Whitebox.setInternalState(rs, "count_values", new byte[] { 65, 64, 0, 0, 65, 64, 0, 0, 65, 64, 0, 0, 65, 64, 0, 0, 65, 64, 0, 0, 65, 64, 0, 0, 65, 64, 0, 0, 65, 64, 0, 0, 65, 64, 0, 0, 65, 64, 0, 0, 65, 48, 0, 0, 65, 64, 0, 0, 65, 64, 0, 0, 65, 64, 0, 0, 65, 64, 0, 0, 65, 64, 0, 0, 65, 64, 0, 0, 65, 64, 0, 0, 65, 64, 0, 0, 65, 64, 0, 0, 65, 64, 0, 0, 65, 64, 0, 0, 65, 64, 0, 0, 65, 64, 0, 0 }); + + final SeekableView it = rs.iterator(); + int count = 0; + while (it.hasNext()) { + ++count; + it.next(); + } + assertEquals(24, count); + } + + @Test + public void rollup1hLong() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_1h_sum); + rs.setRow(getRollupKeyValue(key, 1420070400, 1L, + rollup_config.getIdForAggregator("SUM"), rollup_query_1h_sum)); + rs.addRow(getRollupKeyValue(key, 1420074000, 2L, + rollup_config.getIdForAggregator("SUM"), rollup_query_1h_sum)); + rs.addRow(getRollupKeyValue(key, 1420077600, 3L, + rollup_config.getIdForAggregator("SUM"), rollup_query_1h_sum)); + rs.addRow(getRollupKeyValue(key, 1420081200, 4L, + rollup_config.getIdForAggregator("SUM"), rollup_query_1h_sum)); + rs.addRow(getRollupKeyValue(key, 1420084800, 5L, + rollup_config.getIdForAggregator("SUM"), rollup_query_1h_sum)); + + assertEquals(5, rs.size()); + final SeekableView it = rs.iterator(); + long value = 1; + long ts = 1420070400000L; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(value, dp.longValue()); + assertEquals(1, dp.valueCount()); + ++value; + ts += 3600000; + } + } + + @Test + public void rollup1hFloat() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_1h_sum); + rs.setRow(getRollupKeyValue(key, 1420070400, 0.25, + rollup_config.getIdForAggregator("SUM"), rollup_query_1h_sum)); + rs.addRow(getRollupKeyValue(key, 1420074000, 0.5, + rollup_config.getIdForAggregator("SUM"), rollup_query_1h_sum)); + rs.addRow(getRollupKeyValue(key, 1420077600, 0.75, + rollup_config.getIdForAggregator("SUM"), rollup_query_1h_sum)); + rs.addRow(getRollupKeyValue(key, 1420081200, 1.0, + rollup_config.getIdForAggregator("SUM"), rollup_query_1h_sum)); + rs.addRow(getRollupKeyValue(key, 1420084800, 1.25, + rollup_config.getIdForAggregator("SUM"), rollup_query_1h_sum)); + + assertEquals(5, rs.size()); + final SeekableView it = rs.iterator(); + double value = 0.25; + long ts = 1420070400000L; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertFalse(dp.isInteger()); + assertEquals(value, dp.doubleValue(), 0.0001); + assertEquals(1, dp.valueCount()); + value += 0.25; + ts += 3600000; + } + } + + @Test + public void rollup1hLongWithCount() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_1h_avg); + rs.setRow(getRollupKeyValue(key, 1420070400, 1L, + rollup_config.getIdForAggregator("SUM"), rollup_query_1h_sum)); + rs.addRow(getRollupKeyValue(key, 1420070400, 2L, + rollup_config.getIdForAggregator("count"), rollup_query_1h_count)); + rs.addRow(getRollupKeyValue(key, 1420074000, 2L, + rollup_config.getIdForAggregator("SUM"), rollup_query_1h_sum)); + rs.addRow(getRollupKeyValue(key, 1420074000, 2L, + rollup_config.getIdForAggregator("count"), rollup_query_1h_count)); + rs.addRow(getRollupKeyValue(key, 1420077600, 3L, + rollup_config.getIdForAggregator("SUM"), rollup_query_1h_sum)); + rs.addRow(getRollupKeyValue(key, 1420077600, 2L, + rollup_config.getIdForAggregator("count"), rollup_query_1h_count)); + rs.addRow(getRollupKeyValue(key, 1420081200, 4L, + rollup_config.getIdForAggregator("SUM"), rollup_query_1h_sum)); + rs.addRow(getRollupKeyValue(key, 1420081200, 2L, + rollup_config.getIdForAggregator("count"), rollup_query_1h_count)); + rs.addRow(getRollupKeyValue(key, 1420084800, 5L, + rollup_config.getIdForAggregator("SUM"), rollup_query_1h_sum)); + rs.addRow(getRollupKeyValue(key, 1420084800, 2L, + rollup_config.getIdForAggregator("count"), rollup_query_1h_count)); + + assertEquals(5, rs.size()); + final SeekableView it = rs.iterator(); + long value = 1; + long ts = 1420070400000L; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(value, dp.longValue()); + assertEquals(2, dp.valueCount()); + ++value; + ts += 3600000; + } + } + + @Test + public void rollup1hLongWithCountWithDoubles() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_1h_avg); + rs.setRow(getRollupKeyValue(key, 1420070400, 1L, + rollup_config.getIdForAggregator("sum"), rollup_query_1h_sum)); + rs.addRow(getRollupKeyValue(key, 1420070400, 2L, + rollup_config.getIdForAggregator("count"), rollup_query_1h_count)); + rs.addRow(getRollupKeyValue(key, 1420074000, 2L, + rollup_config.getIdForAggregator("sum"), rollup_query_1h_sum)); + rs.addRow(getRollupKeyValue(key, 1420074000, 2D, + rollup_config.getIdForAggregator("count"), rollup_query_1h_count)); + rs.addRow(getRollupKeyValue(key, 1420077600, 3L, + rollup_config.getIdForAggregator("sum"), rollup_query_1h_sum)); + rs.addRow(getRollupKeyValue(key, 1420077600, 2D, + rollup_config.getIdForAggregator("count"), rollup_query_1h_count)); + rs.addRow(getRollupKeyValue(key, 1420081200, 4L, + rollup_config.getIdForAggregator("sum"), rollup_query_1h_sum)); + rs.addRow(getRollupKeyValue(key, 1420081200, 2D, + rollup_config.getIdForAggregator("count"), rollup_query_1h_count)); + rs.addRow(getRollupKeyValue(key, 1420084800, 5L, + rollup_config.getIdForAggregator("sum"), rollup_query_1h_sum)); + rs.addRow(getRollupKeyValue(key, 1420084800, 2L, + rollup_config.getIdForAggregator("count"), rollup_query_1h_count)); + + assertEquals(5, rs.size()); + final SeekableView it = rs.iterator(); + long value = 1; + long ts = 1420070400000L; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(value, dp.longValue()); + assertEquals(2, dp.valueCount()); + ++value; + ts += 3600000; + } + } + + @Test + public void rollup10mSeekTop() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_sum); + rs.setRow(getRollupKeyValue(key, 1420070400, 1L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 2L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071600, 3L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 4L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072800, 5L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420073400, 6L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420074000, 7L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420074600, 8L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + + assertEquals(8, rs.size()); + final SeekableView it = rs.iterator(); + it.seek(1420070400000L); + long value = 1; + long ts = 1420070400000L; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(value, dp.longValue()); + assertEquals(1, dp.valueCount()); + ++value; + ts += 600000; + } + } + + @Test + public void rollup10mSeek() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_sum); + rs.setRow(getRollupKeyValue(key, 1420070400, 1L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 2L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071600, 3L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 4L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072800, 5L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420073400, 6L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420074000, 7L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420074600, 8L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + + assertEquals(8, rs.size()); + final SeekableView it = rs.iterator(); + it.seek(1420072200000L); + long value = 4; + long ts = 1420072200000L; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(value, dp.longValue()); + assertEquals(1, dp.valueCount()); + ++value; + ts += 600000; + } + } + + @Test + public void rollup10mSeekOOB() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_sum); + rs.setRow(getRollupKeyValue(key, 1420070400, 1L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 2L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071600, 3L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 4L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072800, 5L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420073400, 6L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420074000, 7L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420074600, 8L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + + assertEquals(8, rs.size()); + final SeekableView it = rs.iterator(); + it.seek(1420075200000L); + assertFalse(it.hasNext()); + } + + @Test + public void rollup10mSeekSeconds() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_sum); + rs.setRow(getRollupKeyValue(key, 1420070400, 1L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 2L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071600, 3L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 4L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072800, 5L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420073400, 6L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420074000, 7L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420074600, 8L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + + assertEquals(8, rs.size()); + final SeekableView it = rs.iterator(); + it.seek(1420072200L); + long value = 4; + long ts = 1420072200000L; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(value, dp.longValue()); + assertEquals(1, dp.valueCount()); + ++value; + ts += 600000; + } + } + + @Test + public void rollup10mSeekUnaligned() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_sum); + rs.setRow(getRollupKeyValue(key, 1420070400, 1L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 2L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071600, 3L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 4L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072800, 5L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420073400, 6L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420074000, 7L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420074600, 8L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + + assertEquals(8, rs.size()); + final SeekableView it = rs.iterator(); + it.seek(1420072123456L); + long value = 4; + long ts = 1420072200000L; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(value, dp.longValue()); + assertEquals(1, dp.valueCount()); + ++value; + ts += 600000; + } + } + + @Test + public void rollup10mAvgSeekTopAligned() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_avg); + rs.setRow(getRollupKeyValue(key, 1420070400, 20, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420070400, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071000, 21, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071600, 22, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071600, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420072200, 23, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + + assertEquals(4, rs.size()); + final SeekableView it = rs.iterator(); + it.seek(1420070400000L); + long value = 20; + long ts = 1420070400000L; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(value, dp.longValue()); + assertEquals(2, dp.valueCount()); + ++value; + ts += 600000; + } + assertEquals(24, value); + } + + @Test + public void rollup10mAvgSeekTopUnaligned() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_avg); + rs.setRow(getRollupKeyValue(key, 1420070400, 20, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420070400, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + //rs.addRow(getRollupKeyValue(key, 1420071000, 21, rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071600, 22, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + //rs.addRow(getRollupKeyValue(key, 1420071600, 2, rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420072200, 23, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + + assertEquals(2, rs.size()); + final SeekableView it = rs.iterator(); + it.seek(1420070400000L); + DataPoint dp = it.next(); + assertEquals(1420070400000L, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(20, dp.longValue()); + assertEquals(2, dp.valueCount()); + dp = it.next(); + assertEquals(1420072200000L, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(23, dp.longValue()); + assertEquals(2, dp.valueCount()); + assertFalse(it.hasNext()); + } + + @Test + public void rollup10mAvgSeekTopUnalignedMissingTop() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_avg); + rs.setRow(getRollupKeyValue(key, 1420070400, 20, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + //rs.addRow(getRollupKeyValue(key, 1420070400, 2, rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071000, 21, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071600, 22, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071600, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420072200, 23, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + + assertEquals(3, rs.size()); + final SeekableView it = rs.iterator(); + it.seek(1420070400000L); + long value = 21; + long ts = 1420071000000L; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(value, dp.longValue()); + assertEquals(2, dp.valueCount()); + ++value; + ts += 600000; + } + assertEquals(24, value); + } + + @Test + public void rollup10mAvgSeekAligned() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_avg); + rs.setRow(getRollupKeyValue(key, 1420070400, 20, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420070400, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071000, 21, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071600, 22, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071600, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420072200, 23, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + + assertEquals(4, rs.size()); + final SeekableView it = rs.iterator(); + it.seek(1420071600); + long value = 22; + long ts = 1420071600000L; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(value, dp.longValue()); + assertEquals(2, dp.valueCount()); + ++value; + ts += 600000; + } + assertEquals(24, value); + } + + @Test + public void rollup10mAvgSeekUnaligned() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_avg); + rs.setRow(getRollupKeyValue(key, 1420070400, 20, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420070400, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + //rs.addRow(getRollupKeyValue(key, 1420071000, 21, rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071600, 22, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + //rs.addRow(getRollupKeyValue(key, 1420071600, 2, rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420072200, 23, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + + assertEquals(2, rs.size()); + final SeekableView it = rs.iterator(); + it.seek(1420071600); + long value = 23; + long ts = 1420072200000L; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(value, dp.longValue()); + assertEquals(2, dp.valueCount()); + ++value; + ts += 600000; + } + assertEquals(24, value); + } + + @Test + public void rollup10mAvgSeekUnalignedEmpty() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_avg); + rs.setRow(getRollupKeyValue(key, 1420070400, 20, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + //rs.addRow(getRollupKeyValue(key, 1420070400, 2, rollup_query_10m_count)); + //rs.addRow(getRollupKeyValue(key, 1420071000, 21, rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071600, 22, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + //rs.addRow(getRollupKeyValue(key, 1420071600, 2, rollup_query_10m_count)); + //rs.addRow(getRollupKeyValue(key, 1420072200, 23, rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + + assertEquals(0, rs.size()); + final SeekableView it = rs.iterator(); + it.seek(1420071600); + assertFalse(it.hasNext()); + } + + @Test + public void rollup10mAvgSeekTopAlignedOOB() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_avg); + rs.setRow(getRollupKeyValue(key, 1420070400, 20, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420070400, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071000, 21, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071600, 22, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071600, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420072200, 23, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + + assertEquals(4, rs.size()); + final SeekableView it = rs.iterator(); + it.seek(1420075200000L); + assertFalse(it.hasNext()); + } + + @Test + public void rollup10mAvgSeekTopUnalignedOOB() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_avg); + rs.setRow(getRollupKeyValue(key, 1420070400, 20, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420070400, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + //rs.addRow(getRollupKeyValue(key, 1420071000, 21, rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071600, 22, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + //rs.addRow(getRollupKeyValue(key, 1420071600, 2, rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420072200, 23, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + + assertEquals(2, rs.size()); + final SeekableView it = rs.iterator(); + it.seek(1420075200000L); + assertFalse(it.hasNext()); + } + + @Test + public void rollup10mAvgSeekTopUnalignedEmptyOOB() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_avg); + rs.setRow(getRollupKeyValue(key, 1420070400, 20, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + //rs.addRow(getRollupKeyValue(key, 1420070400, 2, rollup_query_10m_count)); + //rs.addRow(getRollupKeyValue(key, 1420071000, 21, rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + rs.addRow(getRollupKeyValue(key, 1420071600, 22, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + //rs.addRow(getRollupKeyValue(key, 1420071600, 2, rollup_query_10m_count)); + //rs.addRow(getRollupKeyValue(key, 1420072200, 23, rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420072200, 2, + rollup_config.getIdForAggregator("count"), rollup_query_10m_count)); + + assertEquals(0, rs.size()); + final SeekableView it = rs.iterator(); + it.seek(1420075200000L); + assertFalse(it.hasNext()); + } + + @Test + public void rollup10mTimestamp() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_10m_sum); + rs.setRow(getRollupKeyValue(key, 1420070400, 1L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071000, 2L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + rs.addRow(getRollupKeyValue(key, 1420071600, 3L, + rollup_config.getIdForAggregator("sum"), rollup_query_10m_sum)); + + assertEquals(1420070400000L, rs.timestamp(0)); + assertEquals(1420071000000L, rs.timestamp(1)); + assertEquals(1420071600000L, rs.timestamp(2)); + + try { + assertEquals(1420071600000L, rs.timestamp(3)); + fail("Excpected an IndexOutOfBoundsException"); + } catch (IndexOutOfBoundsException e) { } + + try { + assertEquals(1420071600000L, rs.timestamp(-1)); + fail("Excpected an IndexOutOfBoundsException"); + } catch (IndexOutOfBoundsException e) { } + } + + @Test + public void rollupRowWithDifferentAggregators() throws Exception { + + Internal.setBaseTime(key, 1420070400); + final RollupSeq rs = new RollupSeq(tsdb, rollup_query_1h_avg_group_by_sum); + rs.setRow(getRollupKeyValue(key, 1420070400, 1L, + rollup_config.getIdForAggregator("sum"), rollup_query_1h_avg_group_by_sum)); + rs.addRow(getRollupKeyValue(key, 1420074000, 2L, + rollup_config.getIdForAggregator("sum"), rollup_query_1h_avg_group_by_sum)); + rs.addRow(getRollupKeyValue(key, 1420077600, 3L, + rollup_config.getIdForAggregator("sum"), rollup_query_1h_avg_group_by_sum)); + rs.addRow(getRollupKeyValue(key, 1420070400, 1L, + rollup_config.getIdForAggregator("count"), rollup_query_1h_avg_group_by_sum)); + rs.addRow(getRollupKeyValue(key, 1420074000, 2L, + rollup_config.getIdForAggregator("count"), rollup_query_1h_avg_group_by_sum)); + rs.addRow(getRollupKeyValue(key, 1420077600, 3L, + rollup_config.getIdForAggregator("count"), rollup_query_1h_avg_group_by_sum)); + + assertEquals(3, rs.size()); + final SeekableView it = rs.iterator(); + it.seek(1420070400L); + long value = 1; + long ts = 1420070400000L; + while (it.hasNext()) { + final DataPoint dp = it.next(); + assertEquals(ts, dp.timestamp()); + assertTrue(dp.isInteger()); + assertEquals(value, dp.longValue()); + assertEquals(value, dp.valueCount()); + ++value; + ts += 60 * 60 * 1000; + } + } + + private static KeyValue getRollupKeyValue(final byte[] key, + final long timestamp, + final long value, + final int agg_id, + final RollupQuery rollup_query) { + + final byte[] val = Internal.vleEncodeLong(value); + final short flags = (short) (val.length - 1); // Just the length. + return new KeyValue(key, TestRowSeq.FAMILY, getQualifier(timestamp, flags, + agg_id, rollup_query), val); + } + + private static KeyValue getRollupKeyValue(final byte[] key, + final long timestamp, + final float value, + final int agg_id, + final RollupQuery rollup_query) { + + final short flags = Const.FLAG_FLOAT | 0x3; // A float stored on 4 bytes. + final byte[] val = Bytes.fromInt(Float.floatToRawIntBits(value)); + return new KeyValue(key, TestRowSeq.FAMILY, getQualifier(timestamp, flags, + agg_id, rollup_query), val); + } + + private static KeyValue getRollupKeyValue(final byte[] key, + final long timestamp, + final double value, + final int agg_id, + final RollupQuery rollup_query) { + + final short flags = Const.FLAG_FLOAT | 0x7; // A double stored on 8 bytes. + final byte[] val = Bytes.fromLong(Double.doubleToRawLongBits(value)); + return new KeyValue(key, TestRowSeq.FAMILY, getQualifier(timestamp, flags, + agg_id, rollup_query), val); + } + + private static byte[] getQualifier(final long timestamp, + final short flags, + final int agg_id, + final RollupQuery rollup_query) { + + final int base_time = RollupUtils.getRollupBasetime(timestamp, + rollup_query.getRollupInterval()); + return RollupUtils.buildRollupQualifier(timestamp, base_time, flags, + agg_id, rollup_query.getRollupInterval()); + } +} diff --git a/test/rollup/TestRollupUtils.java b/test/rollup/TestRollupUtils.java new file mode 100644 index 0000000000..b207fb2e50 --- /dev/null +++ b/test/rollup/TestRollupUtils.java @@ -0,0 +1,1140 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2010-2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.rollup; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.Before; +import org.junit.Test; + +import net.opentsdb.core.Const; + +public class TestRollupUtils { + private static final String temporal_table = "tsdb-rollup-10m"; + private static final String groupby_table = "tsdb-rollup-agg-10m"; + + private RollupInterval hour_interval; + private RollupInterval tenmin_oneday; + private RollupInterval month_interval; + + @Before + public void before() { + hour_interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("1s") + .setRowSpan("1h") + .build(); + tenmin_oneday = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("10m") + .setRowSpan("1d") + .build(); + month_interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("1h") + .setRowSpan("1n") + .build(); + } + + @Test + public void getRollupBasetimeHourSecondsTop() throws Exception { + // Thu, 06 Jun 2013 15:00:00 GMT + assertEquals(1370530800, RollupUtils.getRollupBasetime(1370530800L, + hour_interval)); + } + + @Test + public void getRollupBasetimeHourMilliSecondsTop() throws Exception { + // Thu, 06 Jun 2013 15:00:00.154 GMT + assertEquals(1370530800, RollupUtils.getRollupBasetime(1370530800154L, + hour_interval)); + } + + @Test + public void getRollupBasetimeHourSecondsMid() throws Exception { + // Thu, 06 Jun 2013 15:35:25 GMT + assertEquals(1370530800, RollupUtils.getRollupBasetime(1370532925L, + hour_interval)); + } + + @Test + public void getRollupBasetimeHourMilliSecondsMid() throws Exception { + // Thu, 06 Jun 2013 15:35:25.154 GMT + assertEquals(1370530800, RollupUtils.getRollupBasetime(1370532925154L, + hour_interval)); + } + + @Test + public void getRollupBasetimeHourSecondsEnd() throws Exception { + // Thu, 06 Jun 2013 15:59:59 GMT + assertEquals(1370530800, RollupUtils.getRollupBasetime(1370534399L, + hour_interval)); + } + + @Test + public void getRollupBasetimeHourMilliSecondsEnd() throws Exception { + // Thu, 06 Jun 2013 15:59:59.999 GMT + assertEquals(1370530800, RollupUtils.getRollupBasetime(1370534399999L, + hour_interval)); + } + + @Test + public void getRollupBasetime6HourSecondsTop() throws Exception { + // Thu, 06 Jun 2013 12:00:00 GMT + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("10m") + .setRowSpan("6h") + .build(); + assertEquals(1370520000, RollupUtils.getRollupBasetime(1370520000L, interval)); + } + + @Test + public void getRollupBasetime6HourSecondsMid() throws Exception { + // Thu, 06 Jun 2013 15:00:00 GMT + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("10m") + .setRowSpan("6h") + .build(); + assertEquals(1370520000, RollupUtils.getRollupBasetime(1370530800L, interval)); + } + + @Test + public void getRollupBasetime6HourSecondsEnd() throws Exception { + // Thu, 06 Jun 2013 23:59:59 GMT + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("10m") + .setRowSpan("6h") + .build(); + assertEquals(1370520000, RollupUtils.getRollupBasetime(1370541599L, interval)); + } + + @Test + public void getRollupBasetime2HourSecondsTop() throws Exception { + // Thu, 06 Jun 2013 12:00:00 GMT + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("10m") + .setRowSpan("2h") + .build(); + assertEquals(1370520000, RollupUtils.getRollupBasetime(1370520000L, interval)); + } + + @Test + public void getRollupBasetime2HourSecondsMid() throws Exception { + // Thu, 06 Jun 2013 13:01:00 GMT + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("10m") + .setRowSpan("2h") + .build(); + assertEquals(1370520000, RollupUtils.getRollupBasetime(1370523660L, interval)); + } + + @Test + public void getRollupBasetime2HourSecondsEnd() throws Exception { + // Thu, 06 Jun 2013 13:59:59 GMT + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("10m") + .setRowSpan("2h") + .build(); + assertEquals(1370520000, RollupUtils.getRollupBasetime(1370527199L, interval)); + } + + @Test + public void getRollupBasetimeDaySecondsTop() throws Exception { + // Thu, 06 Jun 2013 00:00:00 GMT + assertEquals(1370476800, RollupUtils.getRollupBasetime(1370476800L, + tenmin_oneday)); + } + + @Test + public void getRollupBasetimeDayMilliSecondsTop() throws Exception { + // Thu, 06 Jun 2013 00:00:00.154 GMT + assertEquals(1370476800, RollupUtils.getRollupBasetime(1370476800154L, + tenmin_oneday)); + } + + @Test + public void getRollupBasetimeDaySecondsMid() throws Exception { + // Thu, 06 Jun 2013 15:35:25 GMT + assertEquals(1370476800, RollupUtils.getRollupBasetime(1370532925L, + tenmin_oneday)); + } + + @Test + public void getRollupBasetimeDayMilliSecondsMid() throws Exception { + // Thu, 06 Jun 2013 15:35:25.154 GMT + assertEquals(1370476800, RollupUtils.getRollupBasetime(1370532925154L, + tenmin_oneday)); + } + + @Test + public void getRollupBasetimeDaySecondsEnd() throws Exception { + // Thu, 06 Jun 2013 23:59:59 GMT + assertEquals(1370476800, RollupUtils.getRollupBasetime(1370563199L, + tenmin_oneday)); + } + + @Test + public void getRollupBasetimeDayMilliSecondsEnd() throws Exception { + // Thu, 06 Jun 2013 23:59:59.999 GMT + assertEquals(1370476800, RollupUtils.getRollupBasetime(1370563199999L, + tenmin_oneday)); + } + + @Test + public void getRollupBasetimeMonthSecondsTop() throws Exception { + // Sat, 01 Jun 2013 00:00:00 GMT + assertEquals(1370044800, RollupUtils.getRollupBasetime(1370044800L, + month_interval)); + } + + @Test + public void getRollupBasetimeMonthMilliSecondsTop() throws Exception { + // Thu, 01 Jun 2013 00:00:00.154 GMT + assertEquals(1370044800, RollupUtils.getRollupBasetime(1370044800154L, + month_interval)); + } + + @Test + public void getRollupBasetimeMonthSecondsMid() throws Exception { + // Thu, 06 Jun 2013 15:35:25 GMT + assertEquals(1370044800, RollupUtils.getRollupBasetime(1370532925L, + month_interval)); + } + + @Test + public void getRollupBasetimeMonthMilliSecondsMid() throws Exception { + // Thu, 06 Jun 2013 15:35:25.154 GMT + assertEquals(1370044800, RollupUtils.getRollupBasetime(1370532925154L, + month_interval)); + } + + @Test + public void getRollupBasetimeMonthSecondsEnd30days() throws Exception { + // Thu, 30 Jun 2013 23:59:59 GMT + assertEquals(1370044800, RollupUtils.getRollupBasetime(1372636799L, + month_interval)); + } + + @Test + public void getRollupBasetimeMonthMilliSecondsEnd30days() throws Exception { + // Thu, 30 Jun 2013 23:59:59.999 GMT + assertEquals(1370044800, RollupUtils.getRollupBasetime(1372636799999L, + month_interval)); + } + + @Test + public void getRollupBasetimeMonthSecondsEnd31days() throws Exception { + // Wed, 31 Jul 2013 23:59:59 GMT + assertEquals(1372636800, RollupUtils.getRollupBasetime(1375315199L, + month_interval)); + } + + @Test + public void getRollupBasetimeMonthMilliSecondsEnd31days() throws Exception { + // Wed, 31 Jul 2013 23:59:59.999 GMT + assertEquals(1372636800, RollupUtils.getRollupBasetime(1375315199999L, + month_interval)); + } + + @Test + public void getRollupBasetimeMonthSecondsEndFebruary() throws Exception { + // Thu, 28 Feb 2013 23:59:59 GMT + assertEquals(1359676800, RollupUtils.getRollupBasetime(1362095999L, + month_interval)); + } + + @Test + public void getRollupBasetimeMonthMilliSecondsEndFebruary() throws Exception { + // Thu, 28 Feb 2013 23:59:59 GMT + assertEquals(1359676800, RollupUtils.getRollupBasetime(1362095999999L, + month_interval)); + } + + @Test + public void getRollupBasetimeMonthSecondsEndLeapFebruary() throws Exception { + // Wed, 29 Feb 2012 23:59:59 GMT + assertEquals(1328054400, RollupUtils.getRollupBasetime(1330559999L, + month_interval)); + } + + @Test + public void getRollupBasetimeMonthMilliSecondsEndLeapFebruary() throws Exception { + // Wed, 29 Feb 2012 23:59:59 GMT + assertEquals(1328054400, RollupUtils.getRollupBasetime(1330559999999L, + month_interval)); + } + + // NOTE: This is system dependent and leap seconds will usually just bump + // to the next month and overwrite any offset == 0 value there. If this unit + // test fails, that's actually a GOOD thing! + @Test + public void getRollupBasetimeMonthSecondsLeapSecond() throws Exception { + // Tue, 30 Jun 2015 23:59:60 GMT + assertEquals(1435708800, RollupUtils.getRollupBasetime(1435708800L, + month_interval)); + } + + @Test + public void getRollupBasetimeMonthSecondsLeapMilliSecond() throws Exception { + // Tue, 30 Jun 2015 23:59:60.154 GMT + assertEquals(1435708800, RollupUtils.getRollupBasetime(1435708800154L, + month_interval)); + } + + @Test + public void getRollupBasetimeYearSecondsTop() throws Exception { + // Tue, 01 Jan 2013 00:00:00 GMT + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("24h") + .setRowSpan("1y") + .build(); + assertEquals(1356998400, RollupUtils.getRollupBasetime(1356998400L, + interval)); + } + + @Test + public void getRollupBasetimeYearMilliSecondsTop() throws Exception { + // Tue, 01 Jan 2013 00:00:00.154 GMT + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("24h") + .setRowSpan("1y") + .build(); + assertEquals(1356998400, RollupUtils.getRollupBasetime(1356998400154L, + interval)); + } + + @Test + public void getRollupBasetimeYearSecondsMid() throws Exception { + // Thu, 06 Jun 2013 15:35:25 GMT + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("24h") + .setRowSpan("1y") + .build(); + assertEquals(1356998400, RollupUtils.getRollupBasetime(1370532925L, + interval)); + } + + @Test + public void getRollupBasetimeYearMilliSecondsMid() throws Exception { + // Thu, 06 Jun 2013 15:35:25.154 GMT + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("24h") + .setRowSpan("1y") + .build(); + assertEquals(1356998400, RollupUtils.getRollupBasetime(1370532925154L, + interval)); + } + + @Test + public void getRollupBasetimeYearSecondsEnd() throws Exception { + // Tue, 31 Dec 2013 23:59:59 GMT + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("24h") + .setRowSpan("1y") + .build(); + assertEquals(1356998400, RollupUtils.getRollupBasetime(1388534399L, + interval)); + } + + @Test + public void getRollupBasetimeYearMilliSecondsEnd() throws Exception { + // Tue, 31 Dec 2013 23:59:59.999 GMT + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("24h") + .setRowSpan("1y") + .build(); + assertEquals(1356998400, RollupUtils.getRollupBasetime(1388534399999L, + interval)); + } + + @Test + public void getRollupBasetimeHourZero() throws Exception { + // Thu, 01 Jan 1970 00:00:00 GMT + assertEquals(0, RollupUtils.getRollupBasetime(0L, hour_interval)); + } + + @Test + public void getRollupBasetimeDayZero() throws Exception { + // Thu, 01 Jan 1970 00:00:00 GMT + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("10m") + .setRowSpan("1d") + .build(); + assertEquals(0, RollupUtils.getRollupBasetime(0L, interval)); + } + + @Test + public void getRollupBasetimeMonthZero() throws Exception { + // Thu, 01 Jan 1970 00:00:00 GMT + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("1h") + .setRowSpan("1n") + .build(); + assertEquals(0, RollupUtils.getRollupBasetime(0L, interval)); + } + + @Test + public void getRollupBasetimeYearZero() throws Exception { + // Thu, 01 Jan 1970 00:00:00 GMT + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("24h") + .setRowSpan("1y") + .build(); + assertEquals(0, RollupUtils.getRollupBasetime(0L, interval)); + } + + @Test (expected = IllegalArgumentException.class) + public void getRollupBasetimeNegativeTimestamp() throws Exception { + // Thu, 06 Jun 2013 15:00:00 GMT + RollupUtils.getRollupBasetime(-1370530800L, hour_interval); + } + + @Test (expected = NullPointerException.class) + public void getRollupBasetimeNullInterval() throws Exception { + // Tue, 31 Dec 2013 23:59:59.999 GMT + assertEquals(1356998400, RollupUtils.getRollupBasetime(1388534399999L, + null)); + } + + @Test (expected = IllegalArgumentException.class) + public void getRollupBasetimeBadSpan() throws Exception { + // Tue, 31 Dec 2013 23:59:59.999 GMT + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("1s") + .setRowSpan("1w") + .build(); + assertEquals(1356998400, RollupUtils.getRollupBasetime(1388534399999L, + interval)); + } + + @Test + public void buildRollupQualifier1SecondInHourTop() { + final byte[] offset = {0, (byte)0x07}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + //Thu, 06 Jun 2013 15:00:00 GMT + final byte[] q = RollupUtils.buildRollupQualifier(1370530800L, 1370530800, + (byte)7, 42, hour_interval); + + assertArrayEquals(expected_qual, q); + } + + @Test + public void buildRollupQualifier1SecondInHourMid() { + final byte[] offset = {(byte) 0x84, (byte)0xD7}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + //Thu, 06 Jun 2013 15:35:25 GMT + final byte[] q = RollupUtils.buildRollupQualifier(1370532925L, 1370530800, + (byte)7, 42, hour_interval); + + assertArrayEquals(expected_qual, q); + } + + @Test + public void buildRollupQualifier1SecondInHourEnd() { + final byte[] offset = {(byte) 0xE0, (byte)0xF7}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + //Thu, 06 Jun 2013 15:59:59 GMT + final byte[] q = RollupUtils.buildRollupQualifier(1370534399L, 1370530800, + (byte)7, 42, hour_interval); + + assertArrayEquals(expected_qual, q); + } + + @Test (expected = IllegalArgumentException.class) + public void buildRollupQualifier1SecondInHourOver() { + //Thu, 06 Jun 2013 16:00:00 GMT + RollupUtils.buildRollupQualifier(1370534400L, 1370530800, (byte)7, + 42, hour_interval); + } + + @Test + public void buildRollupQualifier30SecondInHourTop() { + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("30s") + .setRowSpan("1h") + .build(); + + final byte[] offset = {0, (byte)0x07}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + //Thu, 06 Jun 2013 15:00:00 GMT + final byte[] q = RollupUtils.buildRollupQualifier(1370530800L, 1370530800, + (byte)7, 42, interval); + + assertArrayEquals(expected_qual, q); + } + + @Test + public void buildRollupQualifier30SecondInHourMid() { + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("30s") + .setRowSpan("1h") + .build(); + + final byte[] offset = {4, (byte)0x67}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + //Thu, 06 Jun 2013 15:35:25 GMT + final byte[] q = RollupUtils.buildRollupQualifier(1370532925L, 1370530800, + (byte)7, 42, interval); + + assertArrayEquals(expected_qual, q); + } + + @Test + public void buildRollupQualifier30SecondInHourEnd() { + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("30s") + .setRowSpan("1h") + .build(); + + final byte[] offset = {7, (byte)0x77}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + //Thu, 06 Jun 2013 15:59:59 GMT + final byte[] q = RollupUtils.buildRollupQualifier(1370534399L, 1370530800, + (byte)7, 42, interval); + + assertArrayEquals(expected_qual, q); + } + + @Test (expected = IllegalArgumentException.class) + public void buildRollupQualifier30SecondInHourOver() { + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("30s") + .setRowSpan("1h") + .build(); + + //Thu, 06 Jun 2013 16:00:00 GMT + RollupUtils.buildRollupQualifier(1370534400L, 1370530800, (byte)7, + 42, interval); + } + + @Test + public void buildRollupQualifier1MinuteInHourTop() { + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("1m") + .setRowSpan("1h") + .build(); + + final byte[] offset = {0, (byte)0x07}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + //Thu, 06 Jun 2013 15:00:00 GMT + final byte[] q = RollupUtils.buildRollupQualifier(1370530800L, 1370530800, + (byte)7, 42, interval); + + assertArrayEquals(expected_qual, q); + } + + @Test + public void buildRollupQualifier1MinuteInHourMid() { + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("1m") + .setRowSpan("1h") + .build(); + + final byte[] offset = {2, (byte)0x37}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + //Thu, 06 Jun 2013 15:35:25 GMT + final byte[] q = RollupUtils.buildRollupQualifier(1370532925L, 1370530800, + (byte)7, 42, interval); + + assertArrayEquals(expected_qual, q); + } + + @Test + public void buildRollupQualifier1MinuteInHourEnd() { + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("1m") + .setRowSpan("1h") + .build(); + + final byte[] offset = {3, (byte)0xB7}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + //Thu, 06 Jun 2013 15:35:25 GMT + final byte[] q = RollupUtils.buildRollupQualifier(1370534399L, 1370530800, + (byte)7, 42, interval); + + assertArrayEquals(expected_qual, q); + } + + @Test (expected = IllegalArgumentException.class) + public void buildRollupQualifier1MinuteInHourOver() { + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("30s") + .setRowSpan("1h") + .build(); + + //Thu, 06 Jun 2013 16:00:00 GMT + RollupUtils.buildRollupQualifier(1370534400L, 1370530800, (byte)7, + 42, interval); + } + + @Test + public void buildRollupQualifier15MinutesInDayTop() { + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("15m") + .setRowSpan("1d") + .build(); + + final byte[] offset = {0, (byte)0x07}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + //Thu, 06 Jun 2013 00:00:00 GMT + final byte[] q = RollupUtils.buildRollupQualifier(1370476800L, 1370476800, + (byte)7, 42, interval); + + assertArrayEquals(expected_qual, q); + } + + @Test + public void buildRollupQualifier15MinutesInDayMid() { + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("15m") + .setRowSpan("1d") + .build(); + + final byte[] offset = {3, (byte)0xE7}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + //Thu, 06 Jun 2013 15:35:25 GMT + final byte[] q = RollupUtils.buildRollupQualifier(1370532925L, 1370476800, + (byte)7, 42, interval); + + assertArrayEquals(expected_qual, q); + } + + @Test + public void buildRollupQualifier15MinutesInDayEnd() { + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("15m") + .setRowSpan("1d") + .build(); + + final byte[] offset = {5, (byte)0xF7}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + //Thu, 06 Jun 2013 23:59:59 GMT + final byte[] q = RollupUtils.buildRollupQualifier(1370563199L, 1370476800, + (byte)7, 42, interval); + + assertArrayEquals(expected_qual, q); + } + + @Test (expected = IllegalArgumentException.class) + public void buildRollupQualifier15MinutesInDayOver() { + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("15m") + .setRowSpan("1d") + .build(); + + //Thu, 07 Jun 2013 00:00:00 GMT + RollupUtils.buildRollupQualifier(1370563200L, 1370476800, (byte)7, 42, + interval); + } + + @Test + public void buildRollupQualifier60MinutesInDayTop() { + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("60m") + .setRowSpan("1d") + .build(); + + final byte[] offset = {0, (byte)0x07}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + //Thu, 06 Jun 2013 00:00:00 GMT + final byte[] q = RollupUtils.buildRollupQualifier(1370476800L, 1370476800, + (byte)7, 42, interval); + + assertArrayEquals(expected_qual, q); + } + + @Test + public void buildRollupQualifier60MinutesInDayMid() { + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("60m") + .setRowSpan("1d") + .build(); + + final byte[] offset = {0, (byte)0xF7}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + //Thu, 06 Jun 2013 15:35:25 GMT + final byte[] q = RollupUtils.buildRollupQualifier(1370532925L, 1370476800, + (byte)7, 42, interval); + + assertArrayEquals(expected_qual, q); + } + + @Test + public void buildRollupQualifier60MinutesInDayEnd() { + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("60m") + .setRowSpan("1d") + .build(); + + final byte[] offset = {1, (byte)0x77}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + //Thu, 06 Jun 2013 23:59:59 GMT + final byte[] q = RollupUtils.buildRollupQualifier(1370563199L, 1370476800, + (byte)7, 42, interval); + + assertArrayEquals(expected_qual, q); + } + + @Test (expected = IllegalArgumentException.class) + public void buildRollupQualifier60MinutesInDayOver() { + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("60m") + .setRowSpan("1d") + .build(); + + //Thu, 07 Jun 2013 00:00:00 GMT + RollupUtils.buildRollupQualifier(1370563200L, 1370476800, (byte)7, 42, + interval); + } + + @Test + public void buildRollupQualifier3HoursInMonthTop() { + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("3h") + .setRowSpan("1n") + .build(); + + final byte[] offset = {0, (byte)0x07}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + //Sat, 01 Jun 2013 00:00:00 GMT + final byte[] q = RollupUtils.buildRollupQualifier(1370044800L, 1370044800, + (byte)7, 42, interval); + + assertArrayEquals(expected_qual, q); + } + + @Test + public void buildRollupQualifier3HoursInMonthMid() { + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("3h") + .setRowSpan("1n") + .build(); + + final byte[] offset = {2, (byte)0xD7}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + //Thu, 06 Jun 2013 15:35:25 GMT + final byte[] q = RollupUtils.buildRollupQualifier(1370532925L, 1370044800, + (byte)7, 42, interval); + + assertArrayEquals(expected_qual, q); + } + + @Test + public void buildRollupQualifier3HoursInMonthEnd() { + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("3h") + .setRowSpan("1n") + .build(); + + final byte[] offset = {0x0E, (byte)0xF7}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + //Thu, 30 Jun 2013 23:59:59 GMT + final byte[] q = RollupUtils.buildRollupQualifier(1372636799L, 1370044800, + (byte)7, 42, interval); + + assertArrayEquals(expected_qual, q); + } + + // NOTE this guy won't overflow since we max our monthlies on 31 days. + @Test + public void buildRollupQualifier3HoursInMonthOver30Days() { + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("3h") + .setRowSpan("1n") + .build(); + + final byte[] offset = {0x0F, (byte)0x07}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + //Thu, 1 July 2013 00:00:00 GMT + final byte[] q = RollupUtils.buildRollupQualifier(1372636800L, 1370044800, + (byte)7, 42, interval); + + assertArrayEquals(expected_qual, q); + } + + // Still only overflows 3 days later + @Test (expected = IllegalArgumentException.class) + public void buildRollupQualifier3HoursInMonthOver() { + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("3h") + .setRowSpan("1n") + .build(); + + //Wed, 03 Jul 2013 23:59:59 GMT + RollupUtils.buildRollupQualifier(1372895999L, 1370044800, (byte)7, 42, + interval); + } + + @Test + public void buildRollupQualifier6HoursInYearTop() { + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("6h") + .setRowSpan("1y") + .build(); + + final byte[] offset = {0, (byte)0x07}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + //Tue, 01 Jan 2013 00:00:00 GMT + final byte[] q = RollupUtils.buildRollupQualifier(1356998400L, 1356998400, + (byte)7, 42, interval); + + assertArrayEquals(expected_qual, q); + } + + @Test + public void buildRollupQualifier6HoursInYearMid() { + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("6h") + .setRowSpan("1y") + .build(); + + final byte[] offset = {0x27, (byte)0x27}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + //Thu, 06 Jun 2013 15:35:25 GMT + final byte[] q = RollupUtils.buildRollupQualifier(1370532925L, 1356998400, + (byte)7, 42, interval); + + assertArrayEquals(expected_qual, q); + } + + @Test + public void buildRollupQualifier6HoursInYearEnd() { + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("6h") + .setRowSpan("1y") + .build(); + + final byte[] offset = {0x5B, (byte)0x37}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + //Tue, 31 Dec 2013 23:59:59 GMT + final byte[] q = RollupUtils.buildRollupQualifier(1388534399, 1356998400, + (byte)7, 42, interval); + + assertArrayEquals(expected_qual, q); + } + + // overflows since our max years are a little larger + @Test (expected = IllegalArgumentException.class) + public void buildRollupQualifier6HoursInYearOver() { + final RollupInterval interval = RollupInterval.builder() + .setTable(temporal_table) + .setPreAggregationTable(groupby_table) + .setInterval("6h") + .setRowSpan("1y") + .build(); + + //Wed, 01 Jan 2014 00:00:00 GMT + RollupUtils.buildRollupQualifier(1388620800, 1356998400, (byte)7, 42, + interval); + } + + // Flag tests ------------------ + @Test + public void buildRollupQualifier8BytesLong() { + final byte[] offset = {(byte) 0x84, (byte)0xD7}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + final byte[] q = RollupUtils.buildRollupQualifier(1370532925L, 1370530800, + (byte)7, 42, hour_interval); + + assertArrayEquals(expected_qual, q); + } + + @Test + public void buildRollupQualifierBytesLong() { + final byte[] offset = {(byte) 0x84, (byte)0xD3}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + final byte[] q = RollupUtils.buildRollupQualifier(1370532925L, 1370530800, + (byte) 3, 42, hour_interval); + + assertArrayEquals(expected_qual, q); + } + + @Test + public void buildRollupQualifier2BytesLong() { + final byte[] offset = {(byte) 0x84, (byte)0xD1}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + final byte[] q = RollupUtils.buildRollupQualifier(1370532925L, 1370530800, + (byte) 1, 42, hour_interval); + + assertArrayEquals(expected_qual, q); + } + + @Test + public void buildRollupQualifierByteLong() { + final byte[] offset = {(byte) 0x84, (byte)0xD0}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + final byte[] q = RollupUtils.buildRollupQualifier(1370532925L, 1370530800, + (byte) 0, 42, hour_interval); + + assertArrayEquals(expected_qual, q); + } + + @Test + public void buildRollupQualifierTenMin8ByteFloat() { + final byte[] offset = {(byte) 0x84, (byte)0xDF}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + final byte[] q = RollupUtils.buildRollupQualifier(1370532925L, 1370530800, + (byte) ( 7 | Const.FLAG_FLOAT), 42, hour_interval); + + assertArrayEquals(expected_qual, q); + } + + @Test + public void buildRollupQualifier4ByteFloat() { + final byte[] offset = {(byte) 0x84, (byte)0xDB}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + final byte[] q = RollupUtils.buildRollupQualifier(1370532925L, 1370530800, + (byte) ( 3 | Const.FLAG_FLOAT), 42, hour_interval); + + assertArrayEquals(expected_qual, q); + } + + @Test + public void buildRollupQualifierTenMinZeroTime() { + final byte[] offset = {0x0, 0x0}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + final byte[] q = + RollupUtils.buildRollupQualifier(0, 0, (byte) 0, 42, hour_interval); + + assertArrayEquals(expected_qual, q); + } + + // this one will be really goofy because we don't really account for negative + // values in the offset when applying the mask to the timestamp to determine if + // it's in millisecond s or not. Fix this up some day. + @Test + public void buildRollupQualifierNegativeTime() { + final byte[] offset = {(byte) 0xF1, 0}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + final byte[] q = RollupUtils.buildRollupQualifier(1420062000L, -1420063200, + (byte) 0, 42, hour_interval); + + assertArrayEquals(expected_qual, q); + } + + @Test + public void buildRollupQualifierAggCase() { + final byte[] offset = {0, (byte)0x07}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + //Thu, 06 Jun 2013 15:00:00 GMT + final byte[] q = RollupUtils.buildRollupQualifier(1370530800L, 1370530800, + (byte)7, 42, hour_interval); + + assertArrayEquals(expected_qual, q); + } + + // verify we truncate the milliseconds + @Test + public void buildRollupQualifierMillisecond() { + final byte[] offset = {(byte) 0x84, (byte)0xD7}; + byte[] expected_qual = new byte[3]; + expected_qual[0] = 42; + System.arraycopy(offset, 0, expected_qual, 1, 2); + + //Thu, 06 Jun 2013 15:35:25 GMT + final byte[] q = RollupUtils.buildRollupQualifier(1370532925154L, 1370530800, + (byte)7, 42, hour_interval); + + assertArrayEquals(expected_qual, q); + } + + @Test + public void mask() throws Exception { + byte[] header = { (byte) 0x81 }; + assertEquals(1, header[0] & RollupUtils.AGGREGATOR_MASK); + assertTrue(RollupUtils.isCompacted(header)); + + header = new byte[] { 0x01 }; + assertEquals(1, header[0] & RollupUtils.AGGREGATOR_MASK); + assertFalse(RollupUtils.isCompacted(header)); + } +} diff --git a/test/search/TestTimeSeriesLookup.java b/test/search/TestTimeSeriesLookup.java index e302f09cfb..60ededd380 100644 --- a/test/search/TestTimeSeriesLookup.java +++ b/test/search/TestTimeSeriesLookup.java @@ -15,15 +15,15 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.when; -import static org.powermock.api.mockito.PowerMockito.mock; -import java.lang.reflect.Field; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import net.opentsdb.core.BaseTsdbTest; import net.opentsdb.core.Const; +import net.opentsdb.core.RowKey; import net.opentsdb.core.TSDB; import net.opentsdb.meta.TSMeta; import net.opentsdb.storage.MockBase; @@ -32,93 +32,54 @@ import net.opentsdb.utils.Config; import net.opentsdb.utils.Pair; +import org.hbase.async.Bytes; import org.hbase.async.HBaseClient; import org.hbase.async.KeyValue; import org.hbase.async.Scanner; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import com.stumbleupon.async.Deferred; + @RunWith(PowerMockRunner.class) @PowerMockIgnore({"javax.management.*", "javax.xml.*", "ch.qos.*", "org.slf4j.*", "com.sum.*", "org.xml.*"}) @PrepareForTest({TSDB.class, Config.class, UniqueId.class, HBaseClient.class, KeyValue.class, Scanner.class, TimeSeriesLookup.class}) -public class TestTimeSeriesLookup { - private Config config; - private TSDB tsdb = null; - private HBaseClient client = mock(HBaseClient.class); - private UniqueId metrics = mock(UniqueId.class); - private UniqueId tag_names = mock(UniqueId.class); - private UniqueId tag_values = mock(UniqueId.class); - private MockBase storage = null; +public class TestTimeSeriesLookup extends BaseTsdbTest { // tsuids - private static List<byte[]> test_tsuids = new ArrayList<byte[]>(7); + public static List<byte[]> test_tsuids = new ArrayList<byte[]>(7); static { test_tsuids.add(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }); test_tsuids.add(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 2 }); test_tsuids.add(new byte[] { 0, 0, 2, 0, 0, 1, 0, 0, 1 }); - test_tsuids.add(new byte[] { 0, 0, 3, 0, 0, 1, 0, 0, 1, 0, 0, 4, 0, 0, 5}); - test_tsuids.add(new byte[] { 0, 0, 3, 0, 0, 1, 0, 0, 2, 0, 0, 4, 0, 0, 5}); - test_tsuids.add(new byte[] { 0, 0, 3, 0, 0, 6, 0, 0, 7, 0, 0, 8, 0, 0, 1, + test_tsuids.add(new byte[] { 0, 0, 4, 0, 0, 1, 0, 0, 1, 0, 0, 3, 0, 0, 5}); + test_tsuids.add(new byte[] { 0, 0, 4, 0, 0, 1, 0, 0, 2, 0, 0, 3, 0, 0, 5}); + test_tsuids.add(new byte[] { 0, 0, 4, 0, 0, 6, 0, 0, 7, 0, 0, 8, 0, 0, 1, 0, 0, 9, 0, 0, 3}); - test_tsuids.add(new byte[] { 0, 0, 3, 0, 0, 6, 0, 0, 7, 0, 0, 8, 0, 0, 10, + test_tsuids.add(new byte[] { 0, 0, 4, 0, 0, 6, 0, 0, 7, 0, 0, 8, 0, 0, 10, 0, 0, 9, 0, 0, 3}); } @Before - public void before() throws Exception { - PowerMockito.whenNew(HBaseClient.class) - .withArguments(anyString(), anyString()).thenReturn(client); - config = new Config(false); - tsdb = new TSDB(config); - - // replace the "real" field objects with mocks - Field met = tsdb.getClass().getDeclaredField("metrics"); - met.setAccessible(true); - met.set(tsdb, metrics); - - Field tagk = tsdb.getClass().getDeclaredField("tag_names"); - tagk.setAccessible(true); - tagk.set(tsdb, tag_names); - - Field tagv = tsdb.getClass().getDeclaredField("tag_values"); - tagv.setAccessible(true); - tagv.set(tsdb, tag_values); - - // mock UniqueId - when(metrics.getId("sys.cpu.user")).thenReturn(new byte[] { 0, 0, 1 }); - when(metrics.getId("sys.cpu.system")) - .thenThrow(new NoSuchUniqueName("sys.cpu.system", "metric")); - when(metrics.getId("sys.cpu.nice")).thenReturn(new byte[] { 0, 0, 2 }); - when(metrics.getId("sys.cpu.idle")).thenReturn(new byte[] { 0, 0, 3 }); - when(metrics.getId("no.values")).thenReturn(new byte[] { 0, 0, 11 }); - - when(tag_names.getId("host")).thenReturn(new byte[] { 0, 0, 1 }); - when(tag_names.getId("dc")) - .thenThrow(new NoSuchUniqueName("dc", "metric")); - when(tag_names.getId("owner")).thenReturn(new byte[] { 0, 0, 4 }); - - when(tag_values.getId("web01")).thenReturn(new byte[] { 0, 0, 1 }); - when(tag_values.getId("web02")).thenReturn(new byte[] { 0, 0, 2 }); - when(tag_values.getId("web03")) - .thenThrow(new NoSuchUniqueName("web03", "metric")); - - when(metrics.width()).thenReturn((short)3); - when(tag_names.width()).thenReturn((short)3); - when(tag_values.width()).thenReturn((short)3); + public void beforeLocal() { + storage = new MockBase(tsdb, client, true, true, true, true); + when(metrics.getIdAsync("no.values")) + .thenReturn(Deferred.fromResult(new byte[] { 0, 0, 11 })); + when(metrics.getIdAsync("filtered")) + .thenReturn(Deferred.fromResult(new byte[] { 0, 0, 4 })); } - + @Test public void metricOnlyMeta() throws Exception { - generateMeta(); - final SearchQuery query = new SearchQuery("sys.cpu.user"); + generateMeta(tsdb, storage); + final SearchQuery query = new SearchQuery(METRIC_STRING); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List<byte[]> tsuids = lookup.lookup(); assertNotNull(tsuids); @@ -130,7 +91,7 @@ public void metricOnlyMeta() throws Exception { // returns everything @Test public void metricOnlyMetaStar() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final SearchQuery query = new SearchQuery("*"); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List<byte[]> tsuids = lookup.lookup(); @@ -140,8 +101,8 @@ public void metricOnlyMetaStar() throws Exception { @Test public void metricOnlyData() throws Exception { - generateData(); - final SearchQuery query = new SearchQuery("sys.cpu.user"); + generateData(tsdb, storage); + final SearchQuery query = new SearchQuery(METRIC_STRING); query.setUseMeta(false); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List<byte[]> tsuids = lookup.lookup(); @@ -153,8 +114,8 @@ public void metricOnlyData() throws Exception { @Test public void metricOnly2Meta() throws Exception { - generateMeta(); - final SearchQuery query = new SearchQuery("sys.cpu.nice"); + generateMeta(tsdb, storage); + final SearchQuery query = new SearchQuery(METRIC_B_STRING); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List<byte[]> tsuids = lookup.lookup(); assertNotNull(tsuids); @@ -164,8 +125,8 @@ public void metricOnly2Meta() throws Exception { @Test public void metricOnly2Data() throws Exception { - generateData(); - final SearchQuery query = new SearchQuery("sys.cpu.nice"); + generateData(tsdb, storage); + final SearchQuery query = new SearchQuery(METRIC_B_STRING); query.setUseMeta(false); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List<byte[]> tsuids = lookup.lookup(); @@ -176,14 +137,14 @@ public void metricOnly2Data() throws Exception { @Test (expected = NoSuchUniqueName.class) public void noSuchMetricMeta() throws Exception { - final SearchQuery query = new SearchQuery("sys.cpu.system"); + final SearchQuery query = new SearchQuery(NSUN_METRIC); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); lookup.lookup(); } @Test public void metricOnlyNoValuesMeta() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final SearchQuery query = new SearchQuery("no.values"); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List<byte[]> tsuids = lookup.lookup(); @@ -193,7 +154,7 @@ public void metricOnlyNoValuesMeta() throws Exception { @Test public void metricOnlyNoValuesData() throws Exception { - generateData(); + generateData(tsdb, storage); final SearchQuery query = new SearchQuery("no.values"); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); query.setUseMeta(false); @@ -204,10 +165,10 @@ public void metricOnlyNoValuesData() throws Exception { @Test public void tagkOnlyMeta() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final List<Pair<String, String>> tags = new ArrayList<Pair<String, String>>(1); - tags.add(new Pair<String, String>("host", null)); + tags.add(new Pair<String, String>(TAGK_STRING, null)); final SearchQuery query = new SearchQuery(tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List<byte[]> tsuids = lookup.lookup(); @@ -220,10 +181,10 @@ public void tagkOnlyMeta() throws Exception { @Test public void tagkOnlyMetaStar() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final List<Pair<String, String>> tags = new ArrayList<Pair<String, String>>(1); - tags.add(new Pair<String, String>("host", "*")); + tags.add(new Pair<String, String>(TAGK_STRING, "*")); final SearchQuery query = new SearchQuery(tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List<byte[]> tsuids = lookup.lookup(); @@ -236,14 +197,15 @@ public void tagkOnlyMetaStar() throws Exception { @Test public void tagkOnlyData() throws Exception { - generateData(); + generateData(tsdb, storage); final List<Pair<String, String>> tags = new ArrayList<Pair<String, String>>(1); - tags.add(new Pair<String, String>("host", null)); + tags.add(new Pair<String, String>(TAGK_STRING, null)); final SearchQuery query = new SearchQuery(tags); query.setUseMeta(false); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List<byte[]> tsuids = lookup.lookup(); + Collections.sort(tsuids, Bytes.MEMCMP); // for salting assertNotNull(tsuids); assertEquals(5, tsuids.size()); for (int i = 0; i < 5; i++) { @@ -253,10 +215,10 @@ public void tagkOnlyData() throws Exception { @Test public void tagkOnly2Meta() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final List<Pair<String, String>> tags = new ArrayList<Pair<String, String>>(1); - tags.add(new Pair<String, String>("owner", null)); + tags.add(new Pair<String, String>(TAGK_B_STRING, null)); final SearchQuery query = new SearchQuery(tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List<byte[]> tsuids = lookup.lookup(); @@ -268,14 +230,15 @@ public void tagkOnly2Meta() throws Exception { @Test public void tagkOnly2Data() throws Exception { - generateData(); + generateData(tsdb, storage); final List<Pair<String, String>> tags = new ArrayList<Pair<String, String>>(1); - tags.add(new Pair<String, String>("owner", null)); + tags.add(new Pair<String, String>(TAGK_B_STRING, null)); final SearchQuery query = new SearchQuery(tags); query.setUseMeta(false); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List<byte[]> tsuids = lookup.lookup(); + Collections.sort(tsuids, Bytes.MEMCMP); // for salting assertNotNull(tsuids); assertEquals(2, tsuids.size()); assertArrayEquals(test_tsuids.get(3), tsuids.get(0)); @@ -286,7 +249,7 @@ public void tagkOnly2Data() throws Exception { public void noSuchTagkMeta() throws Exception { final List<Pair<String, String>> tags = new ArrayList<Pair<String, String>>(1); - tags.add(new Pair<String, String>("dc", null)); + tags.add(new Pair<String, String>(NSUN_TAGK, null)); final SearchQuery query = new SearchQuery(tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); lookup.lookup(); @@ -294,10 +257,10 @@ public void noSuchTagkMeta() throws Exception { @Test public void tagvOnlyMeta() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final List<Pair<String, String>> tags = new ArrayList<Pair<String, String>>(1); - tags.add(new Pair<String, String>(null, "web01")); + tags.add(new Pair<String, String>(null, TAGV_STRING)); final SearchQuery query = new SearchQuery(tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List<byte[]> tsuids = lookup.lookup(); @@ -311,10 +274,10 @@ public void tagvOnlyMeta() throws Exception { @Test public void tagvOnlyMetaStar() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final List<Pair<String, String>> tags = new ArrayList<Pair<String, String>>(1); - tags.add(new Pair<String, String>("*", "web01")); + tags.add(new Pair<String, String>("*", TAGV_STRING)); final SearchQuery query = new SearchQuery(tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List<byte[]> tsuids = lookup.lookup(); @@ -328,14 +291,15 @@ public void tagvOnlyMetaStar() throws Exception { @Test public void tagvOnlyData() throws Exception { - generateData(); + generateData(tsdb, storage); final List<Pair<String, String>> tags = new ArrayList<Pair<String, String>>(1); - tags.add(new Pair<String, String>(null, "web01")); + tags.add(new Pair<String, String>(null, TAGV_STRING)); final SearchQuery query = new SearchQuery(tags); query.setUseMeta(false); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List<byte[]> tsuids = lookup.lookup(); + Collections.sort(tsuids, Bytes.MEMCMP); // for salting assertNotNull(tsuids); assertEquals(4, tsuids.size()); assertArrayEquals(test_tsuids.get(0), tsuids.get(0)); @@ -346,10 +310,10 @@ public void tagvOnlyData() throws Exception { @Test public void tagvOnly2Meta() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final List<Pair<String, String>> tags = new ArrayList<Pair<String, String>>(1); - tags.add(new Pair<String, String>(null, "web02")); + tags.add(new Pair<String, String>(null, TAGV_B_STRING)); final SearchQuery query = new SearchQuery(tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List<byte[]> tsuids = lookup.lookup(); @@ -361,14 +325,15 @@ public void tagvOnly2Meta() throws Exception { @Test public void tagvOnly2Data() throws Exception { - generateData(); + generateData(tsdb, storage); final List<Pair<String, String>> tags = new ArrayList<Pair<String, String>>(1); - tags.add(new Pair<String, String>(null, "web02")); + tags.add(new Pair<String, String>(null, TAGV_B_STRING)); final SearchQuery query = new SearchQuery(tags); query.setUseMeta(false); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List<byte[]> tsuids = lookup.lookup(); + Collections.sort(tsuids, Bytes.MEMCMP); // for salting assertNotNull(tsuids); assertEquals(2, tsuids.size()); assertArrayEquals(test_tsuids.get(1), tsuids.get(0)); @@ -379,7 +344,7 @@ public void tagvOnly2Data() throws Exception { public void noSuchTagvMeta() throws Exception { final List<Pair<String, String>> tags = new ArrayList<Pair<String, String>>(1); - tags.add(new Pair<String, String>(null, "web03")); + tags.add(new Pair<String, String>(null, NSUN_TAGV)); final SearchQuery query = new SearchQuery(tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); lookup.lookup(); @@ -387,11 +352,11 @@ public void noSuchTagvMeta() throws Exception { @Test public void metricAndTagkMeta() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final List<Pair<String, String>> tags = new ArrayList<Pair<String, String>>(1); - tags.add(new Pair<String, String>("host", null)); - final SearchQuery query = new SearchQuery("sys.cpu.nice", + tags.add(new Pair<String, String>(TAGK_STRING, null)); + final SearchQuery query = new SearchQuery(METRIC_B_STRING, tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List<byte[]> tsuids = lookup.lookup(); @@ -402,11 +367,11 @@ public void metricAndTagkMeta() throws Exception { @Test public void metricAndTagkMetaStar() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final List<Pair<String, String>> tags = new ArrayList<Pair<String, String>>(1); - tags.add(new Pair<String, String>("host", "*")); - final SearchQuery query = new SearchQuery("sys.cpu.nice", + tags.add(new Pair<String, String>(TAGK_STRING, "*")); + final SearchQuery query = new SearchQuery(METRIC_B_STRING, tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List<byte[]> tsuids = lookup.lookup(); @@ -417,11 +382,11 @@ public void metricAndTagkMetaStar() throws Exception { @Test public void metricAndTagkData() throws Exception { - generateData(); + generateData(tsdb, storage); final List<Pair<String, String>> tags = new ArrayList<Pair<String, String>>(1); - tags.add(new Pair<String, String>("host", null)); - final SearchQuery query = new SearchQuery("sys.cpu.nice", + tags.add(new Pair<String, String>(TAGK_STRING, null)); + final SearchQuery query = new SearchQuery(METRIC_B_STRING, tags); query.setUseMeta(false); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); @@ -433,12 +398,11 @@ public void metricAndTagkData() throws Exception { @Test public void metricAndTagvMeta() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final List<Pair<String, String>> tags = new ArrayList<Pair<String, String>>(1); - tags.add(new Pair<String, String>(null, "web02")); - final SearchQuery query = new SearchQuery("sys.cpu.idle", - tags); + tags.add(new Pair<String, String>(null, TAGV_B_STRING)); + final SearchQuery query = new SearchQuery("filtered", tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List<byte[]> tsuids = lookup.lookup(); assertNotNull(tsuids); @@ -448,12 +412,11 @@ public void metricAndTagvMeta() throws Exception { @Test public void metricAndTagvMetaStar() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final List<Pair<String, String>> tags = new ArrayList<Pair<String, String>>(1); - tags.add(new Pair<String, String>("*", "web02")); - final SearchQuery query = new SearchQuery("sys.cpu.idle", - tags); + tags.add(new Pair<String, String>("*", TAGV_B_STRING)); + final SearchQuery query = new SearchQuery("filtered",tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List<byte[]> tsuids = lookup.lookup(); assertNotNull(tsuids); @@ -463,12 +426,11 @@ public void metricAndTagvMetaStar() throws Exception { @Test public void metricAndTagvData() throws Exception { - generateData(); + generateData(tsdb, storage); final List<Pair<String, String>> tags = new ArrayList<Pair<String, String>>(1); - tags.add(new Pair<String, String>(null, "web02")); - final SearchQuery query = new SearchQuery("sys.cpu.idle", - tags); + tags.add(new Pair<String, String>(null, TAGV_B_STRING)); + final SearchQuery query = new SearchQuery("filtered", tags); query.setUseMeta(false); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List<byte[]> tsuids = lookup.lookup(); @@ -479,12 +441,11 @@ public void metricAndTagvData() throws Exception { @Test public void metricAndTagPairMeta() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final List<Pair<String, String>> tags = new ArrayList<Pair<String, String>>(1); - tags.add(new Pair<String, String>("host", "web01")); - final SearchQuery query = new SearchQuery("sys.cpu.idle", - tags); + tags.add(new Pair<String, String>(TAGK_STRING, TAGV_STRING)); + final SearchQuery query = new SearchQuery("filtered", tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List<byte[]> tsuids = lookup.lookup(); assertNotNull(tsuids); @@ -494,12 +455,11 @@ public void metricAndTagPairMeta() throws Exception { @Test public void metricAndTagPairData() throws Exception { - generateData(); + generateData(tsdb, storage); final List<Pair<String, String>> tags = new ArrayList<Pair<String, String>>(1); - tags.add(new Pair<String, String>("host", "web01")); - final SearchQuery query = new SearchQuery("sys.cpu.idle", - tags); + tags.add(new Pair<String, String>(TAGK_STRING, TAGV_STRING)); + final SearchQuery query = new SearchQuery("filtered", tags); query.setUseMeta(false); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); query.setUseMeta(false); @@ -511,10 +471,10 @@ public void metricAndTagPairData() throws Exception { @Test public void tagPairOnlyMeta() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final List<Pair<String, String>> tags = new ArrayList<Pair<String, String>>(1); - tags.add(new Pair<String, String>("host", "web01")); + tags.add(new Pair<String, String>(TAGK_STRING, TAGV_STRING)); final SearchQuery query = new SearchQuery(tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List<byte[]> tsuids = lookup.lookup(); @@ -527,14 +487,15 @@ public void tagPairOnlyMeta() throws Exception { @Test public void tagPairOnlyData() throws Exception { - generateData(); + generateData(tsdb, storage); final List<Pair<String, String>> tags = new ArrayList<Pair<String, String>>(1); - tags.add(new Pair<String, String>("host", "web01")); + tags.add(new Pair<String, String>(TAGK_STRING, TAGV_STRING)); final SearchQuery query = new SearchQuery(tags); query.setUseMeta(false); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List<byte[]> tsuids = lookup.lookup(); + Collections.sort(tsuids, Bytes.MEMCMP); // for salting assertNotNull(tsuids); assertEquals(3, tsuids.size()); assertArrayEquals(test_tsuids.get(0), tsuids.get(0)); @@ -542,36 +503,75 @@ public void tagPairOnlyData() throws Exception { assertArrayEquals(test_tsuids.get(3), tsuids.get(2)); } + @Test + public void limitVerification() throws Exception { + generateData(tsdb, storage); + final List<Pair<String, String>> tags = + new ArrayList<Pair<String, String>>(1); + tags.add(new Pair<String, String>(TAGK_STRING, TAGV_STRING)); + final SearchQuery query = new SearchQuery(tags); + query.setUseMeta(false); + query.setLimit(1); + final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); + final List<byte[]> tsuids = lookup.lookup(); + assertNotNull(tsuids); + assertEquals(1, tsuids.size()); + assertArrayEquals(test_tsuids.get(0), tsuids.get(0)); + } + + @Test (expected = RuntimeException.class) + public void scannerException() throws Exception { + generateData(tsdb, storage); + final byte[] row = Const.SALT_WIDTH() > 0 ? + MockBase.stringToBytes( + "0300000400000000000001000001000003000005") : + MockBase.stringToBytes( + "00000400000000000001000001000003000005"); + storage.throwException(row, new RuntimeException("Boo!")); + final List<Pair<String, String>> tags = + new ArrayList<Pair<String, String>>(1); + tags.add(new Pair<String, String>(TAGK_STRING, TAGV_STRING)); + final SearchQuery query = new SearchQuery(tags); + query.setUseMeta(false); + query.setLimit(1); + final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); + lookup.lookup(); + } + // TODO test the dump to stdout /** * Stores some data in the mock tsdb-meta table for unit testing */ - private void generateMeta() { - storage = new MockBase(tsdb, client, true, true, true, true); - storage.setFamily("t".getBytes(MockBase.ASCII())); + public static void generateMeta(final TSDB tsdb, final MockBase storage) { + final List<byte[]> families = new ArrayList<byte[]>(1); + families.add(TSMeta.FAMILY); + storage.addTable("tsdb-meta".getBytes(), families); final byte[] val = new byte[] { 0, 0, 0, 0, 0, 0, 0, 1 }; for (final byte[] tsuid : test_tsuids) { - storage.addColumn(tsuid, TSMeta.COUNTER_QUALIFIER(), val); + storage.addColumn("tsdb-meta".getBytes(), tsuid, TSMeta.FAMILY, + TSMeta.COUNTER_QUALIFIER(), val); } } /** * Stores some data in the mock tsdb data table for unit testing */ - private void generateData() { - storage = new MockBase(tsdb, client, true, true, true, true); + public static void generateData(final TSDB tsdb, final MockBase storage) { storage.setFamily("t".getBytes(MockBase.ASCII())); final byte[] qual = new byte[] { 0, 0 }; final byte[] val = new byte[] { 1 }; for (final byte[] tsuid : test_tsuids) { - byte[] row_key = new byte[tsuid.length + Const.TIMESTAMP_BYTES]; - System.arraycopy(tsuid, 0, row_key, 0, TSDB.metrics_width()); + byte[] row_key = new byte[Const.SALT_WIDTH() + tsuid.length + + Const.TIMESTAMP_BYTES]; + System.arraycopy(tsuid, 0, row_key, Const.SALT_WIDTH(), + TSDB.metrics_width()); System.arraycopy(tsuid, TSDB.metrics_width(), row_key, - TSDB.metrics_width() + Const.TIMESTAMP_BYTES, + Const.SALT_WIDTH() + TSDB.metrics_width() + Const.TIMESTAMP_BYTES, tsuid.length - TSDB.metrics_width()); + RowKey.prefixKeyWithSalt(row_key); storage.addColumn(row_key, qual, val); } } diff --git a/test/search/TestTimeSeriesLookupSalted.java b/test/search/TestTimeSeriesLookupSalted.java new file mode 100644 index 0000000000..a8f1109125 --- /dev/null +++ b/test/search/TestTimeSeriesLookupSalted.java @@ -0,0 +1,40 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.search; + +import net.opentsdb.core.Const; +import net.opentsdb.core.TSDB; +import net.opentsdb.uid.UniqueId; +import net.opentsdb.utils.Config; + +import org.hbase.async.HBaseClient; +import org.hbase.async.KeyValue; +import org.hbase.async.Scanner; +import org.junit.Before; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({TSDB.class, Config.class, UniqueId.class, HBaseClient.class, + KeyValue.class, Scanner.class, TimeSeriesLookup.class, Const.class }) +public class TestTimeSeriesLookupSalted extends TestTimeSeriesLookup { + + @Before + public void beforeLocalSalted() throws Exception { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(4); + } +} diff --git a/test/stats/TestQueryStats.java b/test/stats/TestQueryStats.java new file mode 100644 index 0000000000..e81477d15d --- /dev/null +++ b/test/stats/TestQueryStats.java @@ -0,0 +1,304 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.stats; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.anyLong; + +import java.lang.reflect.Field; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import com.google.common.cache.CacheBuilder; + +import net.opentsdb.core.QueryException; +import net.opentsdb.core.TSQuery; +import net.opentsdb.stats.QueryStats.QueryStat; +import net.opentsdb.utils.DateTime; + +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({ DateTime.class, QueryStats.class }) +public final class TestQueryStats { + + private static String remote = "192.168.1.1:4242"; + private static Field running_queries; + static { + try { + running_queries = QueryStats.class.getDeclaredField("running_queries"); + running_queries.setAccessible(true); + } catch (Exception e) { + throw new RuntimeException("Failed in static initializer", e); + } + } + private static Field completed_queries; + static { + try { + completed_queries = QueryStats.class.getDeclaredField("completed_queries"); + completed_queries.setAccessible(true); + } catch (Exception e) { + throw new RuntimeException("Failed in static initializer", e); + } + } + + private Map<String, String> headers; + + @Before + public void before() throws Exception { + running_queries.set(null, new ConcurrentHashMap<Integer, QueryStats>()); + completed_queries.set(null, CacheBuilder.newBuilder().maximumSize(2).build()); + headers = new HashMap<String, String>(1); + headers.put("Cookie", "Hide me!"); + PowerMockito.mockStatic(DateTime.class); + PowerMockito.doAnswer(new Answer<Long>() { + long ts = 1000L; + @Override + public Long answer(InvocationOnMock invocation) throws Throwable { + return ts += 1000000000L; + } + + }).when(DateTime.class, "nanoTime"); + PowerMockito.doCallRealMethod().when(DateTime.class, + "msFromNano", anyLong()); + PowerMockito.doCallRealMethod().when(DateTime.class, + "msFromNanoDiff", anyLong(), anyLong()); + } + + @Test + public void ctor() throws Exception { + final TSQuery query = new TSQuery(); + query.setStart("1h-ago"); + final QueryStats stats = new QueryStats(remote, query, headers); + assertNotNull(stats); + final Map<String, Object> map = QueryStats.getRunningAndCompleteStats(); + assertNotNull(map); + assertEquals(1, ((List<Object>)map.get("running")).size()); + assertEquals(0, ((Collection<QueryStats>)map.get("completed")).size()); + assertSame(headers, stats.getRequestHeaders()); + } + + @Test + public void ctorDuplicate() throws Exception { + QueryStats.setEnableDuplicates(false); + final TSQuery query = new TSQuery(); + query.setStart("1h-ago"); + final QueryStats stats = new QueryStats(remote, query, headers); + assertNotNull(stats); + final Map<String, Object> map = QueryStats.getRunningAndCompleteStats(); + assertNotNull(map); + assertEquals(1, ((List<Object>)map.get("running")).size()); + assertEquals(0, ((Collection<QueryStats>)map.get("completed")).size()); + try { + new QueryStats(remote, query, headers); + fail("Expected a QueryException"); + } catch (QueryException e) { } + QueryStats.setEnableDuplicates(true); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNullRemote() throws Exception { + final TSQuery query = new TSQuery(); + query.setStart("1h-ago"); + new QueryStats(null, query, headers); + } + + @Test (expected = IllegalArgumentException.class) + public void ctorNullQuery() throws Exception { + final TSQuery query = new TSQuery(); + query.setStart("1h-ago"); + new QueryStats(remote, null, headers); + } + + @Test + public void ctorNullHeaders() throws Exception { + final TSQuery query = new TSQuery(); + query.setStart("1h-ago"); + final QueryStats stats = new QueryStats(remote, query, null); + assertNotNull(stats); + final Map<String, Object> map = QueryStats.getRunningAndCompleteStats(); + assertNotNull(map); + assertEquals(1, ((List<Object>)map.get("running")).size()); + assertEquals(0, ((Collection<QueryStats>)map.get("completed")).size()); + } + + @Test + public void testHashCodeandEquals() throws Exception { + final TSQuery query = new TSQuery(); + query.setStart("1h-ago"); + final QueryStats stats = new QueryStats(remote, query, headers); + assertNotNull(stats); + final int hash_a = stats.hashCode(); + + // have to mark the old one as complete before we can test equality + stats.markSerializationSuccessful(); + + final TSQuery query2 = new TSQuery(); + query2.setStart("1h-ago"); + final QueryStats stats2 = new QueryStats(remote, query2, headers); + assertNotNull(stats); + assertEquals(hash_a, stats2.hashCode()); + assertEquals(stats, stats2); + assertFalse(stats == stats2); + } + + @Test + public void testHashCodeandNotEquals() throws Exception { + final TSQuery query = new TSQuery(); + query.setStart("1h-ago"); + final QueryStats stats = new QueryStats(remote, query, headers); + assertNotNull(stats); + final int hash_a = stats.hashCode(); + + final TSQuery query2 = new TSQuery(); + query2.setStart("2h-ago"); + final QueryStats stats2 = new QueryStats(remote, query2, headers); + assertNotNull(stats); + assertTrue(hash_a != stats2.hashCode()); + assertFalse(stats.equals(stats2)); + assertFalse(stats == stats2); + } + + @Test + public void testEqualsNull() throws Exception { + final TSQuery query = new TSQuery(); + query.setStart("1h-ago"); + final QueryStats stats = new QueryStats(remote, query, headers); + assertFalse(stats.equals(null)); + } + + @Test + public void testEqualsWrongType() throws Exception { + final TSQuery query = new TSQuery(); + query.setStart("1h-ago"); + final QueryStats stats = new QueryStats(remote, query, headers); + assertFalse(stats.equals(new String("foo"))); + } + + @Test + public void testEqualsSame() throws Exception { + final TSQuery query = new TSQuery(); + query.setStart("1h-ago"); + final QueryStats stats = new QueryStats(remote, query, headers); + assertTrue(stats.equals(stats)); + } + + @Test + public void markComplete() throws Exception { + final TSQuery query = new TSQuery(); + query.setStart("1h-ago"); + final QueryStats stats = new QueryStats(remote, query, headers); + stats.markSerializationSuccessful(); + final Map<String, Object> map = QueryStats.getRunningAndCompleteStats(); + assertNotNull(map); + assertEquals(0, ((List<Object>)map.get("running")).size()); + assertEquals(1, ((Collection<QueryStats>)map.get("completed")).size()); + final QueryStats completed = ((Collection<QueryStats>)map.get("completed")) + .iterator().next(); + assertEquals(200, completed.getHttpResponse().getCode()); + } + + @Test + public void markCompleteTimeout() throws Exception { + final TSQuery query = new TSQuery(); + query.setStart("1h-ago"); + final QueryStats stats = new QueryStats(remote, query, headers); + final RuntimeException timeout = new RuntimeException("Timeout!"); + stats.markSerialized(HttpResponseStatus.REQUEST_TIMEOUT, timeout); + final Map<String, Object> map = QueryStats.getRunningAndCompleteStats(); + assertNotNull(map); + assertEquals(0, ((List<Object>)map.get("running")).size()); + assertEquals(1, ((Collection<QueryStats>)map.get("completed")).size()); + final QueryStats completed = ((Collection<QueryStats>)map.get("completed")) + .iterator().next(); + assertEquals(408, completed.getHttpResponse().getCode()); + assertTrue(completed.getException().startsWith("Timeout!\n")); + } + + @Test + public void executed() throws Exception { + final TSQuery query = new TSQuery(); + query.setStart("1h-ago"); + final QueryStats stats = new QueryStats(remote, query, headers); + stats.markSerialized(HttpResponseStatus.REQUEST_TIMEOUT, null); + final Map<String, Object> map = QueryStats.getRunningAndCompleteStats(); + assertNotNull(map); + assertEquals(0, ((List<Object>)map.get("running")).size()); + assertEquals(1, ((Collection<QueryStats>)map.get("completed")).size()); + final QueryStats completed = ((Collection<QueryStats>)map.get("completed")) + .iterator().next(); + assertEquals(1, completed.getExecuted()); + } + + @Test + public void executedTwice() throws Exception { + final TSQuery query = new TSQuery(); + query.setStart("1h-ago"); + QueryStats stats = new QueryStats(remote, query, headers); + stats.markSerialized(HttpResponseStatus.REQUEST_TIMEOUT, null); + Map<String, Object> map = QueryStats.getRunningAndCompleteStats(); + assertNotNull(map); + assertEquals(0, ((List<Object>)map.get("running")).size()); + assertEquals(1, ((Collection<QueryStats>)map.get("completed")).size()); + QueryStats completed = ((Collection<QueryStats>)map.get("completed")) + .iterator().next(); + assertEquals(1, completed.getExecuted()); + + stats = new QueryStats(remote, query, headers); + stats.markSerialized(HttpResponseStatus.REQUEST_TIMEOUT, null); + map = QueryStats.getRunningAndCompleteStats(); + assertNotNull(map); + assertEquals(0, ((List<Object>)map.get("running")).size()); + assertEquals(1, ((Collection<QueryStats>)map.get("completed")).size()); + completed = ((Collection<QueryStats>)map.get("completed")) + .iterator().next(); + assertEquals(2, completed.getExecuted()); + } + + @Test + public void getStat() throws Exception { + final TSQuery query = new TSQuery(); + query.setStart("1h-ago"); + final QueryStats stats = new QueryStats(remote, query, headers); + stats.addStat(QueryStat.AGGREGATED_SIZE, 42); + stats.markSerializationSuccessful(); + assertEquals(42, stats.getStat(QueryStat.AGGREGATED_SIZE)); + assertEquals(-1, stats.getStat(QueryStat.BYTES_FROM_STORAGE)); + } + + @Test + public void getStatTime() throws Exception { + final TSQuery query = new TSQuery(); + query.setStart("1h-ago"); + final QueryStats stats = new QueryStats(remote, query, headers); + stats.markSerializationSuccessful(); + assertEquals(1000.0, stats.getTimeStat(QueryStat.PROCESSING_PRE_WRITE_TIME), 0.001); + assertEquals(Double.NaN, stats.getTimeStat(QueryStat.AVG_AGGREGATION_TIME), 0.001); + } +} diff --git a/test/storage/MockBase.java b/test/storage/MockBase.java index 26942cb651..699824283c 100644 --- a/test/storage/MockBase.java +++ b/test/storage/MockBase.java @@ -18,52 +18,71 @@ import static org.mockito.Mockito.when; import static org.powermock.api.mockito.PowerMockito.mock; -import java.io.IOException; -import java.lang.reflect.Field; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; import java.util.TreeMap; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; import javax.xml.bind.DatatypeConverter; +import net.opentsdb.core.Const; import net.opentsdb.core.TSDB; -import net.opentsdb.utils.Config; +import net.opentsdb.utils.Pair; import org.hbase.async.AtomicIncrementRequest; +import org.hbase.async.BinaryPrefixComparator; import org.hbase.async.Bytes; +import org.hbase.async.Bytes.ByteMap; +import org.hbase.async.AppendRequest; import org.hbase.async.DeleteRequest; +import org.hbase.async.FilterComparator; +import org.hbase.async.FilterList; import org.hbase.async.GetRequest; +import org.hbase.async.GetResultOrException; import org.hbase.async.HBaseClient; +import org.hbase.async.KeyRegexpFilter; import org.hbase.async.KeyValue; import org.hbase.async.PutRequest; +import org.hbase.async.QualifierFilter; +import org.hbase.async.RegexStringComparator; +import org.hbase.async.ScanFilter; import org.hbase.async.Scanner; import org.junit.Ignore; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; +import org.powermock.reflect.Whitebox; +import com.google.common.collect.Lists; import com.stumbleupon.async.Deferred; /** * Mock HBase implementation useful in testing calls to and from storage with * actual pretend data. The underlying data store is an incredibly ugly nesting - * of ByteMaps from AsyncHbase so it stores and orders byte arrays similar to - * HBase. A MockBase instance represents a SINGLE table in HBase but it provides - * support for column families and timestamped entries. + * of ByteMaps from AsyncHbase so it stores and orders byte arrays similar to + * HBase. It supports tables and column families along with timestamps but + * doesn't deal with TTLs or other features. + * <p> + * By default we configure the "'tsdb', {NAME => 't'}" and + * "'tsdb-uid', {NAME => 'id'}, {NAME => 'name'}" tables. If you need more, just + * add em. + * * <p> * It's not a perfect mock but is useful for the majority of unit tests. Gets, * puts, cas, deletes and scans are currently supported. See notes for each * inner class below about what does and doesn't work. * <p> - * Regarding timestamps, whenever you execute an RPC request, the + * Regarding timestamps, whenever you execute an RPC request, the * {@code current_timestamp} will be incremented by one millisecond. By default * the timestamp starts at 1/1/2014 00:00:00 but you can set it to any value - * at any time. If a PutRequest comes in with a specific time, that time will + * at any time. If a PutRequest comes in with a specific time, that time will * be stored and the timestamp will not be incremented. * <p> * <b>Warning:</b> To use this class, you need to prepare the classes for testing @@ -72,6 +91,7 @@ * <li>HBaseClient</li> * <li>GetRequest</li> * <li>PutRequest</li> + * <li>AppendRequest</li> * <li>KeyValue</li> * <li>Scanner</li> * <li>DeleteRequest</li> @@ -82,16 +102,27 @@ public final class MockBase { private static final Charset ASCII = Charset.forName("ISO-8859-1"); private TSDB tsdb; - - // KEY Column Family Qualifier Timestamp Value - private Bytes.ByteMap<Bytes.ByteMap<Bytes.ByteMap<TreeMap<Long, byte[]>>>> - storage = new Bytes.ByteMap<Bytes.ByteMap<Bytes.ByteMap<TreeMap<Long, byte[]>>>>(); + + /** Gross huh? <table, <cf, <row, <qual, <ts, value>>>>> + * Why is CF before row? Because we want to throw exceptions if a CF hasn't + * been "configured" + */ + private ByteMap<ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>>> + storage = new ByteMap<ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>>>(); private HashSet<MockScanner> scanners = new HashSet<MockScanner>(2); + + /** The default family for shortcuts */ private byte[] default_family; - + + /** The default table for shortcuts */ + private byte[] default_table; + /** Incremented every time a new value is stored (without a timestamp) */ private long current_timestamp = 1388534400000L; - + + /** A list of exceptions that can be thrown when working with a row key */ + private ByteMap<Pair<RuntimeException, Boolean>> exceptions; + /** * Setups up mock intercepts for all of the calls. Depending on the given * flags, some mocks may not be enabled, allowing local unit tests to setup @@ -105,37 +136,28 @@ public final class MockBase { * @param default_delete Enable the default .delete() mock * @param default_scan Enable the Scanner mock implementation */ + @SuppressWarnings("unchecked") public MockBase( final TSDB tsdb, final HBaseClient client, - final boolean default_get, + final boolean default_get, final boolean default_put, final boolean default_delete, final boolean default_scan) { this.tsdb = tsdb; - - default_family = "t".getBytes(ASCII); // set a default - + + default_family = "t".getBytes(ASCII); + default_table = "tsdb".getBytes(ASCII); + setupDefaultTables(); + // replace the "real" field objects with mocks - Field cl; - try { - cl = tsdb.getClass().getDeclaredField("client"); - cl.setAccessible(true); - cl.set(tsdb, client); - cl.setAccessible(false); - } catch (SecurityException e) { - e.printStackTrace(); - } catch (NoSuchFieldException e) { - e.printStackTrace(); - } catch (IllegalArgumentException e) { - e.printStackTrace(); - } catch (IllegalAccessException e) { - e.printStackTrace(); - } + Whitebox.setInternalState(tsdb, "client", client); // Default get answer will return one or more columns from the requested row if (default_get) { when(client.get((GetRequest)any())).thenAnswer(new MockGet()); } + + when(client.get(any(List.class))).thenAnswer(new MockMultiGet(client)); // Default put answer will store the given values in the proper location. if (default_put) { @@ -147,7 +169,7 @@ public MockBase( if (default_delete) { when(client.delete((DeleteRequest)any())).thenAnswer(new MockDelete()); } - + if (default_scan) { // to facilitate unit tests where more than one scanner is used (i.e. in a // callback chain) we have to provide a new mock scanner for each new @@ -158,150 +180,323 @@ public MockBase( @Override public Scanner answer(InvocationOnMock arg0) throws Throwable { final Scanner scanner = mock(Scanner.class); - scanners.add(new MockScanner(scanner)); + final byte[] table = (byte[])arg0.getArguments()[0]; + scanners.add(new MockScanner(scanner, table)); return scanner; } - - }); + + }); } - + when(client.atomicIncrement((AtomicIncrementRequest)any())) .then(new MockAtomicIncrement()); when(client.bufferAtomicIncrement((AtomicIncrementRequest)any())) - .then(new MockAtomicIncrement()); + .then(new MockAtomicIncrement()); + when(client.append((AppendRequest)any())).thenAnswer(new MockAppend()); + } + + /** + * Add a table with families to the data store. If the table or family + * exists, it's a no-op. Give real values as we don't check `em. + * @param table The table to add + * @param families A list of one or more famlies to add to the table + */ + public void addTable(final byte[] table, final List<byte[]> families) { + ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>> map = storage.get(table); + if (map == null) { + map = new ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>>(); + storage.put(table, map); + } + for (final byte[] family : families) { + if (!map.containsKey(family)) { + map.put(family, new ByteMap<ByteMap<TreeMap<Long, byte[]>>>()); + } + } + } + + /** + * Pops the table out of the map + * @param table The table to pop + * @return True if the table was there, false if it wasn't. + */ + public boolean deleteTable(final byte[] table) { + return storage.remove(table) != null; } - /** @param family Sets the family for calls that need it */ + /** @param family Sets the default family for calls that need it */ public void setFamily(final byte[] family) { - this.default_family = family; + default_family = family; } - + + /** @param table Sets the default table for calls that need it */ + public void setDefaultTable(final byte[] table) { + default_table = table; + } + /** @param timestamp The timestamp to use for further storage increments */ public void setCurrentTimestamp(final long timestamp) { this.current_timestamp = timestamp; } - + /** @return the incrementing timestamp */ public long getCurrentTimestamp() { return current_timestamp; } - + + /** - * Add a column to the hash table using the default column family. - * The proper row will be created if it doesn't exist. If the column already - * exists, the original value will be overwritten with the new data + * Add a column to the hash table using the default column family. + * The proper row will be created if it doesn't exist. If the column already + * exists, the original value will be overwritten with the new data. + * Uses the default table and family + * @param key The row key + * @param qualifier The qualifier + * @param value The value to store + * @param timestamp The timestamp of cell + */ + public void addColumn(final byte[] key, final byte[] qualifier, + final byte[] value, final long timestamp) { + addColumn(default_table, key, default_family, qualifier, value, + timestamp); + } + + /** + * Add a column to the hash table using the default column family. + * The proper row will be created if it doesn't exist. If the column already + * exists, the original value will be overwritten with the new data. + * Uses the default table and family * @param key The row key * @param qualifier The qualifier * @param value The value to store */ - public void addColumn(final byte[] key, final byte[] qualifier, + public void addColumn(final byte[] key, final byte[] qualifier, final byte[] value) { - addColumn(key, default_family, qualifier, value, current_timestamp++); + addColumn(default_table, key, default_family, qualifier, value, + current_timestamp++); } - + + /** + * Add a column to the hash table + * The proper row will be created if it doesn't exist. If the column already + * exists, the original value will be overwritten with the new data. + * Uses the default table. + * @param key The row key + * @param family The column family to store the value in + * @param qualifier The qualifier + * @param value The value to store + */ + public void addColumn(final byte[] key, final byte[] family, + final byte[] qualifier, final byte[] value) { + addColumn(default_table, key, family, qualifier, value, current_timestamp++); + } + /** - * Add a column to the hash table - * The proper row will be created if it doesn't exist. If the column already + * Add a column to the hash table + * The proper row will be created if it doesn't exist. If the column already * exists, the original value will be overwritten with the new data + * @param table The table * @param key The row key * @param family The column family to store the value in * @param qualifier The qualifier * @param value The value to store */ - public void addColumn(final byte[] key, final byte[] family, + public void addColumn(final byte[] table, final byte[] key, final byte[] family, final byte[] qualifier, final byte[] value) { - addColumn(key, family, qualifier, value, current_timestamp++); + addColumn(table, key, family, qualifier, value, current_timestamp++); } - + /** - * Add a column to the hash table - * The proper row will be created if it doesn't exist. If the column already + * Add a column to the hash table + * The proper row will be created if it doesn't exist. If the column already * exists, the original value will be overwritten with the new data + * @param table The table * @param key The row key * @param family The column family to store the value in * @param qualifier The qualifier * @param value The value to store * @param timestamp The timestamp to store */ - public void addColumn(final byte[] key, final byte[] family, + public void addColumn(final byte[] table, final byte[] key, final byte[] family, final byte[] qualifier, final byte[] value, final long timestamp) { // AsyncHBase will throw an NPE if the user tries to write a NULL value // so we better do the same. An empty value is ok though, i.e. new byte[] {} if (value == null) { throw new NullPointerException(); } - - Bytes.ByteMap<Bytes.ByteMap<TreeMap<Long, byte[]>>> row = storage.get(key); - if (row == null) { - row = new Bytes.ByteMap<Bytes.ByteMap<TreeMap<Long, byte[]>>>(); - storage.put(key, row); + final ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>> map = storage.get(table); + if (map == null) { + throw new RuntimeException( + "No such table " + Bytes.pretty(table)); } - - Bytes.ByteMap<TreeMap<Long, byte[]>> cf = row.get(family); + final ByteMap<ByteMap<TreeMap<Long, byte[]>>> cf = map.get(family); if (cf == null) { - cf = new Bytes.ByteMap<TreeMap<Long, byte[]>>(); - row.put(family, cf); + throw new RuntimeException( + "No such CF " + Bytes.pretty(family)); + } + + ByteMap<TreeMap<Long, byte[]>> row = cf.get(key); + if (row == null) { + row = new ByteMap<TreeMap<Long, byte[]>>(); + cf.put(key, row); } - TreeMap<Long, byte[]> column = cf.get(qualifier); + + TreeMap<Long, byte[]> column = row.get(qualifier); if (column == null) { // remember, most recent at the top! column = new TreeMap<Long, byte[]>(Collections.reverseOrder()); - cf.put(qualifier, column); + row.put(qualifier, column); } column.put(timestamp, value); } - - /** @return TTotal number of rows in the hash table */ + + /** + * Stores an exception so that any operation on the given key will cause it + * to be thrown. + * @param key The key to go pear shaped on + * @param exception The exception to throw + */ + public void throwException(final byte[] key, final RuntimeException exception) { + throwException(key, exception, true); + } + + /** + * Stores an exception so that any operation on the given key will cause it + * to be thrown. + * @param key The key to go pear shaped on + * @param exception The exception to throw + * @param as_result Whether or not to return the exception in the deferred + * result or throw it outright. + */ + public void throwException(final byte[] key, final RuntimeException exception, + final boolean as_result) { + if (exceptions == null) { + exceptions = new ByteMap<Pair<RuntimeException, Boolean>>(); + } + exceptions.put(key, new Pair<RuntimeException, Boolean>(exception, as_result)); + } + + /** Removes all exceptions from the exception list */ + public void clearExceptions() { + exceptions.clear(); + } + + /** @return Total number of unique rows in the default table. Returns 0 if the + * default table does not exist */ public int numRows() { - return storage.size(); + return numRows(default_table); } - + + /** + * Total number of rows in the given table. Returns 0 if the table does not exit. + * @param table The table to scan + * @return The number of rows + */ + public int numRows(final byte[] table) { + final ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>> map = + storage.get(table); + if (map == null) { + return 0; + } + final ByteMap<Void> unique_rows = new ByteMap<Void>(); + for (final ByteMap<ByteMap<TreeMap<Long, byte[]>>> cf : map.values()) { + for (final byte[] key : cf.keySet()) { + unique_rows.put(key, null); + } + } + return unique_rows.size(); + } + /** - * Return the total number of column families for the row + * Return the total number of column families for the row in the default table * @param key The row to search for - * @return -1 if the row did not exist, otherwise the number of column families. + * @return -1 if the table or row did not exist, otherwise the number of + * column families. */ public int numColumnFamilies(final byte[] key) { - final Bytes.ByteMap<Bytes.ByteMap<TreeMap<Long, byte[]>>> row = - storage.get(key); - if (row == null) { + return numColumnFamilies(default_table, key); + } + + /** + * Return the number of column families for the given row key in the given table. + * @param table The table to iterate over + * @param key The row to search for + * @return -1 if the table or row did not exist, otherwise the number of + * column families. + */ + public int numColumnFamilies(final byte[] table, final byte[] key) { + final ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>> map = + storage.get(table); + if (map == null) { return -1; } - return row.size(); + int sum = 0; + for (final ByteMap<ByteMap<TreeMap<Long, byte[]>>> cf : map.values()) { + if (cf.containsKey(key)) { + ++sum; + } + } + return sum == 0 ? -1 : sum; } - + /** - * Total number of columns in the given row across all column families + * Total number of columns in the given row across all column families in the + * default table * @param key The row to search for * @return -1 if the row did not exist, otherwise the number of columns. */ public long numColumns(final byte[] key) { - final Bytes.ByteMap<Bytes.ByteMap<TreeMap<Long, byte[]>>> row = - storage.get(key); - if (row == null) { + return numColumns(default_table, key); + } + + /** + * Total number of columns in the given row across all column families in the + * default table + * @param table The table to iterate over + * @param key The row to search for + * @return -1 if the row did not exist, otherwise the number of columns. + */ + public long numColumns(final byte[] table, final byte[] key) { + final ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>> map = + storage.get(table); + if (map == null) { return -1; } long size = 0; - for (Map.Entry<byte[], Bytes.ByteMap<TreeMap<Long, byte[]>>> entry : row) { - size += entry.getValue().size(); + for (final ByteMap<ByteMap<TreeMap<Long, byte[]>>> cf : map.values()) { + final ByteMap<TreeMap<Long, byte[]>> row = cf.get(key); + if (row != null) { + size += row.size(); + } } - return size; + return size == 0 ? -1 : size; } - + /** - * Return the total number of columns for a specific row and family + * Return the total number of columns for a specific row and family in the + * default table * @param key The row to search for * @param family The column family to search for * @return -1 if the row did not exist, otherwise the number of columns. */ public int numColumnsInFamily(final byte[] key, final byte[] family) { - final Bytes.ByteMap<Bytes.ByteMap<TreeMap<Long, byte[]>>> row = - storage.get(key); - if (row == null) { + return numColumnsInFamily(default_table, key, family); + } + + /** + * Return the total number of columns for a specific row and family + * @param key The row to search for + * @param family The column family to search for + * @return -1 if the row did not exist, otherwise the number of columns. + */ + public int numColumnsInFamily(final byte[] table, final byte[] key, + final byte[] family) { + final ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>> map = + storage.get(table); + if (map == null) { return -1; } - final Bytes.ByteMap<TreeMap<Long, byte[]>> cf = row.get(family); + final ByteMap<ByteMap<TreeMap<Long, byte[]>>> cf = map.get(family); if (cf == null) { return -1; } @@ -310,33 +505,51 @@ public int numColumnsInFamily(final byte[] key, final byte[] family) { /** * Retrieve the most recent contents of a single column with the default family + * and in the default table * @param key The row key of the column * @param qualifier The column qualifier * @return The byte array of data or null if not found */ public byte[] getColumn(final byte[] key, final byte[] qualifier) { - return getColumn(key, default_family, qualifier); + return getColumn(default_table, key, default_family, qualifier); } - + /** - * Retrieve the most recent contents of a single column + * Retrieve the most recent contents of a single column with the default table * @param key The row key of the column * @param family The column family * @param qualifier The column qualifier * @return The byte array of data or null if not found */ - public byte[] getColumn(final byte[] key, final byte[] family, + public byte[] getColumn(final byte[] key, final byte[] family, final byte[] qualifier) { - final Bytes.ByteMap<Bytes.ByteMap<TreeMap<Long, byte[]>>> row = - storage.get(key); - if (row == null) { + return getColumn(default_table, key, family, qualifier); + } + + /** + * Retrieve the most recent contents of a single column + * @param table The table to fetch from + * @param key The row key of the column + * @param family The column family + * @param qualifier The column qualifier + * @return The byte array of data or null if not found + */ + public byte[] getColumn(final byte[] table, final byte[] key, + final byte[] family, final byte[] qualifier) { + final ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>> map = + storage.get(table); + if (map == null) { return null; } - final Bytes.ByteMap<TreeMap<Long, byte[]>> cf = row.get(family); + final ByteMap<ByteMap<TreeMap<Long, byte[]>>> cf = map.get(family); if (cf == null) { return null; } - final TreeMap<Long, byte[]> column = cf.get(qualifier); + final ByteMap<TreeMap<Long, byte[]>> row = cf.get(key); + if (row == null) { + return null; + } + final TreeMap<Long, byte[]> column = row.get(qualifier); if (column == null) { return null; } @@ -344,68 +557,116 @@ public byte[] getColumn(final byte[] key, final byte[] family, } /** - * Retrieve the full map of timestamps and values of a single column with - * the default family + * Retrieve the full map of timestamps and values of a single column with + * the default family and default table * @param key The row key of the column * @param qualifier The column qualifier * @return The byte array of data or null if not found */ - public TreeMap<Long, byte[]> getFullColumn(final byte[] key, + public TreeMap<Long, byte[]> getFullColumn(final byte[] key, final byte[] qualifier) { - return getFullColumn(key, default_family, qualifier); + return getFullColumn(default_table, key, default_family, qualifier); } - + /** * Retrieve the full map of timestamps and values of a single column + * @param table The table to fetch from * @param key The row key of the column * @param family The column family * @param qualifier The column qualifier * @return The tree map of timestamps and values or null if not found */ - public TreeMap<Long, byte[]> getFullColumn(final byte[] key, + public TreeMap<Long, byte[]> getFullColumn(final byte[] table, final byte[] key, final byte[] family, final byte[] qualifier) { - final Bytes.ByteMap<Bytes.ByteMap<TreeMap<Long, byte[]>>> row = - storage.get(key); - if (row == null) { + final ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>> map = + storage.get(table); + if (map == null) { return null; } - final Bytes.ByteMap<TreeMap<Long, byte[]>> cf = row.get(family); + final ByteMap<ByteMap<TreeMap<Long, byte[]>>> cf = map.get(family); if (cf == null) { return null; } - final TreeMap<Long, byte[]> column = cf.get(qualifier); - if (column == null) { + final ByteMap<TreeMap<Long, byte[]>> row = cf.get(key); + if (row == null) { return null; } - return column; + return row.get(qualifier); } - + /** * Returns the most recent value from all columns for a given column family + * in the default table * @param key The row key * @param family The column family ID * @return A map of columns if the CF was found, null if no such CF */ - public Bytes.ByteMap<byte[]> getColumnFamily(final byte[] key, + public ByteMap<byte[]> getColumnFamily(final byte[] key, final byte[] family) { - final Bytes.ByteMap<Bytes.ByteMap<TreeMap<Long, byte[]>>> row = - storage.get(key); - if (row == null) { + return getColumnFamily(default_table, key , family); + } + + /** + * Returns the most recent value from all columns for a given column family + * @param table The table to fetch from + * @param key The row key + * @param family The column family ID + * @return A map of columns if the CF was found, null if no such CF + */ + public ByteMap<byte[]> getColumnFamily(final byte[] table, final byte[] key, + final byte[] family) { + final ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>> map = + storage.get(table); + if (map == null) { return null; } - final Bytes.ByteMap<TreeMap<Long, byte[]>> cf = row.get(family); + final ByteMap<ByteMap<TreeMap<Long, byte[]>>> cf = map.get(family); if (cf == null) { return null; } + final ByteMap<TreeMap<Long, byte[]>> row = cf.get(key); + if (row == null) { + return null; + } // convert to a <qualifier, value> byte map - final Bytes.ByteMap<byte[]> columns = new Bytes.ByteMap<byte[]>(); - for (Map.Entry<byte[], TreeMap<Long, byte[]>> entry : cf.entrySet()) { + final ByteMap<byte[]> columns = new ByteMap<byte[]>(); + for (Entry<byte[], TreeMap<Long, byte[]>> entry : row.entrySet()) { // the <timestamp, value> map should never be null columns.put(entry.getKey(), entry.getValue().firstEntry().getValue()); } return columns; } - + + /** @return the list of keys stored in the default table for all CFs */ + public Set<byte[]> getKeys() { + return getKeys(default_table); + } + + /** + * Return the list of unique keys in the given table for all CFs + * @param table The table to pull from + * @return A list of keys. May be null if the table doesn't exist + */ + public Set<byte[]> getKeys(final byte[] table) { + final ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>> map = + storage.get(table); + if (map == null) { + return null; + } + final ByteMap<Void> unique_rows = new ByteMap<Void>(); + for (final ByteMap<ByteMap<TreeMap<Long, byte[]>>> cf : map.values()) { + for (final byte[] key : cf.keySet()) { + unique_rows.put(key, null); + } + } + return unique_rows.keySet(); + } + + /** @return The set of scanners configured by the caller */ + public HashSet<MockScanner> getScanners() { + return scanners; + } + /** * Return the mocked TSDB object to use for HBaseClient access * @return @@ -413,53 +674,162 @@ public Bytes.ByteMap<byte[]> getColumnFamily(final byte[] key, public TSDB getTSDB() { return tsdb; } - + + /** + * Runs through all rows in the "tsdb" table and compacts them by making a + * call to the {@link TSDB.compact} method. It will delete any columns + * that were compacted and leave others untouched, just as the normal + * method does. + * And only iterates over the 't' family. + * @throws Exception if Whitebox couldn't access the compact method + */ + public void tsdbCompactAllRows() throws Exception { + final ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>> map = + storage.get("tsdb".getBytes(ASCII)); + if (map == null) { + return; + } + final ByteMap<ByteMap<TreeMap<Long, byte[]>>> cf = map.get("t".getBytes(ASCII)); + if (cf == null) { + return; + } + + for (Entry<byte[], ByteMap<TreeMap<Long, byte[]>>> entry : cf.entrySet()) { + final byte[] key = entry.getKey(); + + final ByteMap<TreeMap<Long, byte[]>> row = entry.getValue(); + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(row.size()); + final Set<byte[]> deletes = new HashSet<byte[]>(); + for (Map.Entry<byte[], TreeMap<Long, byte[]>> column : row.entrySet()) { + if (column.getKey().length % 2 == 0) { + kvs.add(new KeyValue(key, default_family, column.getKey(), + column.getValue().firstKey(), + column.getValue().firstEntry().getValue())); + deletes.add(column.getKey()); + } + } + if (kvs.size() > 0) { + for (final byte[] k : deletes) { + row.remove(k); + } + final KeyValue compacted = + Whitebox.invokeMethod(tsdb, "compact", kvs, Collections.EMPTY_LIST, + Collections.EMPTY_LIST); + final TreeMap<Long, byte[]> compacted_value = new TreeMap<Long, byte[]>(); + compacted_value.put(current_timestamp++, compacted.value()); + row.put(compacted.qualifier(), compacted_value); + } + } + } + /** - * Clears the entire hash table. Use it if your unit test needs to start fresh + * Clears out all rows from storage but doesn't delete the tables or families. */ public void flushStorage() { - storage.clear(); + for (final ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>> table : + storage.values()) { + for (final ByteMap<ByteMap<TreeMap<Long, byte[]>>> cf : table.values()) { + cf.clear(); + } + } } - + + /** + * Clears out all rows for a given table + * @param table The table to empty out + */ + public void flushStorage(final byte[] table) { + final ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>> map = storage.get(table); + if (map == null) { + return; + } + for (final ByteMap<ByteMap<TreeMap<Long, byte[]>>> cf : map.values()) { + cf.clear(); + } + } + /** - * Removes the entire row from the hash table + * Removes the entire row from the default table for all column families * @param key The row to remove */ public void flushRow(final byte[] key) { - storage.remove(key); + flushRow(default_table, key); } - + + /** + * Removes the entire row from the table for all column families + * @param table The table to purge + * @param key The row to remove + */ + public void flushRow(final byte[] table, final byte[] key) { + final ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>> map = storage.get(table); + if (map == null) { + return; + } + for (final ByteMap<ByteMap<TreeMap<Long, byte[]>>> cf : map.values()) { + cf.remove(key); + } + } + /** - * Removes the entire column family from the hash table for ALL rows + * Removes all rows from the default table for the given column family * @param family The family to remove */ public void flushFamily(final byte[] family) { - for (Map.Entry<byte[], Bytes.ByteMap<Bytes.ByteMap<TreeMap<Long, byte[]>>>> row : - storage.entrySet()) { - row.getValue().remove(family); + flushFamily(default_table, family); + } + + /** + * Removes all rows from the default table for the given column family + * @param table The table to purge from + * @param family The family to remove + */ + public void flushFamily(final byte[] table, final byte[] family) { + final ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>> map = storage.get(table); + if (map == null) { + return; + } + final ByteMap<ByteMap<TreeMap<Long, byte[]>>> cf = map.get(family); + if (cf != null) { + cf.clear(); } } - + /** - * Removes the given column from the hash map + * Removes the given column from the default table * @param key Row key * @param family Column family * @param qualifier Column qualifier */ - public void flushColumn(final byte[] key, final byte[] family, + public void flushColumn(final byte[] key, final byte[] family, final byte[] qualifier) { - final Bytes.ByteMap<Bytes.ByteMap<TreeMap<Long, byte[]>>> row = - storage.get(key); - if (row == null) { + flushColumn(default_table, key, family, qualifier); + } + + /** + * Removes the given column from the table + * @param table The table to purge from + * @param key Row key + * @param family Column family + * @param qualifier Column qualifier + */ + public void flushColumn(final byte[] table, final byte[] key, + final byte[] family, final byte[] qualifier) { + final ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>> map = storage.get(table); + if (map == null) { return; } - final Bytes.ByteMap<TreeMap<Long, byte[]>> cf = row.get(family); + final ByteMap<ByteMap<TreeMap<Long, byte[]>>> cf = map.get(family); if (cf == null) { return; } - cf.remove(qualifier); + final ByteMap<TreeMap<Long, byte[]>> row = cf.get(key); + if (row == null) { + return; + } + row.remove(qualifier); } - + /** * Dumps the entire storage hash to stdout in a sort of tree style format with * all byte arrays hex encoded @@ -467,7 +837,7 @@ public void flushColumn(final byte[] key, final byte[] family, public void dumpToSystemOut() { dumpToSystemOut(false); } - + /** * Dumps the entire storage hash to stdout in a sort of tree style format * @param ascii Whether or not the values should be converted to ascii @@ -477,33 +847,35 @@ public void dumpToSystemOut(final boolean ascii) { System.out.println("Storage is Empty"); return; } - - for (Map.Entry<byte[], Bytes.ByteMap<Bytes.ByteMap<TreeMap<Long, byte[]>>>> row : + + for (Entry<byte[], ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>>> table : storage.entrySet()) { - System.out.println("[Row] " + (ascii ? new String(row.getKey(), ASCII) : - bytesToString(row.getKey()))); - - for (Map.Entry<byte[], Bytes.ByteMap<TreeMap<Long, byte[]>>> cf : - row.getValue().entrySet()) { - - final String family = ascii ? new String(cf.getKey(), ASCII) : - bytesToString(cf.getKey()); - System.out.println(" [CF] " + family); - - for (Map.Entry<byte[], TreeMap<Long, byte[]>> column : cf.getValue().entrySet()) { - System.out.println(" [Qual] " + (ascii ? - "\"" + new String(column.getKey(), ASCII) + "\"" - : bytesToString(column.getKey()))); - for (Map.Entry<Long, byte[]> cell : column.getValue().entrySet()) { - System.out.println(" [TS] " + cell.getKey() + " [Value] " + - (ascii ? new String(cell.getValue(), ASCII) - : bytesToString(cell.getValue()))); + System.out.println("[Table] " + new String(table.getKey(), ASCII)); + + for (Entry<byte[], ByteMap<ByteMap<TreeMap<Long, byte[]>>>> cf : + table.getValue().entrySet()) { + System.out.println(" [CF] " + new String(cf.getKey(), ASCII)); + + for (Entry<byte[], ByteMap<TreeMap<Long, byte[]>>> row : + cf.getValue().entrySet()) { + System.out.println(" [Row] " + (ascii ? + new String(row.getKey(), ASCII) : bytesToString(row.getKey()))); + + for (Map.Entry<byte[], TreeMap<Long, byte[]>> column : row.getValue().entrySet()) { + System.out.println(" [Qual] " + (ascii ? + "\"" + new String(column.getKey(), ASCII) + "\"" + : bytesToString(column.getKey()))); + for (Map.Entry<Long, byte[]> cell : column.getValue().entrySet()) { + System.out.println(" [TS] " + cell.getKey() + " [Value] " + + (ascii ? new String(cell.getValue(), ASCII) + : bytesToString(cell.getValue()))); + } } } } } } - + /** * Helper to convert an array of bytes to a hexadecimal encoded string. * @param bytes The byte array to convert @@ -512,7 +884,7 @@ public void dumpToSystemOut(final boolean ascii) { public static String bytesToString(final byte[] bytes) { return DatatypeConverter.printHexBinary(bytes); } - + /** * Helper to convert a hex encoded string into a byte array. * <b>Warning:</b> This method won't pad the string to make sure it's an @@ -525,12 +897,12 @@ public static String bytesToString(final byte[] bytes) { public static byte[] stringToBytes(final String bytes) { return DatatypeConverter.parseHexBinary(bytes); } - + /** @return Returns the ASCII character set */ public static Charset ASCII() { return ASCII; } - + /** * Concatenates byte arrays into one big array * @param arrays Any number of arrays to concatenate @@ -549,10 +921,26 @@ public static byte[] concatByteArrays(final byte[]... arrays) { } return result; } - + + /** Creates the TSDB and UID tables */ + private void setupDefaultTables() { + final ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>> tsdb = + new ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>>(); + tsdb.put("t".getBytes(ASCII), new ByteMap<ByteMap<TreeMap<Long, byte[]>>>()); + storage.put("tsdb".getBytes(ASCII), tsdb); + + final ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>> tsdb_uid = + new ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>>(); + tsdb_uid.put("name".getBytes(ASCII), + new ByteMap<ByteMap<TreeMap<Long, byte[]>>>()); + tsdb_uid.put("id".getBytes(ASCII), + new ByteMap<ByteMap<TreeMap<Long, byte[]>>>()); + storage.put("tsdb-uid".getBytes(ASCII), tsdb_uid); + } + /** * Gets one or more columns from a row. If the row does not exist, a null is - * returned. If no qualifiers are given, the entire row is returned. + * returned. If no qualifiers are given, the entire row is returned. * NOTE: all timestamp, value pairs are returned. */ private class MockGet implements Answer<Deferred<ArrayList<KeyValue>>> { @@ -561,59 +949,93 @@ public Deferred<ArrayList<KeyValue>> answer(InvocationOnMock invocation) throws Throwable { final Object[] args = invocation.getArguments(); final GetRequest get = (GetRequest)args[0]; - - final Bytes.ByteMap<Bytes.ByteMap<TreeMap<Long, byte[]>>> row = - storage.get(get.key()); - if (row == null) { - return Deferred.fromResult((ArrayList<KeyValue>)null); - } - - final byte[] family = get.family(); - if (family != null && family.length > 0) { - if (!row.containsKey(family)) { - return Deferred.fromResult((ArrayList<KeyValue>)null); + if (exceptions != null) { + final Pair<RuntimeException, Boolean> ex = exceptions.get(get.key()); + if (ex != null) { + if (ex.getValue()) { + return Deferred.fromError(ex.getKey()); + } else { + throw ex.getKey(); + } } } - + + final ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>> map = + storage.get(get.table()); + if (map == null) { + return Deferred.fromError(new RuntimeException( + "No such table " + Bytes.pretty(get.table()))); + } + // compile a set of qualifiers to use as a filter if necessary - Bytes.ByteMap<Object> qualifiers = new Bytes.ByteMap<Object>(); - if (get.qualifiers() != null && get.qualifiers().length > 0) { + final ByteMap<Object> qualifiers = new ByteMap<Object>(); + if (get.qualifiers() != null && get.qualifiers().length > 0) { for (byte[] q : get.qualifiers()) { qualifiers.put(q, null); } } - - final ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(row.size()); - for (Map.Entry<byte[], Bytes.ByteMap<TreeMap<Long, byte[]>>> cf : - row.entrySet()) { - - // column family filter - if (family != null && family.length > 0 && - !Bytes.equals(family, cf.getKey())) { + + final ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(); + for (final Entry<byte[], ByteMap<ByteMap<TreeMap<Long, byte[]>>>> cf : + map.entrySet()) { + if (get.family() != null && Bytes.memcmp(get.family(), cf.getKey()) != 0) { continue; } - - for (Map.Entry<byte[], TreeMap<Long, byte[]>> column : - cf.getValue().entrySet()) { - // qualifier filter + + final ByteMap<TreeMap<Long, byte[]>> row = cf.getValue().get(get.key()); + if (row == null) { + continue; + } + + for (Entry<byte[], TreeMap<Long, byte[]>> column : row.entrySet()) { if (!qualifiers.isEmpty() && !qualifiers.containsKey(column.getKey())) { continue; } - - // TODO - if we want to support multiple values, iterate over the + + // TODO - if we want to support multiple values, iterate over the // tree map. Otherwise Get returns just the latest value. - KeyValue kv = mock(KeyValue.class); - when(kv.timestamp()).thenReturn(column.getValue().firstKey()); - when(kv.value()).thenReturn(column.getValue().firstEntry().getValue()); - when(kv.qualifier()).thenReturn(column.getKey()); - when(kv.key()).thenReturn(get.key()); - kvs.add(kv); + kvs.add(new KeyValue(get.key(), cf.getKey(), column.getKey(), + column.getValue().firstKey(), + column.getValue().firstEntry().getValue())); } } + if (kvs.isEmpty()) { + return Deferred.fromResult(null); + } return Deferred.fromResult(kvs); } } + + /** + * Handles a multi-get call by routing individual requests to the MockGet + */ + private class MockMultiGet implements + Answer<Deferred<List<GetResultOrException>>> { + final HBaseClient client; + public MockMultiGet(final HBaseClient client) { + this.client = client; + } + + @Override + public Deferred<List<GetResultOrException>> answer( + final InvocationOnMock invocation) throws Throwable { + final Object[] args = invocation.getArguments(); + @SuppressWarnings("unchecked") + final List<GetRequest> gets = (List<GetRequest>) args[0]; + + final List<GetResultOrException> results = Lists.newArrayList(); + for (final GetRequest get : gets) { + try { + // just reuse the logic above. + results.add(new GetResultOrException(client.get(get).join())); + } catch (Exception e) { + results.add(new GetResultOrException(e)); + } + } + return Deferred.fromResult(results); + } + } /** * Stores one or more columns in a row. If the row does not exist, it's @@ -621,189 +1043,346 @@ public Deferred<ArrayList<KeyValue>> answer(InvocationOnMock invocation) */ private class MockPut implements Answer<Deferred<Boolean>> { @Override - public Deferred<Boolean> answer(final InvocationOnMock invocation) + public Deferred<Boolean> answer(final InvocationOnMock invocation) throws Throwable { final Object[] args = invocation.getArguments(); final PutRequest put = (PutRequest)args[0]; - Bytes.ByteMap<Bytes.ByteMap<TreeMap<Long, byte[]>>> row = - storage.get(put.key()); - if (row == null) { - row = new Bytes.ByteMap<Bytes.ByteMap<TreeMap<Long, byte[]>>>(); - storage.put(put.key(), row); + if (exceptions != null) { + final Pair<RuntimeException, Boolean> ex = exceptions.get(put.key()); + if (ex != null) { + if (ex.getValue()) { + return Deferred.fromError(ex.getKey()); + } else { + throw ex.getKey(); + } + } } - - Bytes.ByteMap<TreeMap<Long, byte[]>> cf = row.get(put.family()); + + final ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>> map = + storage.get(put.table()); + if (map == null) { + return Deferred.fromError(new RuntimeException( + "No such table " + Bytes.pretty(put.table()))); + } + + final ByteMap<ByteMap<TreeMap<Long, byte[]>>> cf = map.get(put.family()); if (cf == null) { - cf = new Bytes.ByteMap<TreeMap<Long, byte[]>>(); - row.put(put.family(), cf); + return Deferred.fromError(new RuntimeException( + "No such CF " + Bytes.pretty(put.table()))); } - + + ByteMap<TreeMap<Long, byte[]>> row = cf.get(put.key()); + if (row == null) { + row = new ByteMap<TreeMap<Long, byte[]>>(); + cf.put(put.key(), row); + } + for (int i = 0; i < put.qualifiers().length; i++) { - TreeMap<Long, byte[]> column = cf.get(put.qualifiers()[i]); + TreeMap<Long, byte[]> column = row.get(put.qualifiers()[i]); if (column == null) { column = new TreeMap<Long, byte[]>(Collections.reverseOrder()); - cf.put(put.qualifiers()[i], column); + row.put(put.qualifiers()[i], column); + } else { + long storedTs = column.firstKey(); + if(put.timestamp() >= storedTs) { + column.clear(); + } else { + continue; + } } - - column.put(put.timestamp() != Long.MAX_VALUE ? put.timestamp() : + + column.put(put.timestamp() != Long.MAX_VALUE ? put.timestamp() : current_timestamp++, put.values()[i]); + assert column.size() == 1 : "Since max versions allowed is 1, there can " + + "never be two entries at similar timestamp. To resolve change the " + + "code to only keep the entry with higher timestamp"; } - + return Deferred.fromResult(true); } } - + + /** + * Stores one or more columns in a row. If the row does not exist, it's + * created. + */ + private class MockAppend implements Answer<Deferred<Boolean>> { + @Override + public Deferred<Boolean> answer(final InvocationOnMock invocation) + throws Throwable { + final Object[] args = invocation.getArguments(); + final AppendRequest append = (AppendRequest)args[0]; + + if (exceptions != null) { + final Pair<RuntimeException, Boolean> ex = exceptions.get(append.key()); + if (ex != null) { + if (ex.getValue()) { + return Deferred.fromError(ex.getKey()); + } else { + throw ex.getKey(); + } + } + } + + final ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>> map = + storage.get(append.table()); + if (map == null) { + return Deferred.fromError(new RuntimeException( + "No such table " + Bytes.pretty(append.table()))); + } + + final ByteMap<ByteMap<TreeMap<Long, byte[]>>> cf = map.get(append.family()); + if (cf == null) { + return Deferred.fromError(new RuntimeException( + "No such CF " + Bytes.pretty(append.table()))); + } + + ByteMap<TreeMap<Long, byte[]>> row = cf.get(append.key()); + if (row == null) { + row = new ByteMap<TreeMap<Long, byte[]>>(); + cf.put(append.key(), row); + } + + for (int i = 0; i < append.qualifiers().length; i++) { + TreeMap<Long, byte[]> column = row.get(append.qualifiers()[i]); + if (column == null) { + column = new TreeMap<Long, byte[]>(Collections.reverseOrder()); + row.put(append.qualifiers()[i], column); + } + + final byte[] values; + long column_timestamp = 0; + /* + * If there is no Map for Timestamp -> Value, then we create one, add the value with current timestamp + * else we fetch the top most KeyValue from the column map and append the value in the request with the value + * already present. The timestamp is also updated. The larger of either current time or the first timestamp from + * the map is incremented by 1 and used for the updated KeyValue. + */ + + if (column.isEmpty()) { + values = null; + column_timestamp = current_timestamp++; + } else { + values = column.firstEntry().getValue(); + column_timestamp = current_timestamp > column.firstKey() ? current_timestamp++ : column.firstKey() + 1; + } + + final int current_len = values != null ? values.length : 0; + final byte[] append_value = new byte[current_len + append.values()[i].length]; + if (current_len > 0) { + System.arraycopy(values, 0, append_value, 0, values.length); + } + + System.arraycopy(append.value(), 0, append_value, current_len, + append.values()[i].length); + // Remove all the old KeyValues, if any, since we need to have only 1 version of data + // Column_timestamp will hold the new timestamp and append_value holds the corresponding new data + column.clear(); + column.put(column_timestamp, append_value); + assert column.size() == 1 : "MockBase is designed to store only a single version of cell since OpenTSDB table schema for HBase is designed in that manner"; + } + + return Deferred.fromResult(true); + } + } + /** * Imitates the compareAndSet client call where a {@code PutRequest} is passed * along with a byte array to compared the stored value against. If the stored - * value doesn't match, the put is ignored and a "false" is returned. If the + * value doesn't match, the put is ignored and a "false" is returned. If the * comparator matches, the new put is recorded. * <b>Warning:</b> While a put works on multiple qualifiers, CAS only works * with one. So if the put includes more than one qualifier, only the first - * one will be processed in this CAS call. + * one will be processed in this CAS call. */ private class MockCAS implements Answer<Deferred<Boolean>> { - + @Override - public Deferred<Boolean> answer(final InvocationOnMock invocation) + public Deferred<Boolean> answer(final InvocationOnMock invocation) throws Throwable { final Object[] args = invocation.getArguments(); final PutRequest put = (PutRequest)args[0]; final byte[] expected = (byte[])args[1]; - - Bytes.ByteMap<Bytes.ByteMap<TreeMap<Long, byte[]>>> row = - storage.get(put.key()); - if (row == null) { - if (expected != null && expected.length > 0) { - return Deferred.fromResult(false); + + if (exceptions != null) { + final Pair<RuntimeException, Boolean> ex = exceptions.get(put.key()); + if (ex != null) { + if (ex.getValue()) { + return Deferred.fromError(ex.getKey()); + } else { + throw ex.getKey(); + } } - - row = new Bytes.ByteMap<Bytes.ByteMap<TreeMap<Long, byte[]>>>(); - storage.put(put.key(), row); } - - Bytes.ByteMap<TreeMap<Long, byte[]>> cf = row.get(put.family()); + + final ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>> map = + storage.get(put.table()); + if (map == null) { + return Deferred.fromError(new RuntimeException( + "No such table " + Bytes.pretty(put.table()))); + } + + final ByteMap<ByteMap<TreeMap<Long, byte[]>>> cf = map.get(put.family()); if (cf == null) { + return Deferred.fromError(new RuntimeException( + "No such CF " + Bytes.pretty(put.table()))); + } + + ByteMap<TreeMap<Long, byte[]>> row = cf.get(put.key()); + if (row == null) { if (expected != null && expected.length > 0) { return Deferred.fromResult(false); } - - cf = new Bytes.ByteMap<TreeMap<Long, byte[]>>(); - row.put(put.family(), cf); + row = new ByteMap<TreeMap<Long, byte[]>>(); + cf.put(put.key(), row); } - - // CAS can only operate on one cell, so if the put request has more than + + // CAS can only operate on one cell, so if the put request has more than // one, we ignore any but the first - TreeMap<Long, byte[]> column = cf.get(put.qualifiers()[0]); + TreeMap<Long, byte[]> column = row.get(put.qualifiers()[0]); if (column == null && (expected != null && expected.length > 0)) { return Deferred.fromResult(false); } - // if a timestamp was specified, maybe we're CASing against a specific - // cell. Otherwise we deal with the latest value - final byte[] stored = column == null ? null : - put.timestamp() != Long.MAX_VALUE ? column.get(put.timestamp()) : - column.firstEntry().getValue(); + + // HBase CAS doesn't use Timestamps for comparison + // Since OpenTSDB uses an HBase Table with single version, the final TreeMap 'column' should always + // contain a single entry, the one with higher timestamp + + final byte[] stored = column == null ? null : column.firstEntry().getValue(); + if (stored == null && (expected != null && expected.length > 0)) { return Deferred.fromResult(false); } if (stored != null && (expected == null || expected.length < 1)) { return Deferred.fromResult(false); } - if (stored != null && expected != null && + if (stored != null && expected != null && Bytes.memcmp(stored, expected) != 0) { return Deferred.fromResult(false); } - + // passed CAS! if (column == null) { column = new TreeMap<Long, byte[]>(Collections.reverseOrder()); - cf.put(put.qualifiers()[0], column); + row.put(put.qualifiers()[0], column); + } else { + long storedTs = column.firstKey(); + if(put.timestamp() >= storedTs) { + column.clear(); + } else { + return Deferred.fromResult(true); + } } - column.put(put.timestamp() != Long.MAX_VALUE ? put.timestamp() : + + column.put(put.timestamp() != Long.MAX_VALUE ? put.timestamp() : current_timestamp++, put.value()); + + assert column.size() == 1 : "MockBase is designed to store only a single version of cell since OpenTSDB table schema for HBase is designed in that manner"; return Deferred.fromResult(true); } - + } - + /** * Deletes one or more columns. If a row no longer has any valid columns, the * entire row will be removed. */ private class MockDelete implements Answer<Deferred<Object>> { - + @Override public Deferred<Object> answer(InvocationOnMock invocation) throws Throwable { final Object[] args = invocation.getArguments(); final DeleteRequest delete = (DeleteRequest)args[0]; - - Bytes.ByteMap<Bytes.ByteMap<TreeMap<Long, byte[]>>> row = - storage.get(delete.key()); - if (row == null) { - return Deferred.fromResult(null); + + if (exceptions != null) { + final Pair<RuntimeException, Boolean> ex = exceptions.get(delete.key()); + if (ex != null) { + if (ex.getValue()) { + return Deferred.fromError(ex.getKey()); + } else { + throw ex.getKey(); + } + } } - - // if no qualifiers or family, then delete the row - if ((delete.qualifiers() == null || delete.qualifiers().length < 1 || - delete.qualifiers()[0].length < 1) && (delete.family() == null || + + final ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>> map = + storage.get(delete.table()); + if (map == null) { + return Deferred.fromError(new RuntimeException( + "No such table " + Bytes.pretty(delete.table()))); + } + + // if no qualifiers or family, then delete the row from all families + if ((delete.qualifiers() == null || delete.qualifiers().length < 1 || + delete.qualifiers()[0].length < 1) && (delete.family() == null || delete.family().length < 1)) { - storage.remove(delete.key()); + for (final Entry<byte[], ByteMap<ByteMap<TreeMap<Long, byte[]>>>> cf : + map.entrySet()) { + cf.getValue().remove(delete.key()); + } return Deferred.fromResult(new Object()); } - + final byte[] family = delete.family(); if (family != null && family.length > 0) { - if (!row.containsKey(family)) { - return Deferred.fromResult(null); + if (!map.containsKey(family)) { + return Deferred.fromError(new RuntimeException( + "No such CF " + Bytes.pretty(family))); } } - - // compile a set of qualifiers to use as a filter if necessary - Bytes.ByteMap<Object> qualifiers = new Bytes.ByteMap<Object>(); - if (delete.qualifiers() != null || delete.qualifiers().length > 0) { + + // compile a set of qualifiers + ByteMap<Object> qualifiers = new ByteMap<Object>(); + if (delete.qualifiers() != null || delete.qualifiers().length > 0) { for (byte[] q : delete.qualifiers()) { qualifiers.put(q, null); } } - + + // TODO - validate the assumption that a delete with a row key and qual + // but without a family would delete the columns in ALL families + // if the request only has a column family and no qualifiers, we delete - // the entire family + // the row from the entire family if (family != null && qualifiers.isEmpty()) { - row.remove(family); - if (row.isEmpty()) { - storage.remove(delete.key()); - } + final ByteMap<ByteMap<TreeMap<Long, byte[]>>> cf = map.get(delete.family()); + // cf != null validated above + cf.remove(delete.key()); return Deferred.fromResult(new Object()); } - - List<byte[]> cf_removals = new ArrayList<byte[]>(row.entrySet().size()); - for (Map.Entry<byte[], Bytes.ByteMap<TreeMap<Long, byte[]>>> cf : - row.entrySet()) { - + + for (final Entry<byte[], ByteMap<ByteMap<TreeMap<Long, byte[]>>>> cf : + map.entrySet()) { + // column family filter - if (family != null && family.length > 0 && + if (family != null && family.length > 0 && !Bytes.equals(family, cf.getKey())) { continue; } - + + ByteMap<TreeMap<Long, byte[]>> row = cf.getValue().get(delete.key()); + if (row == null) { + continue; + } + for (byte[] qualifier : qualifiers.keySet()) { - final TreeMap<Long, byte[]> column = cf.getValue().get(qualifier); + final TreeMap<Long, byte[]> column = row.get(qualifier); if (column == null) { continue; } - + // with this flag we delete a single timestamp if (delete.deleteAtTimestampOnly()) { if (column != null) { column.remove(delete.timestamp()); if (column.isEmpty()) { - cf.getValue().remove(qualifier); + row.remove(qualifier); } } } else { - // otherwise we delete everything less than or equal to the + // otherwise we delete everything less than or equal to the // delete timestamp List<Long> column_removals = new ArrayList<Long>(column.size()); for (Map.Entry<Long, byte[]> cell : column.entrySet()) { @@ -815,29 +1394,20 @@ public Deferred<Object> answer(InvocationOnMock invocation) column.remove(ts); } if (column.isEmpty()) { - cf.getValue().remove(qualifier); + row.remove(qualifier); } } } - - if (cf.getValue().isEmpty()) { - cf_removals.add(cf.getKey()); + + if (row.isEmpty()) { + cf.getValue().remove(delete.key()); } } - - for (byte[] cf : cf_removals) { - row.remove(cf); - } - - if (row.isEmpty()) { - storage.remove(delete.key()); - } - return Deferred.fromResult(new Object()); } - + } - + /** * This is a limited implementation of the scanner object. The only fields * caputred and acted on are: @@ -851,67 +1421,89 @@ public Deferred<Object> answer(InvocationOnMock invocation) * call. The second {@code nextRows} call will always return null. Multiple * qualifiers are supported for matching. * <p> - * The KeyRegexp can be set and it will run against the hex value of the + * The KeyRegexp can be set and it will run against the hex value of the * row key. In testing it seems to work nicely even with byte patterns. */ - private class MockScanner implements + public class MockScanner implements Answer<Deferred<ArrayList<ArrayList<KeyValue>>>> { - + + private final Scanner mock_scanner; + private final byte[] table; private byte[] start = null; private byte[] stop = null; private HashSet<String> scnr_qualifiers = null; private byte[] family = null; - private String regex = null; - private boolean called; - - public MockScanner(final Scanner mock_scanner) { + private ScanFilter filter = null; + private int max_num_rows = Scanner.DEFAULT_MAX_NUM_ROWS; + private ByteMap<Iterator<Entry<byte[], ByteMap<TreeMap<Long, byte[]>>>>> + cursors; + private ByteMap<Entry<byte[], ByteMap<TreeMap<Long, byte[]>>>> cf_rows; + private byte[] last_row; + + /** + * Default ctor + * @param mock_scanner The scanner we're using + * @param table The table (confirmed to exist) + */ + public MockScanner(final Scanner mock_scanner, final byte[] table) { + this.mock_scanner = mock_scanner; + this.table = table; // capture the scanner fields when set doAnswer(new Answer<Object>() { @Override public Object answer(InvocationOnMock invocation) throws Throwable { final Object[] args = invocation.getArguments(); - regex = (String)args[0]; + filter = new KeyRegexpFilter((String)args[0], Const.ASCII_CHARSET); return null; } }).when(mock_scanner).setKeyRegexp(anyString()); - + doAnswer(new Answer<Object>() { @Override public Object answer(InvocationOnMock invocation) throws Throwable { final Object[] args = invocation.getArguments(); - regex = (String)args[0]; + filter = new KeyRegexpFilter((String)args[0], (Charset)args[1]); return null; } }).when(mock_scanner).setKeyRegexp(anyString(), (Charset)any()); - + + doAnswer(new Answer<Object>() { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + final Object[] args = invocation.getArguments(); + filter = (ScanFilter)args[0]; + return null; + } + }).when(mock_scanner).setFilter(any(ScanFilter.class)); + doAnswer(new Answer<Object>() { @Override public Object answer(InvocationOnMock invocation) throws Throwable { final Object[] args = invocation.getArguments(); start = (byte[])args[0]; return null; - } + } }).when(mock_scanner).setStartKey((byte[])any()); - + doAnswer(new Answer<Object>() { @Override public Object answer(InvocationOnMock invocation) throws Throwable { final Object[] args = invocation.getArguments(); stop = (byte[])args[0]; return null; - } + } }).when(mock_scanner).setStopKey((byte[])any()); - + doAnswer(new Answer<Object>() { @Override public Object answer(InvocationOnMock invocation) throws Throwable { final Object[] args = invocation.getArguments(); family = (byte[])args[0]; return null; - } + } }).when(mock_scanner).setFamily((byte[])any()); - + doAnswer(new Answer<Object>() { @Override public Object answer(InvocationOnMock invocation) throws Throwable { @@ -919,9 +1511,9 @@ public Object answer(InvocationOnMock invocation) throws Throwable { scnr_qualifiers = new HashSet<String>(1); scnr_qualifiers.add(bytesToString((byte[])args[0])); return null; - } + } }).when(mock_scanner).setQualifier((byte[])any()); - + doAnswer(new Answer<Object>() { @Override public Object answer(InvocationOnMock invocation) throws Throwable { @@ -932,106 +1524,344 @@ public Object answer(InvocationOnMock invocation) throws Throwable { scnr_qualifiers.add(bytesToString(qualifier)); } return null; - } + } }).when(mock_scanner).setQualifiers((byte[][])any()); + + doAnswer(new Answer<Object>() { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + final Object[] args = invocation.getArguments(); + filter = (ScanFilter)args[0]; + return null; + } + }).when(mock_scanner).setFilter(any(ScanFilter.class)); + doAnswer(new Answer<byte[]>() { + @Override + public byte[] answer(InvocationOnMock invocation) throws Throwable { + return start; + } + }).when(mock_scanner).getCurrentKey(); + when(mock_scanner.nextRows()).thenAnswer(this); + + doAnswer(new Answer<ScanFilter>() { + @Override + public ScanFilter answer(InvocationOnMock invocation) throws Throwable { + return filter; + } + }).when(mock_scanner).getFilter(); + doAnswer(new Answer<String>() { + @Override + public String answer(final InvocationOnMock ignored) throws Throwable { + return MockScanner.this.toString(); + } + }).when(mock_scanner).toString(); + } + + @Override + public String toString() { + final StringBuilder buf = new StringBuilder(); + buf.append("table=") + .append(Bytes.pretty(table)) + .append(", start=") + .append(Bytes.pretty(start)) + .append(", stop=") + .append(Bytes.pretty(stop)) + .append(", family=") + .append(Bytes.pretty(family)) + .append(", filter=") + .append(filter); + return buf.toString(); } @Override public Deferred<ArrayList<ArrayList<KeyValue>>> answer( final InvocationOnMock invocation) throws Throwable { - - // It's critical to see if this scanner has been processed before, - // otherwise the code under test will likely wind up in an infinite loop. - // If the scanner has been seen before, we return null. - if (called) { + + if (cursors == null) { + final ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>> map = + storage.get(table); + if (map == null) { + return Deferred.fromError( new RuntimeException( + "No such table " + Bytes.pretty(table))); + } + + cursors = new ByteMap<Iterator<Entry<byte[], + ByteMap<TreeMap<Long, byte[]>>>>>(); + cf_rows = new ByteMap<Entry<byte[], ByteMap<TreeMap<Long, byte[]>>>>(); + + if (family == null || family.length < 1) { + for (final Entry<byte[], ByteMap<ByteMap<TreeMap<Long, byte[]>>>> cf : map) { + final Iterator<Entry<byte[], ByteMap<TreeMap<Long, byte[]>>>> + cursor = cf.getValue().iterator(); + cursors.put(cf.getKey(), cursor); + cf_rows.put(cf.getKey(), null); + } + } else { + final ByteMap<ByteMap<TreeMap<Long, byte[]>>> cf = map.get(family); + if (cf == null) { + return Deferred.fromError(new RuntimeException( + "No such CF " + Bytes.pretty(family))); + } + final Iterator<Entry<byte[], ByteMap<TreeMap<Long, byte[]>>>> + cursor = cf.iterator(); + cursors.put(family, cursor); + cf_rows.put(family, null); + } + } + + // If we're out of rows to scan, then you HAVE to return null as the + // HBase client does. + if (!hasNext()) { return Deferred.fromResult(null); } - called = true; - + + // TODO - fuzzy filter support + // TODO - fix the regex comparator Pattern pattern = null; - if (regex != null && !regex.isEmpty()) { - try { - pattern = Pattern.compile(regex); - } catch (PatternSyntaxException e) { - e.printStackTrace(); + Charset regex_charset = null; + if (filter != null) { + KeyRegexpFilter regex_filter = null; + + if (filter instanceof KeyRegexpFilter) { + regex_filter = (KeyRegexpFilter)filter; + } else if (filter instanceof FilterList) { + for (final ScanFilter f : ((FilterList)filter).filters()) { + if (f instanceof KeyRegexpFilter) { + regex_filter = (KeyRegexpFilter)f; + } + } + } + + if (regex_filter != null) { + try { + // key regex filter uses Bytes.UTF8(<string>) + pattern = Pattern.compile(new String(regex_filter.getRegexp(), + Charset.forName("UTF-8"))); + regex_charset = regex_filter.getCharset(); + } catch (PatternSyntaxException e) { + e.printStackTrace(); + return Deferred.fromError(e); + } } } - + // return all matches - ArrayList<ArrayList<KeyValue>> results = + final ArrayList<ArrayList<KeyValue>> results = new ArrayList<ArrayList<KeyValue>>(); - for (Map.Entry<byte[], Bytes.ByteMap<Bytes.ByteMap<TreeMap<Long, byte[]>>>> row : - storage.entrySet()) { - + int rows_read = 0; + while (hasNext()) { + advance(); + // if it's before the start row, after the end row or doesn't // match the given regex, continue on to the next row - if (start != null && Bytes.memcmp(row.getKey(), start) < 0) { + if (start != null && Bytes.memcmp(last_row, start) < 0) { continue; } // asynchbase Scanner's logic: - // - start_key is inclusive, stop key is exclusive, - // - when start key is equal to the stop key, include the key in scan result, - if (stop != null && Bytes.memcmp(row.getKey(), stop) >= 0 - && Bytes.memcmp(start, stop) != 0) { + // - start_key is inclusive, stop key is exclusive + // - when start key is equal to the stop key, + // include the key in scan result + // - if stop key is empty, scan till the end + if (stop != null && stop.length > 0 && + Bytes.memcmp(last_row, stop) >= 0 && + Bytes.memcmp(start, stop) != 0) { continue; } if (pattern != null) { - final String from_bytes = new String(row.getKey(), MockBase.ASCII); + final String from_bytes = new String(last_row, regex_charset); if (!pattern.matcher(from_bytes).find()) { - continue; + if (filter instanceof FilterList) { + FilterList.Operator op = Whitebox.getInternalState(filter, "op"); + if (op == FilterList.Operator.MUST_PASS_ALL) { + continue; + } + } else { + continue; + } + } + } + + // throws AFTER we match on a row key + if (exceptions != null) { + final Pair<RuntimeException, Boolean> ex = exceptions.get(last_row); + if (ex != null) { + if (ex.getValue()) { + return Deferred.fromError(ex.getKey()); + } else { + throw ex.getKey(); + } } } - - // loop on the column families - final ArrayList<KeyValue> kvs = - new ArrayList<KeyValue>(row.getValue().size()); - for (Map.Entry<byte[], Bytes.ByteMap<TreeMap<Long, byte[]>>> cf : - row.getValue().entrySet()) { - - // column family filter - if (family != null && family.length > 0 && - !Bytes.equals(family, cf.getKey())) { + + // loop over the column family rows to see if they match + final ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(); + for (final Entry<byte[], Entry<byte[], ByteMap<TreeMap<Long, byte[]>>>> row : + cf_rows.entrySet()) { + if (row.getValue() == null || + Bytes.memcmp(last_row, row.getValue().getKey()) != 0) { continue; } - - for (Map.Entry<byte[], TreeMap<Long, byte[]>> column : - cf.getValue().entrySet()) { - + + for (final Entry<byte[], TreeMap<Long, byte[]>> column : + row.getValue().getValue().entrySet()) { // if the qualifier isn't in the set, continue - if (scnr_qualifiers != null && + if (scnr_qualifiers != null && !scnr_qualifiers.contains(bytesToString(column.getKey()))) { continue; } + + + // handle qualifier filters. + if (filter != null) { + List<QualifierFilter> qfs = Lists.newArrayList(); + if (filter instanceof FilterList) { + for (final ScanFilter f : ((FilterList) filter).filters()) { + if (f instanceof QualifierFilter) { + qfs.add((QualifierFilter) f); + } else if (f instanceof FilterList) { // nested + for (final ScanFilter nf : ((FilterList) f).filters()) { + if (nf instanceof QualifierFilter) { + qfs.add((QualifierFilter) nf); + } + } + } + } + } else if (filter instanceof QualifierFilter) { + qfs.add((QualifierFilter) filter); + } + + if (!qfs.isEmpty()) { + boolean matched = false; + for (final QualifierFilter qf : qfs) { + final FilterComparator fc = Whitebox + .getInternalState(qf, "comparator"); + if (fc instanceof BinaryPrefixComparator) { + final byte[] comparator = Whitebox + .getInternalState(fc, "value"); + if (Bytes.memcmp(comparator, column.getKey(), 0, + comparator.length) == 0) { + matched = true; + } + } else if (fc instanceof RegexStringComparator) { + // not using this yet but.... *shrug* + final Pattern p = Pattern.compile((String) Whitebox + .getInternalState(fc, "expr")); + + final String qualifier = new String(column.getKey(), + (Charset) Whitebox.getInternalState(fc, "charset")); + if (p.matcher(qualifier).matches()) { + matched = true; + } + } + } + if (!matched) { + continue; + } + } + } - KeyValue kv = mock(KeyValue.class); - when(kv.key()).thenReturn(row.getKey()); - when(kv.value()).thenReturn(column.getValue().firstEntry().getValue()); - when(kv.qualifier()).thenReturn(column.getKey()); - when(kv.timestamp()).thenReturn(column.getValue().firstKey()); - when(kv.family()).thenReturn(cf.getKey()); - when(kv.toString()).thenReturn("[k '" + bytesToString(row.getKey()) + - "' q '" + bytesToString(column.getKey()) + "' v '" + - bytesToString(column.getValue().firstEntry().getValue()) + "']"); - kvs.add(kv); + kvs.add(new KeyValue(row.getValue().getKey(), row.getKey(), + column.getKey(), column.getValue().firstKey(), + column.getValue().firstEntry().getValue())); } - } - + if (!kvs.isEmpty()) { results.add(kvs); } + rows_read++; + + if (rows_read >= max_num_rows) { + Thread.sleep(10); // this is here for time based unit tests + break; + } } - + if (results.isEmpty()) { return Deferred.fromResult(null); } return Deferred.fromResult(results); } + + /** @return Returns true if any of the CF iterators have another value */ + private boolean hasNext() { + for (final Iterator<Entry<byte[], ByteMap<TreeMap<Long, byte[]>>>> cursor : + cursors.values()) { + if (cursor.hasNext()) { + return true; + } + } + return false; + } + + /** Insanely inefficient and ugly way of advancing the cursors */ + private void advance() { + // first time to get the ceiling + if (last_row == null) { + for (final Entry<byte[], + Iterator<Entry<byte[], ByteMap<TreeMap<Long, byte[]>>>>> iterator : + cursors.entrySet()) { + final Entry<byte[], ByteMap<TreeMap<Long, byte[]>>> row = + iterator.getValue().hasNext() ? iterator.getValue().next() : null; + cf_rows.put(iterator.getKey(), row); + if (last_row == null) { + last_row = row.getKey(); + } else { + if (Bytes.memcmp(last_row, row.getKey()) < 0) { + last_row = row.getKey(); + } + } + } + return; + } + + for (final Entry<byte[], Entry<byte[], ByteMap<TreeMap<Long, byte[]>>>> cf : + cf_rows.entrySet()) { + final Entry<byte[], ByteMap<TreeMap<Long, byte[]>>> row = cf.getValue(); + if (row == null) { + continue; + } + + if (Bytes.memcmp(last_row, row.getKey()) == 0) { + if (!cursors.get(cf.getKey()).hasNext()) { + cf_rows.put(cf.getKey(), null); // EX? + } else { + cf_rows.put(cf.getKey(), cursors.get(cf.getKey()).next()); + } + } + } + + last_row = null; + for (final Entry<byte[], ByteMap<TreeMap<Long, byte[]>>> row : + cf_rows.values()) { + if (row == null) { + continue; + } + + if (last_row == null) { + last_row = row.getKey(); + } else { + if (Bytes.memcmp(last_row, row.getKey()) < 0) { + last_row = row.getKey(); + } + } + } + } + + /** @return The scanner for this mock */ + public Scanner getScanner() { + return mock_scanner; + } + + /** @return The filter for this mock */ + public ScanFilter getFilter() { + return filter; + } } - + /** * Creates or increments (possibly decrements) a Long in the hash table at the * given location. @@ -1044,32 +1874,51 @@ public Deferred<Long> answer(InvocationOnMock invocation) throws Throwable { final Object[] args = invocation.getArguments(); final AtomicIncrementRequest air = (AtomicIncrementRequest)args[0]; final long amount = air.getAmount(); - Bytes.ByteMap<Bytes.ByteMap<TreeMap<Long, byte[]>>> row = - storage.get(air.key()); - if (row == null) { - row = new Bytes.ByteMap<Bytes.ByteMap<TreeMap<Long, byte[]>>>(); - storage.put(air.key(), row); + + if (exceptions != null) { + final Pair<RuntimeException, Boolean> ex = exceptions.get(air.key()); + if (ex != null) { + if (ex.getValue()) { + return Deferred.fromError(ex.getKey()); + } else { + throw ex.getKey(); + } + } } - - Bytes.ByteMap<TreeMap<Long, byte[]>> cf = row.get(air.family()); + + final ByteMap<ByteMap<ByteMap<TreeMap<Long, byte[]>>>> map = + storage.get(air.table()); + if (map == null) { + return Deferred.fromError(new RuntimeException( + "No such table " + Bytes.pretty(air.table()))); + } + + final ByteMap<ByteMap<TreeMap<Long, byte[]>>> cf = map.get(air.family()); if (cf == null) { - cf = new Bytes.ByteMap<TreeMap<Long, byte[]>>(); - row.put(air.family(), cf); + return Deferred.fromError(new RuntimeException( + "No such CF " + Bytes.pretty(air.table()))); } - - TreeMap<Long, byte[]> column = cf.get(air.qualifier()); + + ByteMap<TreeMap<Long, byte[]>> row = cf.get(air.key()); + if (row == null) { + row = new ByteMap<TreeMap<Long, byte[]>>(); + cf.put(air.key(), row); + } + + TreeMap<Long, byte[]> column = row.get(air.qualifier()); if (column == null) { column = new TreeMap<Long, byte[]>(Collections.reverseOrder()); - cf.put(air.qualifier(), column); + row.put(air.qualifier(), column); column.put(current_timestamp++, Bytes.fromLong(amount)); return Deferred.fromResult(amount); } - + long incremented_value = Bytes.getLong(column.firstEntry().getValue()); incremented_value += amount; column.put(column.firstKey(), Bytes.fromLong(incremented_value)); return Deferred.fromResult(incremented_value); } - + } + } diff --git a/test/storage/MockDataPoints.java b/test/storage/MockDataPoints.java new file mode 100644 index 0000000000..dc52724088 --- /dev/null +++ b/test/storage/MockDataPoints.java @@ -0,0 +1,137 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.storage; + +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.when; +import static org.powermock.api.mockito.PowerMockito.mock; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.Ignore; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import com.stumbleupon.async.Deferred; + +import net.opentsdb.core.DataPoint; +import net.opentsdb.core.DataPoints; +import net.opentsdb.core.SeekableView; +import net.opentsdb.meta.Annotation; + +/** + * A class that implements a mock of the DataPoints and DataPoint interfaces + * for use in serializing data out to the user. + */ +@Ignore +public class MockDataPoints { + + private final DataPoints dps = mock(DataPoints.class); + private final DataPoint dp = mock(DataPoint.class); + private final SeekableView it = mock(SeekableView.class); + + private long timestamp = 1356998400000L; + private int interval = 300000; // in ms + private long value = 0; + private int limit = 400; // only checks timeout every 100 dps + + private final String metric = "system.cpu.user"; + private final Map<String, String> tags = new HashMap<String, String>(1); + private final List<String> agg_tags = new ArrayList<String>(1); + private final List<String> tsuids = new ArrayList<String>(2); + private final List<Annotation> annotations = new ArrayList<Annotation>(1); + + /** + * Default Ctor that stores some values in the tags and tsuid as well as a + * single annotation. + */ + public MockDataPoints() { + tags.put("dc", "lga"); + agg_tags.add("host"); + tsuids.add("000001000001000001"); + tsuids.add("000001000001000002"); + + final Annotation note = new Annotation(); + note.setTSUID("000001000001000001"); + note.setStartTime(1356998401000L); + note.setDescription("Just a simple note"); + annotations.add(note); + } + + /** + * Retrieves the DataPoints object and sets up the mock calls + * @return A DataPoints object for iteration + */ + public DataPoints getMock() { + when(dps.metricName()).thenReturn(metric); + when(dps.metricNameAsync()).thenReturn(Deferred.fromResult(metric)); + when(dps.getTags()).thenReturn(tags); + when(dps.getTagsAsync()).thenReturn(Deferred.fromResult(tags)); + when(dps.getAggregatedTags()).thenReturn(agg_tags); + when(dps.getAggregatedTagsAsync()).thenReturn(Deferred.fromResult(agg_tags)); + when(dps.getTSUIDs()).thenReturn(tsuids); + when(dps.getAnnotations()).thenReturn(annotations); + when(dps.size()).thenReturn(limit); + when(dps.aggregatedSize()).thenReturn(limit * 2); + when(dps.iterator()).thenReturn(it); + when(dps.getQueryIndex()).thenReturn(0); + + // iterator mocking + when(it.hasNext()).thenAnswer(new Answer<Boolean>() { + @Override + public Boolean answer(final InvocationOnMock args) throws Throwable { + if (value > limit) { + return false; + } + return true; + } + }); + when(it.next()).thenAnswer(new Answer<DataPoint>() { + @Override + public DataPoint answer(final InvocationOnMock args) throws Throwable { + value++; + timestamp += interval; + return dp; + } + }); + doThrow(new RuntimeException("OpenTSDB doesn't support remove")) + .when(it).remove(); + + // data point mocking + when(dp.timestamp()).thenAnswer(new Answer<Long>() { + @Override + public Long answer(final InvocationOnMock args) throws Throwable { + return timestamp; + } + }); + when(dp.isInteger()).thenReturn(true); + when(dp.longValue()).thenAnswer(new Answer<Long>() { + @Override + public Long answer(final InvocationOnMock args) throws Throwable { + return value; + } + }); + return dps; + } + + /** + * Returns the DataPoint object so that the test can override the mocks. + * @return A DataPoint in the DataPoints class + */ + public DataPoint getMockDP() { + return dp; + } +} diff --git a/test/tools/TestCliUtils.java b/test/tools/TestCliUtils.java new file mode 100644 index 0000000000..7745494961 --- /dev/null +++ b/test/tools/TestCliUtils.java @@ -0,0 +1,201 @@ +package net.opentsdb.tools; + +import static org.powermock.api.mockito.PowerMockito.mock; +import static org.powermock.api.mockito.PowerMockito.when; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; + +import java.util.ArrayList; +import java.util.List; + +import net.opentsdb.core.Const; +import net.opentsdb.core.RowKey; +import net.opentsdb.core.TSDB; + +import org.hbase.async.Bytes; +import org.hbase.async.GetRequest; +import org.hbase.async.HBaseClient; +import org.hbase.async.KeyValue; +import org.hbase.async.Scanner; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +import com.stumbleupon.async.Deferred; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ TSDB.class, HBaseClient.class, Scanner.class, Const.class }) +public class TestCliUtils { + + private TSDB tsdb = null; + private HBaseClient client = null; + private List<byte[]> start_keys; + private List<byte[]> stop_keys; + + @Before + public void before() throws Exception { + tsdb = mock(TSDB.class); + client = mock(HBaseClient.class); + when(tsdb.getClient()).thenReturn(client); + when(tsdb.uidTable()).thenReturn("tsdb-uid".getBytes()); + + } + + @Test + public void getDataTableScanners1Thread() throws Exception { + setupGetDataTableScanners(256); + + final List<Scanner> scanners = CliUtils.getDataTableScanners(tsdb, 1); + assertEquals(1, scanners.size()); + assertArrayEquals(HBaseClient.EMPTY_ARRAY, start_keys.get(0)); + assertArrayEquals(HBaseClient.EMPTY_ARRAY, stop_keys.get(0)); + } + + @Test + public void getDataTableScannersMultiThreaded() throws Exception { + setupGetDataTableScanners(256); + final List<Scanner> scanners = CliUtils.getDataTableScanners(tsdb, 15); + assertEquals(15, scanners.size()); + byte[] key = new byte[3]; + int last_value = 0; + for (int i = 0; i < 15; i++) { + if (i == 0) { + assertArrayEquals(HBaseClient.EMPTY_ARRAY, start_keys.get(i)); + } else { + assertArrayEquals(key, start_keys.get(i)); + } + last_value += 18; + if (i == 14) { + assertArrayEquals(HBaseClient.EMPTY_ARRAY, stop_keys.get(i)); + } else { + System.arraycopy(Bytes.fromInt(last_value), 1, key, 0, 3); + assertArrayEquals(key, stop_keys.get(i)); + } + } + } + + @Test + public void getDataTableScannersRandom1Thread() throws Exception { + setupGetDataTableScanners(0); + + final List<Scanner> scanners = CliUtils.getDataTableScanners(tsdb, 1); + assertEquals(1, scanners.size()); + assertArrayEquals(HBaseClient.EMPTY_ARRAY, start_keys.get(0)); + assertArrayEquals(HBaseClient.EMPTY_ARRAY, stop_keys.get(0)); + } + + @Test + public void getDataTableScannersRandomMultiThreaded() throws Exception { + setupGetDataTableScanners(0); + final List<Scanner> scanners = CliUtils.getDataTableScanners(tsdb, 15); + assertEquals(15, scanners.size()); + byte[] key = new byte[3]; + int last_value = 0; + for (int i = 0; i < 15; i++) { + if (i == 0) { + assertArrayEquals(HBaseClient.EMPTY_ARRAY, start_keys.get(i)); + } else { + assertArrayEquals(key, start_keys.get(i)); + } + last_value += 1118481; + if (i == 14) { + assertArrayEquals(HBaseClient.EMPTY_ARRAY, stop_keys.get(i)); + } else { + System.arraycopy(Bytes.fromInt(last_value), 1, key, 0, 3); + assertArrayEquals(key, stop_keys.get(i)); + } + } + } + + @Test + public void getDataTableScannersSalted() throws Exception { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(20); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + + setupGetDataTableScanners(256); + final List<Scanner> scanners = CliUtils.getDataTableScanners(tsdb, 15); + assertEquals(20, scanners.size()); + byte[] key = new byte[1]; + for (int i = 0; i < 20; i++) { + if (i == 0) { + assertArrayEquals(HBaseClient.EMPTY_ARRAY, start_keys.get(i)); + } else { + assertArrayEquals(key, start_keys.get(i)); + } + if (i == 19) { + assertArrayEquals(HBaseClient.EMPTY_ARRAY, stop_keys.get(i)); + } else { + key = RowKey.getSaltBytes(i + 1); + assertArrayEquals(key, stop_keys.get(i)); + } + } + } + + @Test + public void getDataTableScannersSaltedRandom() throws Exception { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(20); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + + setupGetDataTableScanners(0); + final List<Scanner> scanners = CliUtils.getDataTableScanners(tsdb, 15); + assertEquals(20, scanners.size()); + byte[] key = new byte[1]; + for (int i = 0; i < 20; i++) { + if (i == 0) { + assertArrayEquals(HBaseClient.EMPTY_ARRAY, start_keys.get(i)); + } else { + assertArrayEquals(key, start_keys.get(i)); + } + if (i == 19) { + assertArrayEquals(HBaseClient.EMPTY_ARRAY, stop_keys.get(i)); + } else { + key = RowKey.getSaltBytes(i + 1); + assertArrayEquals(key, stop_keys.get(i)); + } + } + } + + private void setupGetDataTableScanners(final long max) { + final KeyValue kv = new KeyValue(new byte[] {}, + TSDB.FAMILY(), "metrics".getBytes(), Bytes.fromLong(max)); + final ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); + kvs.add(kv); + when(client.get(any(GetRequest.class))) + .thenReturn(Deferred.<ArrayList<KeyValue>>fromResult(kvs)); + + start_keys = new ArrayList<byte[]>(); + stop_keys = new ArrayList<byte[]>(); + + final Scanner scanner = mock(Scanner.class); + when(client.newScanner(any(byte[].class))).thenReturn(scanner); + + PowerMockito.doAnswer(new Answer<Void>() { + @Override + public Void answer(final InvocationOnMock invocation) throws Throwable { + start_keys.add((byte[])invocation.getArguments()[0]); + return null; + } + }).when(scanner).setStartKey(any(byte[].class)); + + PowerMockito.doAnswer(new Answer<Void>() { + @Override + public Void answer(final InvocationOnMock invocation) throws Throwable { + stop_keys.add((byte[])invocation.getArguments()[0]); + return null; + } + }).when(scanner).setStopKey(any(byte[].class)); + } +} diff --git a/test/tools/TestConfigArgP.java b/test/tools/TestConfigArgP.java new file mode 100644 index 0000000000..07951c8103 --- /dev/null +++ b/test/tools/TestConfigArgP.java @@ -0,0 +1,628 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2014 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.tools; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.StringReader; +import java.lang.Thread.UncaughtExceptionHandler; +import java.lang.management.ManagementFactory; +import java.net.InetSocketAddress; +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; + +import net.opentsdb.tools.ConfigArgP.ConfigurationItem; +import net.opentsdb.utils.Config; + +import org.jboss.netty.bootstrap.ServerBootstrap; +import org.jboss.netty.buffer.ChannelBuffers; +import org.jboss.netty.channel.ChannelFutureListener; +import org.jboss.netty.channel.ChannelHandlerContext; +import org.jboss.netty.channel.ChannelPipeline; +import org.jboss.netty.channel.ChannelPipelineFactory; +import org.jboss.netty.channel.Channels; +import org.jboss.netty.channel.ExceptionEvent; +import org.jboss.netty.channel.MessageEvent; +import org.jboss.netty.channel.SimpleChannelUpstreamHandler; +import org.jboss.netty.channel.socket.ServerSocketChannel; +import org.jboss.netty.channel.socket.oio.OioServerSocketChannelFactory; +import org.jboss.netty.handler.codec.http.DefaultHttpResponse; +import org.jboss.netty.handler.codec.http.HttpChunkAggregator; +import org.jboss.netty.handler.codec.http.HttpHeaders; +import org.jboss.netty.handler.codec.http.HttpHeaders.Names; +import org.jboss.netty.handler.codec.http.HttpRequest; +import org.jboss.netty.handler.codec.http.HttpRequestDecoder; +import org.jboss.netty.handler.codec.http.HttpResponse; +import org.jboss.netty.handler.codec.http.HttpResponseEncoder; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.jboss.netty.handler.codec.http.HttpVersion; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * <p>Title: TestConfigArgP</p> + * <p>Description: Test cases for the fat-jar launcher configuration manager</p> + */ +public class TestConfigArgP { + /** The number of cores available to this JVM */ + static final int CORES = ManagementFactory.getOperatingSystemMXBean().getAvailableProcessors(); + /** Platform EOL */ + static final String EOL = System.getProperty("line.separator", "\n"); + /** The quickie web server */ + static QuickieWebServer webServer = null; + /** The port the web server is listening on */ + static int port = -1; + /** true/false string values */ + static final Set<String> trueFalseValues = Collections.unmodifiableSet(new HashSet<String>(Arrays.asList("true", "false"))); + + static final boolean runningFatJar; + + static { + boolean tmp = false; + InputStream is = null; + try { + is = TestConfigArgP.class.getClassLoader().getResourceAsStream("opentsdb.conf.json"); + BufferedReader bin = new BufferedReader(new InputStreamReader(is)); + StringBuilder b = new StringBuilder(); + String line = null; + while((line = bin.readLine())!=null) { + b.append(line); + } + JSONObject jo = new JSONObject(b.toString()); + tmp = true; + } catch (Exception x) { + tmp = false; + } finally { + if(is!=null) try { is.close(); } catch (Exception x) {/* No Op */} + } + runningFatJar = tmp; + } + + /** + * Loads a fresh new JSONObject with the config + * from the classpath loaded <code>opentsdb.conf.json</code>. + * @return A loaded JSONObject + */ + static JSONObject newTSDBConfig() { + InputStream is = null; + try { + is = TestConfigArgP.class.getClassLoader().getResourceAsStream("opentsdb.conf.json"); + BufferedReader bin = new BufferedReader(new InputStreamReader(is)); + StringBuilder b = new StringBuilder(); + String line = null; + while((line = bin.readLine())!=null) { + b.append(line); + } + return new JSONObject(b.toString()); + } catch (Exception ex) { + throw new RuntimeException("Failed to load opentsdb.conf.json from the classpath", ex); + } finally { + if(is!=null) try { is.close(); } catch (Exception x) {/* No Op */} + } + } + + + + /** + * Creates a new QuickieResponder that will server the passed content + * @param uri The uri to register the responder for + * @param content The content to serve + * @param contentType The optional content type + * @return the URL to get the content with + */ + public static String newResponderForContent(final String uri, final String content, final String contentType) { + final QuickieResponder qr = new QuickieResponder(){ + @Override + public void writeResponse(final HttpResponse response) { + final byte[] bytes = content.getBytes(Charset.defaultCharset()); + HttpHeaders.addHeader(response, Names.CONTENT_TYPE, contentType==null ? "text/plain" : contentType); + HttpHeaders.addHeader(response, Names.CONTENT_LENGTH, bytes.length); + response.setContent(ChannelBuffers.wrappedBuffer(bytes)); + } + }; + webServer.addResponder(uri, qr); + InetSocketAddress isa = webServer.serverChannel.getLocalAddress(); + return String.format("http://%s:%s%s", isa.getAddress().getHostAddress(), isa.getPort(), uri); + } + + + + + + /** + * Starts the test web server + */ + @BeforeClass + public static void startHttpServer() { + webServer = new QuickieWebServer(); + port = webServer.getPort(); + org.junit.Assume.assumeTrue(runningFatJar); + } + + /** + * Stops all the running servers + */ + @AfterClass + public static void stopHttpServer() { + if(webServer!=null) { + try { + webServer.stop(); + log("Stopped OIO HTTP Server on port [" + port + "]"); + } catch (Exception x) { /* No Op */ } + webServer = null; + } + } + + + + /** + * System out logger + * @param format The message format + * @param args The message arg tokens + */ + public static void log(String format, Object...args) { + System.out.println(String.format(format, args)); + } + + /** + * Validates that a the config value for <code>tsd.network.worker_threads</code> + * for which the default value is calced by a js scriptlet, + * is two times the number of cores. + * @throws Exception on any error + */ + @Test + public void testWorkerThreadDefault() throws Exception { + ConfigArgP cap = new ConfigArgP(); + Config config = cap.getConfig(); + assertEquals("The value for key [tsd.network.worker_threads]", (CORES * 2), config.getInt("tsd.network.worker_threads")); + // now test an override + cap = new ConfigArgP("--worker-threads", "7"); + config = cap.getConfig(); + assertEquals("The overriden value for key [tsd.network.worker_threads]", 7, config.getInt("tsd.network.worker_threads")); + // now test an override with an "=" + cap = new ConfigArgP("--worker-threads=7"); + config = cap.getConfig(); + assertEquals("The overriden value for key [tsd.network.worker_threads]", 7, config.getInt("tsd.network.worker_threads")); + + } + + /** + * Validates that a ConfigArgP with no arguments creates a Config with the expected defaults + * @throws Exception on any error + */ + @Test + public void testAllDefaults() throws Exception { + ConfigArgP cap = new ConfigArgP(); + Config config = cap.getConfig(); + int validatedItems = 0; + JSONArray configItems = newTSDBConfig().getJSONArray("config-items"); + for(int i = 0; i < configItems.length(); i++) { + JSONObject configItem = configItems.getJSONObject(i); + if(!configItem.has("defaultValue")) continue; + String key = configItem.getString("key"); + String expectedValue = ConfigArgP + // Converts any javascript evals or system property token decodes + .processConfigValue(configItem.getString("defaultValue")); + String value = config.getString(key); + assertEquals("The value for key [" + key + "]", expectedValue, value); + validatedItems++; + } + log("Validated %s Config Items", validatedItems); + } + + /** + * Validates that all {@link ConfigMetaType#BOOL} type config definitions have defaults, and vice-versa. + * This is necessary to allow a command line specifier with no value (e.g. <code>--auto-metric</code>) + * @throws Exception on any error + */ + @Test + public void testAllBoolMetaTypesHaveDefault() throws Exception { + JSONArray configItems = newTSDBConfig().getJSONArray("config-items"); + for(int i = 0; i < configItems.length(); i++) { + JSONObject configItem = configItems.getJSONObject(i); + final String name = configItem.getString("key"); + String meta = configItem.has("meta") ? configItem.getString("meta") : null; + if(!"BOOL".equals(meta)) continue; + String defaultValue = configItem.has("defaultValue") ? configItem.getString("defaultValue") : null; + assertNotNull("Config Item [" + name + "] of type BOOL has null default value", defaultValue); + assertTrue("Config Item [" + name + "] of type BOOL has invalid default value", trueFalseValues.contains(defaultValue)); + } + for(int i = 0; i < configItems.length(); i++) { + JSONObject configItem = configItems.getJSONObject(i); + final String name = configItem.getString("key"); + String defaultValue = configItem.has("defaultValue") ? configItem.getString("defaultValue") : null; + if(defaultValue==null || !trueFalseValues.contains(defaultValue)) continue; + String meta = configItem.has("meta") ? configItem.getString("meta") : null; + assertEquals("Config Item [" + name + "] with defaultValue of true/false meta", "BOOL", meta); + } + + } + + /** + * Validates that an HTTP URL defined config overrides the default values. + * @throws Exception thrown on any error + */ + @SuppressWarnings("unchecked") + @Test + public void httpExternalConfigTest() throws Exception { + try { + final String url = newResponderForContent("/content", configToContent( + new ConfigArgP().getConfig(), + Collections.singletonMap("tsd.network.worker_threads", "7")), + "plain/text"); + ConfigArgP cap = new ConfigArgP("--config", url); // include overwrites config + Config config = cap.getConfig(); + assertEquals("The HTTP overriden value for key [tsd.network.worker_threads]", 7, config.getInt("tsd.network.worker_threads")); + } finally { + webServer.removeResponder("/content"); + } + } + + /** + * Validates that an HTTP URL defined include successfully overrides + * the default values. + * @throws Exception thrown on any error + */ + @SuppressWarnings("unchecked") + @Test + public void httpIncludeOverrideTest() throws Exception { + try { + final String url = newResponderForContent("/content", configToContent( + new ConfigArgP().getConfig(), + Collections.singletonMap("tsd.network.worker_threads", "7")), + "plain/text"); + ConfigArgP cap = new ConfigArgP("--include", url); // include overwrites config + Config config = cap.getConfig(); + assertEquals("The HTTP overriden value for key [tsd.network.worker_threads]", 7, config.getInt("tsd.network.worker_threads")); + } finally { + webServer.removeResponder("/content"); + } + } + + + /** + * Validates that an HTTP URL defined command line config successfully loads from a URL + * @throws Exception thrown on any error + */ + @Test + public void httpCommandLineURLConfigTest() throws Exception { + try { + final Properties p = contentToProps(ALL_NON_DEFAULTS); + final String url = newResponderForContent("/content", ALL_NON_DEFAULTS, "plain/text"); + ConfigArgP cap = new ConfigArgP("--config", url); + Config config = cap.getConfig(); + assertEquals("The HTTP overriden value for key [tsd.network.worker_threads]", (CORES * 3 + 1), config.getInt("tsd.network.worker_threads")); + for(String key: p.stringPropertyNames()) { + String value = p.getProperty(key); + String cvalue = config.getString(key); + assertEquals("The value for config item [" + key + "]", value, cvalue); + } + } finally { + webServer.removeResponder("/content"); + } + } + + /** + * Validates that an HTTP URL defined command line config is overriden by an include config + * @throws Exception thrown on any error + */ + @Test + public void httpCommandLineOverridenByIncludeTest() throws Exception { + try { + final Properties p = contentToProps(ALL_DEFAULTS); + p.remove("tsd.network.worker_threads"); + final String url = newResponderForContent("/content", ALL_DEFAULTS, "plain/text"); + final String url2 = newResponderForContent("/content2", "tsd.network.worker_threads=5", "plain/text"); + ConfigArgP cap = new ConfigArgP("--config", url, "--include", url2); + Config config = cap.getConfig(); + + assertEquals("The HTTP overriden value for key [tsd.network.worker_threads]", 5, config.getInt("tsd.network.worker_threads")); + // this is a bit of a hack (and redundant ?), but we're trying to test the rest of the values. + for(String key: p.stringPropertyNames()) { + ConfigurationItem di = cap.getDefaultItem(key); + String value = di.isBool() ? // For a bool, presence means true + cap.isClArg(di.getClOption()) ? "true" : di.getDefaultValue() + : p.getProperty(key); + String cvalue = config.getString(key); + assertEquals("The value for config item [" + key + "]", value, cvalue); + } + } finally { + webServer.removeResponder("/content"); + } + } + + /** + * Validates that BOOL type items with a default value of false + * eval as true if configured on the command line + * @throws Exception thrown on any error + */ + @Test + public void validateCLEnablesFalseBools() throws Exception { + // find all the applicable items, which are bools with a default value of false + Set<String> keysToEnable = new HashSet<String>(); + Set<String> clsToEnable = new HashSet<String>(); + JSONArray configItems = newTSDBConfig().getJSONArray("config-items"); + for(int i = 0; i < configItems.length(); i++) { + JSONObject configItem = configItems.getJSONObject(i); + if(!configItem.has("meta")) continue; + if("BOOL".equals(configItem.getString("meta")) && "false".equals(configItem.getString("defaultValue")) ) { + keysToEnable.add(configItem.getString("key")); + clsToEnable.add(configItem.getString("cl-option")); + } + } + ConfigArgP cap = new ConfigArgP(clsToEnable.toArray(new String[clsToEnable.size()])); + Config cfg = cap.getConfig(); + for(String key: keysToEnable) { + assertEquals("The bool value of [" + key + "]", "true", cfg.getString(key)); + } + + } + + + /** + * Converts the passed stringy to a properties instance + * @param content The content to format + * @return the properties instance + */ + protected Properties contentToProps(final CharSequence content) { + Properties p = new Properties(); + try { + p.load(new StringReader(content.toString())); + Properties converted = new Properties(); + for(String key: p.stringPropertyNames()) { + converted.put(key, ConfigArgP.processConfigValue(p.getProperty(key))); + } + return converted; + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + /** + * Converts the passed config to a readable properties file, applying the passed overrides + * @param config The config to write + * @param overrides The overrides to apply + * @return The rendered content + */ + protected String configToContent(final Config config, Map<String, String>...overrides) { + Properties p = new Properties(); + for(Map.Entry<String, String> entry: config.getMap().entrySet()) { + String vl = entry.getValue(); + if(vl==null || vl.trim().isEmpty()) continue; + p.put(entry.getKey(), vl); + } + for(Map<String, String> map: overrides) { + p.putAll(map); + } + StringBuilder b = new StringBuilder(); + for(String key: p.stringPropertyNames()) { + b.append(key).append("=") + .append(p.getProperty(key)) + .append(EOL); + } + return b.toString(); + } + + + static interface QuickieResponder { + public void writeResponse(final HttpResponse response); + } + + static class QuickieWebServer extends SimpleChannelUpstreamHandler implements ChannelPipelineFactory { + final OioServerSocketChannelFactory scf; + final ServerBootstrap bootstrap; + final ServerSocketChannel serverChannel; + final int port; + /** Thread serial for the http server handler threads */ + final AtomicInteger serial = new AtomicInteger(0); + /** The http server thread factory */ + final ThreadFactory threadFactory = new ThreadFactory() { + @Override + public Thread newThread(final Runnable r) { + Thread t = new Thread(r, "HttpServerHandlerThread#" + serial.incrementAndGet()); + t.setDaemon(true); + t.setUncaughtExceptionHandler(new UncaughtExceptionHandler() { + @Override + public void uncaughtException(Thread t, Throwable e) { + System.err.println("Uncaught exception on [" + t + "]"); + e.printStackTrace(System.err); + } + }); + return t; + } + }; + + final ConcurrentHashMap<String, QuickieResponder> responders = new ConcurrentHashMap<String, QuickieResponder>(); + static final DefaultHttpResponse response404 = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.NOT_FOUND); + public QuickieWebServer() { + scf = new OioServerSocketChannelFactory(Executors.newCachedThreadPool(threadFactory), Executors.newCachedThreadPool(threadFactory)); + bootstrap = new ServerBootstrap(scf); + bootstrap.setOption("child.tcpNoDelay", true); + bootstrap.setPipelineFactory(this); + serverChannel = (ServerSocketChannel) bootstrap.bind(new InetSocketAddress("127.0.0.1", 0)); + port = serverChannel.getLocalAddress().getPort(); + log("Started OIO HTTP Server on port [" + port + "]"); + } + + public int getPort() { + return port; + } + + public void addResponder(final String uri, final QuickieResponder responder) { + if(responders.putIfAbsent(uri, responder)!=null) throw new RuntimeException("Handler for URI [" + uri + "] exists"); + } + + public void removeResponder(final String uri) { + responders.remove(uri); + } + + public void stop() { + scf.releaseExternalResources(); + } + /** + * {@inheritDoc} + * @see org.jboss.netty.channel.SimpleChannelUpstreamHandler#messageReceived(org.jboss.netty.channel.ChannelHandlerContext, org.jboss.netty.channel.MessageEvent) + */ + @Override + public void messageReceived(final ChannelHandlerContext ctx, final MessageEvent e) { + Object msg = e.getMessage(); + if (msg instanceof HttpRequest) { + HttpRequest req = (HttpRequest)msg; + QuickieResponder responder = responders.get(req.getUri()); + if(responder==null) { + ctx.getChannel().write(response404).addListener(ChannelFutureListener.CLOSE); + } else { + DefaultHttpResponse response = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK); + responder.writeResponse(response); + ctx.getChannel().write(response).addListener(ChannelFutureListener.CLOSE); + } + } + } + + /** + * {@inheritDoc} + * @see org.jboss.netty.channel.ChannelPipelineFactory#getPipeline() + */ + @Override + public ChannelPipeline getPipeline() throws Exception { + ChannelPipeline pipeline = Channels.pipeline(); + pipeline.addLast("decoder", new HttpRequestDecoder()); + pipeline.addLast("aggregator", new HttpChunkAggregator(1048576)); + pipeline.addLast("encoder", new HttpResponseEncoder()); + pipeline.addLast("handler", this); + return pipeline; + } + + /** + * {@inheritDoc} + * @see org.jboss.netty.channel.SimpleChannelUpstreamHandler#exceptionCaught(org.jboss.netty.channel.ChannelHandlerContext, org.jboss.netty.channel.ExceptionEvent) + */ + @Override + public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) { + e.getCause().printStackTrace(); + e.getChannel().close(); + } + } + + private static String ALL_DEFAULTS = "tsd.core.auto_create_tagvs=true\n" + + "tsd.http.request.enable_chunked=false\n" + + "tsd.network.tcp_no_delay=true\n" + + "tsd.ui.noexport=false\n" + + "tsd.core.preload_uid_cache.max_entries=300000\n" + + "tsd.http.staticroot=${java.io.tmpdir}/.tsdb/static-content\n" + + "tsd.storage.hbase.zk_basedir=/hbase\n" + + "tsd.storage.flush_interval=1000\n" + + "tsd.core.meta.enable_tsuid_tracking=false\n" + + "tsd.core.auto_create_metrics=false\n" + + "tsd.http.request.max_chunk=4096\n" + + "tsd.search.enable=false\n" + + "tsd.network.backlog=3072\n" + + "tsd.logback.rollpattern=_%d{yyyy-MM-dd}.%i\n" + + "tsd.http.request.cors_headers=Authorization, Content-Type, Accept, Origin, User-Agent, DNT, Cache-Control, X-Mx-ReqToken, Keep-Alive, X-Requested-With, If-Modified-Since\n" + + "tsd.storage.hbase.data_table=tsdb\n" + + "tsd.network.keep_alive=true\n" + + "tsd.core.timezone=${user.timezone}\n" + + "tsd.network.port=4242\n" + + "tsd.core.auto_create_tagks=true\n" + + "tsd.mode=rw\n" + + "tsd.network.reuse_address=true\n" + + "tsd.http.cachedir=${java.io.tmpdir}/.tsdb/http-cache/\n" + + "tsd.core.meta.enable_realtime_ts=false\n" + + "tsd.rtpublisher.enable=false\n" + + "tsd.stats.canonical=false\n" + + "tsd.process.pid.ignore.existing=false\n" + + "tsd.core.meta.enable_tsuid_incrementing=false\n" + + "tsd.core.socket.timeout=0\n" + + "tsd.core.tree.enable_processing=false\n" + + "tsd.storage.hbase.uid_table=tsdb-uid\n" + + "tsd.storage.hbase.tree_table=tsdb-tree\n" + + "tsd.process.pid.file=${java.io.tmpdir}/.tsdb/opentsdb.pid\n" + + "tsd.core.preload_uid_cache=false\n" + + "tsd.network.async_io=true\n" + + "tsd.storage.fix_duplicates=false\n" + + "tsd.network.bind=0.0.0.0\n" + + "tsd.storage.hbase.zk_quorum=localhost\n" + + "tsd.storage.enable_compaction=true\n" + + "tsd.no_diediedie=false\n" + + "tsd.network.worker_threads=$[new Integer(cores * 2)]\n" + + "tsd.http.show_stack_trace=true\n" + + "tsd.logback.console=false\n" + + "tsd.core.meta.enable_realtime_uid=false\n" + + "tsd.storage.hbase.meta_table=tsdb-meta\n"; + + private static String ALL_NON_DEFAULTS = "tsd.core.auto_create_tagvs=true\n" + + "tsd.http.request.enable_chunked=true\n" + + "tsd.network.tcp_no_delay=false\n" + + "tsd.ui.noexport=true\n" + + "tsd.core.preload_uid_cache.max_entries=3\n" + + "tsd.http.staticroot=${user.home}/.tsdb/static-content\n" + + "tsd.storage.hbase.zk_basedir=/hbasex\n" + + "tsd.storage.flush_interval=10\n" + + "tsd.core.meta.enable_tsuid_tracking=true\n" + + "tsd.core.auto_create_metrics=true\n" + + "tsd.http.request.max_chunk=9203\n" + + "tsd.search.enable=true\n" + + "tsd.network.backlog=1\n" + + "tsd.logback.rollpattern=_%d{yyyy}.%i\n" + + "tsd.http.request.cors_headers=Authorization\n" + + "tsd.storage.hbase.data_table=tsdbx\n" + + "tsd.network.keep_alive=false\n" + + "tsd.core.timezone=Pacific/Kiritimati\n" + + "tsd.network.port=7272\n" + + "tsd.core.auto_create_tagks=false\n" + + "tsd.mode=ro\n" + + "tsd.network.reuse_address=false\n" + + "tsd.http.cachedir=${user.home}/.tsdb/http-cache/\n" + + "tsd.core.meta.enable_realtime_ts=true\n" + + "tsd.rtpublisher.enable=true\n" + + "tsd.stats.canonical=true\n" + + "tsd.process.pid.ignore.existing=true\n" + + "tsd.core.meta.enable_tsuid_incrementing=true\n" + + "tsd.core.socket.timeout=9024\n" + + "tsd.core.tree.enable_processing=true\n" + + "tsd.storage.hbase.uid_table=tsdb-uidx\n" + + "tsd.storage.hbase.tree_table=tsdb-treex\n" + + "tsd.process.pid.file=${user.home}/.tsdb/opentsdb.pid\n" + + "tsd.core.preload_uid_cache=true\n" + + "tsd.network.async_io=false\n" + + "tsd.storage.fix_duplicates=true\n" + + "tsd.network.bind=127.0.0.1\n" + + "tsd.storage.hbase.zk_quorum=127.0.0.1\n" + + "tsd.storage.enable_compaction=false\n" + + "tsd.no_diediedie=true\n" + + "tsd.network.worker_threads=$[new Integer(cores * 3 + 1)]\n" + + "tsd.http.show_stack_trace=false\n" + + "tsd.logback.console=true\n" + + "tsd.core.meta.enable_realtime_uid=true\n" + + "tsd.storage.hbase.meta_table=tsdb-metax\n"; + +} + + diff --git a/test/tools/TestDumpSeries.java b/test/tools/TestDumpSeries.java index d6366378fa..a589b34068 100644 --- a/test/tools/TestDumpSeries.java +++ b/test/tools/TestDumpSeries.java @@ -14,7 +14,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.when; import static org.powermock.api.mockito.PowerMockito.mock; @@ -24,6 +23,7 @@ import java.lang.reflect.Method; import java.util.HashMap; +import net.opentsdb.core.AppendDataPoints; import net.opentsdb.core.TSDB; import net.opentsdb.meta.Annotation; import net.opentsdb.storage.MockBase; @@ -31,8 +31,8 @@ import net.opentsdb.uid.UniqueId; import net.opentsdb.utils.Config; -import org.apache.zookeeper.proto.DeleteRequest; import org.hbase.async.Bytes; +import org.hbase.async.DeleteRequest; import org.hbase.async.GetRequest; import org.hbase.async.HBaseClient; import org.hbase.async.KeyValue; @@ -42,7 +42,6 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; @@ -85,10 +84,8 @@ public class TestDumpSeries { @Before public void before() throws Exception { - PowerMockito.whenNew(HBaseClient.class) - .withArguments(anyString(), anyString()).thenReturn(client); config = new Config(false); - tsdb = new TSDB(config); + tsdb = new TSDB(client, config); storage = new MockBase(tsdb, client, true, true, true, true); storage.setFamily("t".getBytes(MockBase.ASCII())); @@ -316,7 +313,18 @@ public void dumpImportCompacted() throws Exception { assertEquals("sys.cpu.user 1356998400004 6 host=web01", log_lines[1]); assertEquals("sys.cpu.user 1356998400008 5 host=web01", log_lines[2]); } - + + @Test + public void dumpImportAppendDataPoints() throws Exception { + writeAppendDataPoints(); + doDump.invoke(null, tsdb, client, "tsdb".getBytes(MockBase.ASCII()), false, + true, new String[] { "1356998400", "1357002000", "sum", "sys.cpu.user" }); + final String[] log_lines = buffer.toString("ISO-8859-1").split("\n"); + assertNotNull(log_lines); + assertEquals("sys.cpu.user 1356998402 42 host=web01", log_lines[0]); + assertEquals("sys.cpu.user 1356998404 6 host=web01", log_lines[1]); + } + @Test public void dumpRawCompactedAndDelete() throws Exception { writeCompactedData(); @@ -401,4 +409,11 @@ private void writeCompactedData() throws Exception { // kvs.add(makekv(qual12, MockBase.concatByteArrays(val1, val2, ZERO))); } + + private void writeAppendDataPoints() throws Exception { + storage.addColumn(MockBase.stringToBytes("00000150E22700000001000001"), + "t".getBytes(MockBase.ASCII()), + AppendDataPoints.APPEND_COLUMN_QUALIFIER, + new byte[] { 0, 0x20, 42, 0, 0x40, 6 }); + } } diff --git a/test/tools/TestFsck.java b/test/tools/TestFsck.java index 94aa3b64f1..30ba044509 100644 --- a/test/tools/TestFsck.java +++ b/test/tools/TestFsck.java @@ -16,6 +16,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyInt; import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.when; import static org.powermock.api.mockito.PowerMockito.mock; @@ -26,23 +27,24 @@ import java.util.List; import net.opentsdb.core.Query; -import net.opentsdb.core.RowKey; import net.opentsdb.core.TSDB; import net.opentsdb.core.Tags; +import net.opentsdb.core.BaseTsdbTest.FakeTaskTimer; import net.opentsdb.meta.Annotation; import net.opentsdb.storage.MockBase; import net.opentsdb.uid.NoSuchUniqueId; import net.opentsdb.uid.NoSuchUniqueName; import net.opentsdb.uid.UniqueId; import net.opentsdb.utils.Config; +import net.opentsdb.utils.Threads; -import org.apache.zookeeper.proto.DeleteRequest; import org.hbase.async.Bytes; import org.hbase.async.GetRequest; import org.hbase.async.HBaseClient; import org.hbase.async.KeyValue; import org.hbase.async.PutRequest; import org.hbase.async.Scanner; +import org.jboss.netty.util.HashedWheelTimer; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -57,26 +59,27 @@ @PowerMockIgnore({"javax.management.*", "javax.xml.*", "ch.qos.*", "org.slf4j.*", "com.sum.*", "org.xml.*"}) -@PrepareForTest({TSDB.class, Config.class, UniqueId.class, HBaseClient.class, +@PrepareForTest({ TSDB.class, Config.class, UniqueId.class, HBaseClient.class, GetRequest.class, PutRequest.class, KeyValue.class, Fsck.class, - FsckOptions.class, Scanner.class, DeleteRequest.class, Annotation.class, - RowKey.class, Tags.class}) -public final class TestFsck { - private final static byte[] ROW = - MockBase.stringToBytes("00000150E22700000001000001"); - private final static byte[] ROW2 = - MockBase.stringToBytes("00000150E23510000001000001"); - private final static byte[] ROW3 = - MockBase.stringToBytes("00000150E24320000001000001"); - private Config config; - private TSDB tsdb = null; - private HBaseClient client = mock(HBaseClient.class); - private UniqueId metrics = mock(UniqueId.class); - private UniqueId tag_names = mock(UniqueId.class); - private UniqueId tag_values = mock(UniqueId.class); - private MockBase storage; - private FsckOptions options = mock(FsckOptions.class); - private final static List<byte[]> tags = new ArrayList<byte[]>(1); + FsckOptions.class, Scanner.class, Annotation.class, Tags.class, + HashedWheelTimer.class, Threads.class }) +public class TestFsck { + protected byte[] GLOBAL_ROW = + new byte[] {0, 0, 0, 0x52, (byte)0xC3, 0x5A, (byte)0x80}; + protected byte[] ROW = MockBase.stringToBytes("00000150E22700000001000001"); + protected byte[] ROW2 = MockBase.stringToBytes("00000150E23510000001000001"); + protected byte[] ROW3 = MockBase.stringToBytes("00000150E24320000001000001"); + protected byte[] BAD_KEY = { 0x00, 0x00, 0x01 }; + protected Config config; + protected TSDB tsdb = null; + protected HBaseClient client; + protected UniqueId metrics; + protected UniqueId tag_names; + protected UniqueId tag_values; + protected MockBase storage; + protected FsckOptions options; + protected FakeTaskTimer timer; + protected final static List<byte[]> tags = new ArrayList<byte[]>(1); static { tags.add(new byte[] { 0, 0, 1, 0, 0, 1}); } @@ -84,10 +87,24 @@ public final class TestFsck { @SuppressWarnings("unchecked") @Before public void before() throws Exception { - PowerMockito.whenNew(HBaseClient.class) - .withArguments(anyString(), anyString()).thenReturn(client); + client = mock(HBaseClient.class); + metrics = mock(UniqueId.class); + tag_names = mock(UniqueId.class); + tag_values = mock(UniqueId.class); + options = mock(FsckOptions.class); + timer = new FakeTaskTimer(); + + PowerMockito.mockStatic(Threads.class); + PowerMockito.when(Threads.newTimer(anyString())).thenReturn(timer); + PowerMockito.when(Threads.newTimer(anyInt(), anyString())).thenReturn(timer); + + PowerMockito.whenNew(HashedWheelTimer.class).withNoArguments() + .thenReturn(timer); + PowerMockito.whenNew(HBaseClient.class).withAnyArguments() + .thenReturn(client); + config = new Config(false); - tsdb = new TSDB(config); + tsdb = new TSDB(client, config); when(client.flush()).thenReturn(Deferred.fromResult(null)); storage = new MockBase(tsdb, client, true, true, true, true); @@ -119,7 +136,8 @@ public void before() throws Exception { // mock UniqueId when(metrics.getId("sys.cpu.user")).thenReturn(new byte[] { 0, 0, 1 }); - when(metrics.getName(new byte[] { 0, 0, 1 })).thenReturn("sys.cpu.user"); + when(metrics.getNameAsync(new byte[] { 0, 0, 1 })) + .thenReturn(Deferred.fromResult("sys.cpu.user")); when(metrics.getId("sys.cpu.system")) .thenThrow(new NoSuchUniqueName("sys.cpu.system", "metric")); when(metrics.getId("sys.cpu.nice")).thenReturn(new byte[] { 0, 0, 2 }); @@ -136,19 +154,11 @@ public void before() throws Exception { when(tag_values.getOrCreateId("web02")).thenReturn(new byte[] { 0, 0, 2 }); when(tag_values.getId("web03")) .thenThrow(new NoSuchUniqueName("web03", "metric")); - - PowerMockito.mockStatic(RowKey.class); - when(RowKey.metricNameAsync((TSDB)any(), (byte[])any())) - .thenReturn(Deferred.fromResult("sys.cpu.user")); PowerMockito.mockStatic(Tags.class); when(Tags.resolveIds((TSDB)any(), (ArrayList<byte[]>)any())) .thenReturn(null); // don't care - -// PowerMockito.mockStatic(Thread.class); -// PowerMockito.doNothing().when(Thread.class); -// Thread.sleep(anyLong()); - + when(metrics.width()).thenReturn((short)3); when(tag_names.width()).thenReturn((short)3); when(tag_values.width()).thenReturn((short)3); @@ -156,10 +166,7 @@ public void before() throws Exception { @Test public void globalAnnotation() throws Exception { - // make sure we don't catch this during a query. We should start with - // the first metric (0, 0, 1) whereas globals are on metric (0, 0, 0). - storage.addColumn(new byte[] {0, 0, 0, 0x52, (byte)0xC3, 0x5A, (byte)0x80}, - new byte[] {1, 0, 0}, "{}".getBytes()); + storage.addColumn(GLOBAL_ROW, new byte[] {1, 0, 0}, "{}".getBytes()); final Fsck fsck = new Fsck(tsdb, options); fsck.runFullTable(); @@ -204,6 +211,7 @@ public void noErrorsMultipleRows() throws Exception { storage.addColumn(ROW2, qual2, val2); storage.addColumn(ROW3, qual1, val1); storage.addColumn(ROW3, qual2, val2); +storage.dumpToSystemOut(); final Fsck fsck = new Fsck(tsdb, options); fsck.runFullTable(); assertEquals(6, fsck.kvs_processed.get()); @@ -410,7 +418,7 @@ public void singleValueCompactedFix() throws Exception { @Test public void noSuchMetricId() throws Exception { when(options.fix()).thenReturn(true); - when(RowKey.metricNameAsync((TSDB)any(), (byte[])any())) + when(metrics.getNameAsync((byte[])any())) .thenThrow(new NoSuchUniqueId("metric", new byte[] { 0, 0, 1 })); final byte[] qual1 = { 0x00, 0x07 }; @@ -433,7 +441,7 @@ public void noSuchMetricId() throws Exception { public void noSuchMetricIdFix() throws Exception { when(options.fix()).thenReturn(true); when(options.deleteOrphans()).thenReturn(true); - when(RowKey.metricNameAsync((TSDB)any(), (byte[])any())) + when(metrics.getNameAsync((byte[])any())) .thenThrow(new NoSuchUniqueId("metric", new byte[] { 0, 0, 1 })); final byte[] qual1 = { 0x00, 0x07 }; @@ -506,11 +514,11 @@ public void badRowKey() throws Exception { final byte[] val1 = Bytes.fromLong(4L); final byte[] qual2 = { 0x00, 0x27 }; final byte[] val2 = Bytes.fromLong(5L); - final byte[] bad_key = { 0x00, 0x00, 0x01 }; + storage.addColumn(ROW, qual1, val1); storage.addColumn(ROW, qual2, val2); - storage.addColumn(bad_key, qual1, val1); - storage.addColumn(bad_key, qual2, val2); + storage.addColumn(BAD_KEY, qual1, val1); + storage.addColumn(BAD_KEY, qual2, val2); storage.addColumn(ROW3, qual1, val1); storage.addColumn(ROW3, qual2, val2); @@ -520,7 +528,7 @@ public void badRowKey() throws Exception { assertEquals(3, fsck.rows_processed.get()); assertEquals(1, fsck.totalErrors()); assertEquals(2, storage.numColumns(ROW)); - assertEquals(2, storage.numColumns(bad_key)); + assertEquals(2, storage.numColumns(BAD_KEY)); assertEquals(2, storage.numColumns(ROW3)); } @@ -533,11 +541,11 @@ public void badRowKeyFix() throws Exception { final byte[] val1 = Bytes.fromLong(4L); final byte[] qual2 = { 0x00, 0x27 }; final byte[] val2 = Bytes.fromLong(5L); - final byte[] bad_key = { 0x00, 0x00, 0x01 }; + storage.addColumn(ROW, qual1, val1); storage.addColumn(ROW, qual2, val2); - storage.addColumn(bad_key, qual1, val1); - storage.addColumn(bad_key, qual2, val2); + storage.addColumn(BAD_KEY, qual1, val1); + storage.addColumn(BAD_KEY, qual2, val2); storage.addColumn(ROW3, qual1, val1); storage.addColumn(ROW3, qual2, val2); @@ -547,7 +555,7 @@ public void badRowKeyFix() throws Exception { assertEquals(3, fsck.rows_processed.get()); assertEquals(1, fsck.totalErrors()); assertEquals(2, storage.numColumns(ROW)); - assertEquals(-1, storage.numColumns(bad_key)); + assertEquals(-1, storage.numColumns(BAD_KEY)); assertEquals(2, storage.numColumns(ROW3)); } @@ -1489,6 +1497,120 @@ public void badCompactTooLongFix() throws Exception { assertEquals(-1, storage.numColumns(ROW)); } + @Test + public void appendOK() throws Exception { + final byte[] qual1 = { 0x0, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x0, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] appendq = { 0x05, 0x0, 0x0 }; + storage.addColumn(ROW, appendq, + MockBase.concatByteArrays(qual1, val1, qual2, val2)); + + final Fsck fsck = new Fsck(tsdb, options); + fsck.runFullTable(); + assertEquals(1, fsck.kvs_processed.get()); + assertEquals(0, fsck.bad_compacted_columns.get()); + assertEquals(1, fsck.append_dps.get()); + assertEquals(0, fsck.append_dps_fixed.get()); + assertEquals(0, fsck.totalErrors()); + assertEquals(0, fsck.correctable()); + assertArrayEquals( MockBase.concatByteArrays(qual1, val1, qual2, val2), + storage.getColumn(ROW, appendq)); + } + + @Test + public void appendOutOfOrder() throws Exception { + final byte[] qual1 = { 0x0, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x0, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] appendq = { 0x05, 0x0, 0x0 }; + storage.addColumn(ROW, appendq, + MockBase.concatByteArrays(qual2, val2, qual1, val1)); + + final Fsck fsck = new Fsck(tsdb, options); + fsck.runFullTable(); + assertEquals(1, fsck.kvs_processed.get()); + assertEquals(0, fsck.bad_compacted_columns.get()); + assertEquals(1, fsck.append_dps.get()); + assertEquals(0, fsck.append_dps_fixed.get()); + assertEquals(0, fsck.totalErrors()); + assertEquals(0, fsck.correctable()); + assertArrayEquals( MockBase.concatByteArrays(qual2, val2, qual1, val1), + storage.getColumn(ROW, appendq)); + } + + @Test + public void appendOutOfOrderFixed() throws Exception { + config.overrideConfig("tsd.storage.repair_appends", "true"); + final byte[] qual1 = { 0x0, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x0, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] appendq = { 0x05, 0x0, 0x0 }; + storage.addColumn(ROW, appendq, + MockBase.concatByteArrays(qual2, val2, qual1, val1)); + + final Fsck fsck = new Fsck(tsdb, options); + fsck.runFullTable(); + assertEquals(1, fsck.kvs_processed.get()); + assertEquals(0, fsck.bad_compacted_columns.get()); + assertEquals(1, fsck.append_dps.get()); + assertEquals(1, fsck.append_dps_fixed.get()); + assertEquals(0, fsck.totalErrors()); + assertEquals(0, fsck.correctable()); + assertArrayEquals( MockBase.concatByteArrays(qual1, val1, qual2, val2), + storage.getColumn(ROW, appendq)); + } + + @Test + public void appendDupe() throws Exception { + final byte[] qual1 = { 0x0, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x0, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] appendq = { 0x05, 0x0, 0x0 }; + storage.addColumn(ROW, appendq, + MockBase.concatByteArrays(qual1, val1, qual1, val1, qual2, val2)); + + final Fsck fsck = new Fsck(tsdb, options); + fsck.runFullTable(); + assertEquals(1, fsck.kvs_processed.get()); + assertEquals(0, fsck.bad_compacted_columns.get()); + assertEquals(1, fsck.append_dps.get()); + assertEquals(0, fsck.append_dps_fixed.get()); + assertEquals(0, fsck.totalErrors()); + assertEquals(0, fsck.correctable()); + assertArrayEquals( MockBase.concatByteArrays(qual1, val1, qual1, val1, qual2, val2), + storage.getColumn(ROW, appendq)); + } + /* + * TODO - Fix dupes in the appends by re-writing the data. Right now it just + * resolves them at query time but leaves the values in storage. + @Test + public void appendDupeFix() throws Exception { + final byte[] qual1 = { 0x0, 0x07 }; + final byte[] val1 = Bytes.fromLong(4L); + final byte[] qual2 = { 0x0, 0x27 }; + final byte[] val2 = Bytes.fromLong(5L); + final byte[] appendq = { 0x05, 0x0, 0x0 }; + storage.addColumn(ROW, appendq, + MockBase.concatByteArrays(qual1, val1, qual1, val1, qual2, val2)); + + final Fsck fsck = new Fsck(tsdb, options); + fsck.runFullTable(); + assertEquals(1, fsck.kvs_processed.get()); + assertEquals(0, fsck.bad_compacted_columns.get()); + assertEquals(1, fsck.append_dps.get()); + assertEquals(1, fsck.append_dps_fixed.get()); + assertEquals(0, fsck.totalErrors()); + assertEquals(0, fsck.correctable()); + assertArrayEquals( MockBase.concatByteArrays(qual1, val1, qual2, val2), + storage.getColumn(ROW, appendq)); + } + */ + // VLE -------------------------------------------- @Test diff --git a/test/tools/TestFsckSalted.java b/test/tools/TestFsckSalted.java new file mode 100644 index 0000000000..192e8dfd71 --- /dev/null +++ b/test/tools/TestFsckSalted.java @@ -0,0 +1,123 @@ +package net.opentsdb.tools; + +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.when; +import static org.powermock.api.mockito.PowerMockito.mock; + +import java.lang.reflect.Field; +import java.util.ArrayList; + +import org.hbase.async.HBaseClient; +import org.jboss.netty.util.HashedWheelTimer; +import org.junit.Before; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; + +import com.stumbleupon.async.Deferred; + +import net.opentsdb.core.Const; +import net.opentsdb.core.TSDB; +import net.opentsdb.core.Tags; +import net.opentsdb.core.BaseTsdbTest.FakeTaskTimer; +import net.opentsdb.storage.MockBase; +import net.opentsdb.uid.NoSuchUniqueName; +import net.opentsdb.uid.UniqueId; +import net.opentsdb.utils.Config; +import net.opentsdb.utils.Threads; + +@PrepareForTest({ Const.class }) +public class TestFsckSalted extends TestFsck { + + @SuppressWarnings("unchecked") + @Before + public void before() throws Exception { + client = mock(HBaseClient.class); + metrics = mock(UniqueId.class); + tag_names = mock(UniqueId.class); + tag_values = mock(UniqueId.class); + options = mock(FsckOptions.class); + timer = new FakeTaskTimer(); + + PowerMockito.mockStatic(Threads.class); + PowerMockito.when(Threads.newTimer(anyString())).thenReturn(timer); + PowerMockito.when(Threads.newTimer(anyInt(), anyString())).thenReturn(timer); + + PowerMockito.whenNew(HashedWheelTimer.class).withNoArguments() + .thenReturn(timer); + PowerMockito.whenNew(HBaseClient.class).withAnyArguments() + .thenReturn(client); + + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(2); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + PowerMockito.when(Const.MAX_NUM_TAGS()).thenReturn((short) 8); + + GLOBAL_ROW = new byte[] {0, 0, 0, 0, 0x52, (byte)0xC3, 0x5A, (byte)0x80}; + ROW = MockBase.stringToBytes("0000000150E22700000001000001"); + ROW2 = MockBase.stringToBytes("0100000150E23510000001000001"); + ROW3 = MockBase.stringToBytes("0100000150E24320000001000001"); + BAD_KEY = new byte[] { 0x01, 0x00, 0x00, 0x01 }; + + config = new Config(false); + tsdb = new TSDB(client, config); + when(client.flush()).thenReturn(Deferred.fromResult(null)); + + storage = new MockBase(tsdb, client, true, true, true, true); + storage.setFamily("t".getBytes(MockBase.ASCII())); + + when(options.fix()).thenReturn(false); + when(options.compact()).thenReturn(false); + when(options.resolveDupes()).thenReturn(false); + when(options.lastWriteWins()).thenReturn(false); + when(options.deleteOrphans()).thenReturn(false); + when(options.deleteUnknownColumns()).thenReturn(false); + when(options.deleteBadValues()).thenReturn(false); + when(options.deleteBadRows()).thenReturn(false); + when(options.deleteBadCompacts()).thenReturn(false); + when(options.threads()).thenReturn(1); + + // replace the "real" field objects with mocks + Field met = tsdb.getClass().getDeclaredField("metrics"); + met.setAccessible(true); + met.set(tsdb, metrics); + + Field tagk = tsdb.getClass().getDeclaredField("tag_names"); + tagk.setAccessible(true); + tagk.set(tsdb, tag_names); + + Field tagv = tsdb.getClass().getDeclaredField("tag_values"); + tagv.setAccessible(true); + tagv.set(tsdb, tag_values); + + // mock UniqueId + when(metrics.getId("sys.cpu.user")).thenReturn(new byte[] { 0, 0, 1 }); + when(metrics.getNameAsync(new byte[] { 0, 0, 1 })) + .thenReturn(Deferred.fromResult("sys.cpu.user")); + when(metrics.getId("sys.cpu.system")) + .thenThrow(new NoSuchUniqueName("sys.cpu.system", "metric")); + when(metrics.getId("sys.cpu.nice")).thenReturn(new byte[] { 0, 0, 2 }); + when(metrics.getName(new byte[] { 0, 0, 2 })).thenReturn("sys.cpu.nice"); + when(tag_names.getId("host")).thenReturn(new byte[] { 0, 0, 1 }); + when(tag_names.getName(new byte[] { 0, 0, 1 })).thenReturn("host"); + when(tag_names.getOrCreateId("host")).thenReturn(new byte[] { 0, 0, 1 }); + when(tag_names.getId("dc")).thenThrow(new NoSuchUniqueName("dc", "metric")); + when(tag_values.getId("web01")).thenReturn(new byte[] { 0, 0, 1 }); + when(tag_values.getName(new byte[] { 0, 0, 1 })).thenReturn("web01"); + when(tag_values.getOrCreateId("web01")).thenReturn(new byte[] { 0, 0, 1 }); + when(tag_values.getId("web02")).thenReturn(new byte[] { 0, 0, 2 }); + when(tag_values.getName(new byte[] { 0, 0, 2 })).thenReturn("web02"); + when(tag_values.getOrCreateId("web02")).thenReturn(new byte[] { 0, 0, 2 }); + when(tag_values.getId("web03")) + .thenThrow(new NoSuchUniqueName("web03", "metric")); + + PowerMockito.mockStatic(Tags.class); + when(Tags.resolveIds((TSDB)any(), (ArrayList<byte[]>)any())) + .thenReturn(null); // don't care + + when(metrics.width()).thenReturn((short)3); + when(tag_names.width()).thenReturn((short)3); + when(tag_values.width()).thenReturn((short)3); + } +} diff --git a/test/tools/TestFsckWRollups.java b/test/tools/TestFsckWRollups.java new file mode 100644 index 0000000000..60656fcd0a --- /dev/null +++ b/test/tools/TestFsckWRollups.java @@ -0,0 +1,204 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2021 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.tools; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.when; +import static org.powermock.api.mockito.PowerMockito.mock; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import net.opentsdb.core.Query; +import net.opentsdb.core.TSDB; +import net.opentsdb.core.Tags; +import net.opentsdb.core.BaseTsdbTest.FakeTaskTimer; +import net.opentsdb.meta.Annotation; +import net.opentsdb.rollup.RollupConfig; +import net.opentsdb.rollup.RollupInterval; +import net.opentsdb.storage.MockBase; +import net.opentsdb.uid.NoSuchUniqueId; +import net.opentsdb.uid.NoSuchUniqueName; +import net.opentsdb.uid.UniqueId; +import net.opentsdb.utils.Config; +import net.opentsdb.utils.Threads; + +import org.hbase.async.Bytes; +import org.hbase.async.GetRequest; +import org.hbase.async.HBaseClient; +import org.hbase.async.KeyValue; +import org.hbase.async.PutRequest; +import org.hbase.async.Scanner; +import org.jboss.netty.util.HashedWheelTimer; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +import com.stumbleupon.async.Deferred; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@PrepareForTest({ TSDB.class, Config.class, UniqueId.class, HBaseClient.class, + GetRequest.class, PutRequest.class, KeyValue.class, Fsck.class, + FsckOptions.class, Scanner.class, Annotation.class, Tags.class, + HashedWheelTimer.class, Threads.class }) +public class TestFsckWRollups { + protected byte[] GLOBAL_ROW = + new byte[] {0, 0, 0, 0x52, (byte)0xC3, 0x5A, (byte)0x80}; + protected byte[] ROW = MockBase.stringToBytes("00000150E22700000001000001"); + protected byte[] ROW2 = MockBase.stringToBytes("00000150E23510000001000001"); + protected byte[] ROW3 = MockBase.stringToBytes("00000150E24320000001000001"); + protected byte[] BAD_KEY = { 0x00, 0x00, 0x01 }; + protected Config config; + protected TSDB tsdb = null; + protected HBaseClient client; + protected UniqueId metrics; + protected UniqueId tag_names; + protected UniqueId tag_values; + protected MockBase storage; + protected FsckOptions options; + protected FakeTaskTimer timer; + protected final static List<byte[]> tags = new ArrayList<byte[]>(1); + static { + tags.add(new byte[] { 0, 0, 1, 0, 0, 1}); + } + + @SuppressWarnings("unchecked") + @Before + public void before() throws Exception { + client = mock(HBaseClient.class); + metrics = mock(UniqueId.class); + tag_names = mock(UniqueId.class); + tag_values = mock(UniqueId.class); + options = mock(FsckOptions.class); + timer = new FakeTaskTimer(); + + PowerMockito.mockStatic(Threads.class); + PowerMockito.when(Threads.newTimer(anyString())).thenReturn(timer); + PowerMockito.when(Threads.newTimer(anyInt(), anyString())).thenReturn(timer); + + PowerMockito.whenNew(HashedWheelTimer.class).withNoArguments() + .thenReturn(timer); + PowerMockito.whenNew(HBaseClient.class).withAnyArguments() + .thenReturn(client); + + config = new Config(false); + tsdb = new TSDB(client, config); + when(client.flush()).thenReturn(Deferred.fromResult(null)); + + storage = new MockBase(tsdb, client, true, true, true, true); + storage.setFamily("t".getBytes(MockBase.ASCII())); + + when(options.fix()).thenReturn(false); + when(options.compact()).thenReturn(false); + when(options.resolveDupes()).thenReturn(false); + when(options.lastWriteWins()).thenReturn(false); + when(options.deleteOrphans()).thenReturn(false); + when(options.deleteUnknownColumns()).thenReturn(false); + when(options.deleteBadValues()).thenReturn(false); + when(options.deleteBadRows()).thenReturn(false); + when(options.deleteBadCompacts()).thenReturn(false); + when(options.threads()).thenReturn(1); + + // replace the "real" field objects with mocks + Field met = tsdb.getClass().getDeclaredField("metrics"); + met.setAccessible(true); + met.set(tsdb, metrics); + + Field tagk = tsdb.getClass().getDeclaredField("tag_names"); + tagk.setAccessible(true); + tagk.set(tsdb, tag_names); + + Field tagv = tsdb.getClass().getDeclaredField("tag_values"); + tagv.setAccessible(true); + tagv.set(tsdb, tag_values); + + // mock UniqueId + when(metrics.getId("sys.cpu.user")).thenReturn(new byte[] { 0, 0, 1 }); + when(metrics.getNameAsync(new byte[] { 0, 0, 1 })) + .thenReturn(Deferred.fromResult("sys.cpu.user")); + when(metrics.getId("sys.cpu.system")) + .thenThrow(new NoSuchUniqueName("sys.cpu.system", "metric")); + when(metrics.getId("sys.cpu.nice")).thenReturn(new byte[] { 0, 0, 2 }); + when(metrics.getName(new byte[] { 0, 0, 2 })).thenReturn("sys.cpu.nice"); + when(tag_names.getId("host")).thenReturn(new byte[] { 0, 0, 1 }); + when(tag_names.getName(new byte[] { 0, 0, 1 })).thenReturn("host"); + when(tag_names.getOrCreateId("host")).thenReturn(new byte[] { 0, 0, 1 }); + when(tag_names.getId("dc")).thenThrow(new NoSuchUniqueName("dc", "metric")); + when(tag_values.getId("web01")).thenReturn(new byte[] { 0, 0, 1 }); + when(tag_values.getName(new byte[] { 0, 0, 1 })).thenReturn("web01"); + when(tag_values.getOrCreateId("web01")).thenReturn(new byte[] { 0, 0, 1 }); + when(tag_values.getId("web02")).thenReturn(new byte[] { 0, 0, 2 }); + when(tag_values.getName(new byte[] { 0, 0, 2 })).thenReturn("web02"); + when(tag_values.getOrCreateId("web02")).thenReturn(new byte[] { 0, 0, 2 }); + when(tag_values.getId("web03")) + .thenThrow(new NoSuchUniqueName("web03", "metric")); + + PowerMockito.mockStatic(Tags.class); + when(Tags.resolveIds((TSDB)any(), (ArrayList<byte[]>)any())) + .thenReturn(null); // don't care + + when(metrics.width()).thenReturn((short)3); + when(tag_names.width()).thenReturn((short)3); + when(tag_values.width()).thenReturn((short)3); + + RollupConfig rollup_config = RollupConfig.builder() + .addAggregationId("sum", 0) + .addAggregationId("count", 1) + .addAggregationId("min", 2) + .addAggregationId("max", 3) + .addInterval(RollupInterval.builder() + .setInterval("1m") + .setRowSpan("24h") + .setTable("tsdb-1m") + .setPreAggregationTable("tsdb-preagg-1m")) + .addInterval(RollupInterval.builder() + .setInterval("1h") + .setRowSpan("1d") + .setTable("tsdb-1h") + .setPreAggregationTable("tsdb-preagg-1h")) + .build(); + Whitebox.setInternalState(tsdb, "rollup_config", rollup_config); + } + + @Test + public void oneBadOneGood() throws Exception { + final byte[] qual1 = { 0x01, 0x00, 0x00 }; + final byte[] val1 = { 4 }; + final byte[] qual2 = { (byte) 0x2, 0x00, 0x02 }; + final byte[] val2 = new byte[] { 0, 0, 0, 5 }; + storage.addColumn(ROW, qual1, val1); + storage.addColumn(ROW, qual2, val2); + + final Fsck fsck = new Fsck(tsdb, options); + fsck.runFullTable(); + assertEquals(2, fsck.kvs_processed.get()); + assertEquals(1, fsck.bad_values.get()); + assertEquals(1, fsck.totalErrors()); + } + +} diff --git a/test/tools/TestTextImporter.java b/test/tools/TestTextImporter.java index bf703a4578..851a9af0f4 100644 --- a/test/tools/TestTextImporter.java +++ b/test/tools/TestTextImporter.java @@ -37,8 +37,8 @@ import net.opentsdb.uid.UniqueId; import net.opentsdb.utils.Config; -import org.apache.zookeeper.proto.DeleteRequest; import org.hbase.async.Bytes; +import org.hbase.async.DeleteRequest; import org.hbase.async.GetRequest; import org.hbase.async.HBaseClient; import org.hbase.async.KeyValue; @@ -85,7 +85,7 @@ public class TestTextImporter { static { try { importFile = TextImporter.class.getDeclaredMethod("importFile", - HBaseClient.class, TSDB.class, String.class); + HBaseClient.class, TSDB.class, String.class, boolean.class); importFile.setAccessible(true); } catch (Exception e) { throw new RuntimeException("Failed in static initializer", e); @@ -94,10 +94,8 @@ public class TestTextImporter { @Before public void before() throws Exception { - PowerMockito.whenNew(HBaseClient.class) - .withArguments(anyString(), anyString()).thenReturn(client); config = new Config(false); - tsdb = new TSDB(config); + tsdb = new TSDB(client, config); storage = new MockBase(tsdb, client, true, true, true, true); storage.setFamily("t".getBytes(MockBase.ASCII())); @@ -158,7 +156,7 @@ public void importFileGoodIntegers1Byte() throws Exception { "sys.cpu.user 1356998400 0 host=web01\n" + "sys.cpu.user 1356998400 127 host=web02"; setData(data); - Integer points = (Integer)importFile.invoke(null, client, tsdb, "file"); + Integer points = (Integer)importFile.invoke(null, client, tsdb, "file", false); assertEquals(2, (int)points); byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, @@ -179,7 +177,7 @@ public void importFileGoodIntegers1ByteNegative() throws Exception { "sys.cpu.user 1356998400 -0 host=web01\n" + "sys.cpu.user 1356998400 -128 host=web02"; setData(data); - Integer points = (Integer)importFile.invoke(null, client, tsdb, "file"); + Integer points = (Integer)importFile.invoke(null, client, tsdb, "file", false); assertEquals(2, (int)points); byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, @@ -200,7 +198,7 @@ public void importFileGoodIntegers2Byte() throws Exception { "sys.cpu.user 1356998400 128 host=web01\n" + "sys.cpu.user 1356998400 32767 host=web02"; setData(data); - Integer points = (Integer)importFile.invoke(null, client, tsdb, "file"); + Integer points = (Integer)importFile.invoke(null, client, tsdb, "file", false); assertEquals(2, (int)points); byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, @@ -221,7 +219,7 @@ public void importFileGoodIntegers2ByteNegative() throws Exception { "sys.cpu.user 1356998400 -129 host=web01\n" + "sys.cpu.user 1356998400 -32768 host=web02"; setData(data); - Integer points = (Integer)importFile.invoke(null, client, tsdb, "file"); + Integer points = (Integer)importFile.invoke(null, client, tsdb, "file", false); assertEquals(2, (int)points); byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, @@ -242,7 +240,7 @@ public void importFileGoodIntegers4Byte() throws Exception { "sys.cpu.user 1356998400 32768 host=web01\n" + "sys.cpu.user 1356998400 2147483647 host=web02"; setData(data); - Integer points = (Integer)importFile.invoke(null, client, tsdb, "file"); + Integer points = (Integer)importFile.invoke(null, client, tsdb, "file", false); assertEquals(2, (int)points); byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, @@ -263,7 +261,7 @@ public void importFileGoodIntegers4ByteNegative() throws Exception { "sys.cpu.user 1356998400 -32769 host=web01\n" + "sys.cpu.user 1356998400 -2147483648 host=web02"; setData(data); - Integer points = (Integer)importFile.invoke(null, client, tsdb, "file"); + Integer points = (Integer)importFile.invoke(null, client, tsdb, "file", false); assertEquals(2, (int)points); byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, @@ -284,7 +282,7 @@ public void importFileGoodIntegers8Byte() throws Exception { "sys.cpu.user 1356998400 2147483648 host=web01\n" + "sys.cpu.user 1356998400 9223372036854775807 host=web02"; setData(data); - Integer points = (Integer)importFile.invoke(null, client, tsdb, "file"); + Integer points = (Integer)importFile.invoke(null, client, tsdb, "file", false); assertEquals(2, (int)points); byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, 0, 0, 1, 0, 0, 1}; @@ -304,7 +302,7 @@ public void importFileGoodIntegers8ByteNegative() throws Exception { "sys.cpu.user 1356998400 -2147483649 host=web01\n" + "sys.cpu.user 1356998400 -9223372036854775808 host=web02"; setData(data); - Integer points = (Integer)importFile.invoke(null, client, tsdb, "file"); + Integer points = (Integer)importFile.invoke(null, client, tsdb, "file", false); assertEquals(2, (int)points); byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, @@ -325,7 +323,16 @@ public void importFileTimestampZero() throws Exception { "sys.cpu.user 0 0 host=web01\n" + "sys.cpu.user 0 127 host=web02"; setData(data); - importFile.invoke(null, client, tsdb, "file"); + importFile.invoke(null, client, tsdb, "file", false); + } + + @Test + public void importFileTimestampZeroSkip() throws Exception { + String data = + "sys.cpu.user 0 0 host=web01\n" + + "sys.cpu.user 0 127 host=web02"; + setData(data); + importFile.invoke(null, client, tsdb, "file", true); } @Test (expected = RuntimeException.class) @@ -334,7 +341,16 @@ public void importFileTimestampNegative() throws Exception { "sys.cpu.user -11356998400 0 host=web01\n" + "sys.cpu.user -11356998400 127 host=web02"; setData(data); - importFile.invoke(null, client, tsdb, "file"); + importFile.invoke(null, client, tsdb, "file", false); + } + + @Test + public void importFileTimestampNegativeSkip() throws Exception { + String data = + "sys.cpu.user -11356998400 0 host=web01\n" + + "sys.cpu.user -11356998400 127 host=web02"; + setData(data); + importFile.invoke(null, client, tsdb, "file", true); } @Test @@ -343,7 +359,7 @@ public void importFileMaxSecondTimestamp() throws Exception { "sys.cpu.user 4294967295 24 host=web01\n" + "sys.cpu.user 4294967295 42 host=web02"; setData(data); - Integer points = (Integer)importFile.invoke(null, client, tsdb, "file"); + Integer points = (Integer)importFile.invoke(null, client, tsdb, "file", false); assertEquals(2, (int)points); byte[] row = new byte[] { 0, 0, 1, (byte) 0xFF, (byte) 0xFF, (byte) 0xF9, @@ -364,7 +380,7 @@ public void importFileMinMSTimestamp() throws Exception { "sys.cpu.user 4294967296 24 host=web01\n" + "sys.cpu.user 4294967296 42 host=web02"; setData(data); - Integer points = (Integer)importFile.invoke(null, client, tsdb, "file"); + Integer points = (Integer)importFile.invoke(null, client, tsdb, "file", false); assertEquals(2, (int)points); byte[] row = new byte[] { 0, 0, 1, 0, (byte) 0x41, (byte) 0x88, (byte) 0x90, @@ -387,7 +403,7 @@ public void importFileMSTimestamp() throws Exception { "sys.cpu.user 1356998400500 24 host=web01\n" + "sys.cpu.user 1356998400500 42 host=web02"; setData(data); - Integer points = (Integer)importFile.invoke(null, client, tsdb, "file"); + Integer points = (Integer)importFile.invoke(null, client, tsdb, "file", false); assertEquals(2, (int)points); byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, @@ -408,7 +424,7 @@ public void importFileMSTimestampTooBig() throws Exception { "sys.cpu.user 13569984005001 24 host=web01\n" + "sys.cpu.user 13569984005001 42 host=web02"; setData(data); - importFile.invoke(null, client, tsdb, "file"); + importFile.invoke(null, client, tsdb, "file", false); } @Test (expected = IllegalArgumentException.class) @@ -417,7 +433,25 @@ public void importFileMSTimestampNegative() throws Exception { "sys.cpu.user -2147483648000L 24 host=web01\n" + "sys.cpu.user -2147483648000L 42 host=web02"; setData(data); - importFile.invoke(null, client, tsdb, "file"); + importFile.invoke(null, client, tsdb, "file", false); + } + + @Test (expected = NumberFormatException.class) + public void importFileTimestampNFE() throws Exception { + String data = + "sys.cpu.user 1356998400 0 host=web01\n" + + "sys.cpu.user notatimestamp 127 host=web02"; + setData(data); + importFile.invoke(null, client, tsdb, "file", false); + } + + @Test + public void importFileTimestampNFESkip() throws Exception { + String data = + "sys.cpu.user 1356998400 0 host=web01\n" + + "sys.cpu.user notatimestamp 127 host=web02"; + setData(data); + importFile.invoke(null, client, tsdb, "file", true); } @Test @@ -426,7 +460,7 @@ public void importFileGoodFloats() throws Exception { "sys.cpu.user 1356998400 24.5 host=web01\n" + "sys.cpu.user 1356998400 42.5 host=web02"; setData(data); - Integer points = (Integer)importFile.invoke(null, client, tsdb, "file"); + Integer points = (Integer)importFile.invoke(null, client, tsdb, "file", false); assertEquals(2, (int)points); byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, @@ -447,7 +481,7 @@ public void importFileGoodFloatsNegative() throws Exception { "sys.cpu.user 1356998400 -24.5 host=web01\n" + "sys.cpu.user 1356998400 -42.5 host=web02"; setData(data); - Integer points = (Integer)importFile.invoke(null, client, tsdb, "file"); + Integer points = (Integer)importFile.invoke(null, client, tsdb, "file", false); assertEquals(2, (int)points); byte[] row = new byte[] { 0, 0, 1, 0x50, (byte) 0xE2, 0x27, 0, @@ -468,7 +502,16 @@ public void importFileNSUTagv() throws Exception { "sys.cpu.user 1356998400 24 host=web01\n" + "sys.cpu.user 1356998400 42 host=web03"; setData(data); - importFile.invoke(null, client, tsdb, "file"); + importFile.invoke(null, client, tsdb, "file", false); + } + + @Test + public void importFileNSUTagvSkip() throws Exception { + String data = + "sys.cpu.user 1356998400 24 host=web01\n" + + "sys.cpu.user 1356998400 42 host=web03"; + setData(data); + importFile.invoke(null, client, tsdb, "file", true); } @Test (expected = NoSuchUniqueName.class) @@ -477,7 +520,16 @@ public void importFileNSUTagk() throws Exception { "sys.cpu.user 1356998400 24 host=web01\n" + "sys.cpu.user 1356998400 42 fqdn=web02"; setData(data); - importFile.invoke(null, client, tsdb, "file"); + importFile.invoke(null, client, tsdb, "file", false); + } + + @Test + public void importFileNSUTagkSkip() throws Exception { + String data = + "sys.cpu.user 1356998400 24 host=web01\n" + + "sys.cpu.user 1356998400 42 fqdn=web02"; + setData(data); + importFile.invoke(null, client, tsdb, "file", true); } @Test (expected = NoSuchUniqueName.class) @@ -486,7 +538,16 @@ public void importFileNSUMetric() throws Exception { "sys.cpu.user 1356998400 24 host=web01\n" + "sys.cpu.system 1356998400 42 host=web02"; setData(data); - importFile.invoke(null, client, tsdb, "file"); + importFile.invoke(null, client, tsdb, "file", false); + } + + @Test + public void importFileNSUMetricSkip() throws Exception { + String data = + "sys.cpu.user 1356998400 24 host=web01\n" + + "sys.cpu.system 1356998400 42 host=web02"; + setData(data); + importFile.invoke(null, client, tsdb, "file", true); } @Test (expected = RuntimeException.class) @@ -495,7 +556,16 @@ public void importFileEmptyMetric() throws Exception { "sys.cpu.user 1356998400 24 host=web01\n" + " 1356998400 42 host=web03"; setData(data); - importFile.invoke(null, client, tsdb, "file"); + importFile.invoke(null, client, tsdb, "file", false); + } + + @Test + public void importFileEmptyMetricSkip() throws Exception { + String data = + "sys.cpu.user 1356998400 24 host=web01\n" + + " 1356998400 42 host=web03"; + setData(data); + importFile.invoke(null, client, tsdb, "file", true); } @Test (expected = RuntimeException.class) @@ -504,7 +574,16 @@ public void importFileEmptyTimestamp() throws Exception { "sys.cpu.user 1356998400 24 host=web01\n" + "sys.cpu.user 42 host=web03"; setData(data); - importFile.invoke(null, client, tsdb, "file"); + importFile.invoke(null, client, tsdb, "file", false); + } + + @Test + public void importFileEmptyTimestampSkip() throws Exception { + String data = + "sys.cpu.user 1356998400 24 host=web01\n" + + "sys.cpu.user 42 host=web03"; + setData(data); + importFile.invoke(null, client, tsdb, "file", true); } @Test (expected = RuntimeException.class) @@ -513,7 +592,34 @@ public void importFileEmptyValue() throws Exception { "sys.cpu.user 1356998400 24 host=web01\n" + "sys.cpu.user 1356998400 host=web03"; setData(data); - importFile.invoke(null, client, tsdb, "file"); + importFile.invoke(null, client, tsdb, "file", false); + } + + @Test + public void importFileEmptyValueSkip() throws Exception { + String data = + "sys.cpu.user 1356998400 24 host=web01\n" + + "sys.cpu.user 1356998400 host=web03"; + setData(data); + importFile.invoke(null, client, tsdb, "file", true); + } + + @Test (expected = NumberFormatException.class) + public void importFileEmptyValueNFE() throws Exception { + String data = + "sys.cpu.user 1356998400 24 host=web01\n" + + "sys.cpu.user 1356998400 notanumber host=web02"; + setData(data); + importFile.invoke(null, client, tsdb, "file", false); + } + + @Test + public void importFileEmptyValueNFESkip() throws Exception { + String data = + "sys.cpu.user 1356998400 24 host=web01\n" + + "sys.cpu.user 1356998400 notanumber host=web02"; + setData(data); + importFile.invoke(null, client, tsdb, "file", true); } @Test (expected = RuntimeException.class) @@ -522,7 +628,16 @@ public void importFileEmptyTags() throws Exception { "sys.cpu.user 1356998400 24 host=web01\n" + "sys.cpu.user 1356998400 42"; setData(data); - importFile.invoke(null, client, tsdb, "file"); + importFile.invoke(null, client, tsdb, "file", false); + } + + @Test + public void importFileEmptyTagsSkip() throws Exception { + String data = + "sys.cpu.user 1356998400 24 host=web01\n" + + "sys.cpu.user 1356998400 42"; + setData(data); + importFile.invoke(null, client, tsdb, "file", true); } @Test (expected = RuntimeException.class) @@ -531,43 +646,52 @@ public void importFileEmptyTagv() throws Exception { "sys.cpu.user 1356998400 24 host=web01\n" + "sys.cpu.user 1356998400 42 host"; setData(data); - importFile.invoke(null, client, tsdb, "file"); + importFile.invoke(null, client, tsdb, "file", false); } - @Test (expected = RuntimeException.class) - public void importFileEmptyTagvEquals() throws Exception { + @Test + public void importFileEmptyTagvSkip() throws Exception { String data = "sys.cpu.user 1356998400 24 host=web01\n" + - "sys.cpu.user 1356998400 42 host="; + "sys.cpu.user 1356998400 42 host"; setData(data); - importFile.invoke(null, client, tsdb, "file"); + importFile.invoke(null, client, tsdb, "file", true); } @Test (expected = RuntimeException.class) - public void importFile0Timestamp() throws Exception { + public void importFileEmptyTagvEquals() throws Exception { String data = "sys.cpu.user 1356998400 24 host=web01\n" + - "sys.cpu.user 0 42 host=web02"; + "sys.cpu.user 1356998400 42 host="; setData(data); - importFile.invoke(null, client, tsdb, "file"); + importFile.invoke(null, client, tsdb, "file", false); } - @Test (expected = RuntimeException.class) - public void importFileNegativeTimestamp() throws Exception { + @Test + public void importFileEmptyTagvEqualsSkip() throws Exception { String data = "sys.cpu.user 1356998400 24 host=web01\n" + - "sys.cpu.user -1356998400 42 host=web02"; + "sys.cpu.user 1356998400 42 host="; setData(data); - importFile.invoke(null, client, tsdb, "file"); + importFile.invoke(null, client, tsdb, "file", true); } - + @Test (expected = IllegalArgumentException.class) public void importFileSameTimestamp() throws Exception { String data = "sys.cpu.user 1356998400 24 host=web01\n" + "sys.cpu.user 1356998400 42 host=web01"; setData(data); - importFile.invoke(null, client, tsdb, "file"); + importFile.invoke(null, client, tsdb, "file", false); + } + + @Test + public void importFileSameTimestampSkip() throws Exception { + String data = + "sys.cpu.user 1356998400 24 host=web01\n" + + "sys.cpu.user 1356998400 42 host=web01"; + setData(data); + importFile.invoke(null, client, tsdb, "file", true); } @Test (expected = IllegalArgumentException.class) @@ -576,7 +700,16 @@ public void importFileLessthanTimestamp() throws Exception { "sys.cpu.user 1356998400 24 host=web01\n" + "sys.cpu.user 1356998300 42 host=web01"; setData(data); - importFile.invoke(null, client, tsdb, "file"); + importFile.invoke(null, client, tsdb, "file", false); + } + + @Test + public void importFileLessthanTimestampSkip() throws Exception { + String data = + "sys.cpu.user 1356998400 24 host=web01\n" + + "sys.cpu.user 1356998300 42 host=web01"; + setData(data); + importFile.invoke(null, client, tsdb, "file", true); } // doesn't throw an exception, just returns "processed 0 data points" @@ -584,16 +717,16 @@ public void importFileLessthanTimestamp() throws Exception { public void importFileEmptyFile() throws Exception { String data = ""; setData(data); - Integer points = (Integer)importFile.invoke(null, client, tsdb, "file"); + Integer points = (Integer)importFile.invoke(null, client, tsdb, "file", false); assertEquals(0, (int)points); } @Test (expected = FileNotFoundException.class) - public void inportFileNotFound() throws Exception { + public void importFileNotFound() throws Exception { PowerMockito.doThrow(new FileNotFoundException()).when(TextImporter.class, PowerMockito.method(TextImporter.class, "open", String.class)) .withArguments(anyString()); - Integer points = (Integer)importFile.invoke(null, client, tsdb, "file"); + Integer points = (Integer)importFile.invoke(null, client, tsdb, "file", false); assertEquals(0, (int)points); } diff --git a/test/tools/TestUID.java b/test/tools/TestUID.java index 2da55c3303..ef52bebd2d 100644 --- a/test/tools/TestUID.java +++ b/test/tools/TestUID.java @@ -15,19 +15,24 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertNull; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.anyString; import static org.powermock.api.mockito.PowerMockito.mock; import java.lang.reflect.Method; import net.opentsdb.core.TSDB; +import net.opentsdb.core.BaseTsdbTest.FakeTaskTimer; import net.opentsdb.storage.MockBase; import net.opentsdb.utils.Config; +import net.opentsdb.utils.Threads; -import org.apache.zookeeper.proto.DeleteRequest; import org.hbase.async.Bytes; +import org.hbase.async.DeleteRequest; import org.hbase.async.HBaseClient; import org.hbase.async.KeyValue; import org.hbase.async.Scanner; +import org.jboss.netty.util.HashedWheelTimer; import org.junit.Before; import org.junit.After; import org.junit.Test; @@ -41,13 +46,14 @@ @PowerMockIgnore({"javax.management.*", "javax.xml.*", "ch.qos.*", "org.slf4j.*", "com.sum.*", "org.xml.*"}) @PrepareForTest({TSDB.class, Config.class, HBaseClient.class, - KeyValue.class, UidManager.class, + KeyValue.class, UidManager.class, HashedWheelTimer.class, Threads.class, Scanner.class, DeleteRequest.class }) public class TestUID { private Config config; private TSDB tsdb = null; private HBaseClient client = mock(HBaseClient.class); private MockBase storage; + private FakeTaskTimer timer = new FakeTaskTimer(); // names used for testing private byte[] NAME_FAMILY = "name".getBytes(MockBase.ASCII()); @@ -55,7 +61,7 @@ public class TestUID { private byte[] METRICS = "metrics".getBytes(MockBase.ASCII()); private byte[] TAGK = "tagk".getBytes(MockBase.ASCII()); private byte[] TAGV = "tagv".getBytes(MockBase.ASCII()); - + private byte[] UID_TABLE = "tsdb-uid".getBytes(MockBase.ASCII()); private final static Method fsck; static { try { @@ -69,8 +75,15 @@ public class TestUID { @Before public void before() throws Exception { - - PowerMockito.whenNew(HBaseClient.class).withAnyArguments().thenReturn(client); + PowerMockito.mockStatic(Threads.class); + PowerMockito.when(Threads.newTimer(anyString())).thenReturn(timer); + PowerMockito.when(Threads.newTimer(anyInt(), anyString())).thenReturn(timer); + + PowerMockito.whenNew(HashedWheelTimer.class).withNoArguments() + .thenReturn(timer); + PowerMockito.whenNew(HBaseClient.class).withAnyArguments() + .thenReturn(client); + config = new Config(false); tsdb = new TSDB(client, config); PowerMockito.spy(System.class); @@ -112,14 +125,14 @@ public void before() throws Exception { public void fsckNoData() throws Exception { storage.flushStorage(); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @Test public void fsckNoErrors() throws Exception { int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @@ -131,18 +144,18 @@ public void fsckNoErrors() throws Exception { @Test public void fsckMetricsUIDHigh() throws Exception { // currently a warning, not an error - storage.addColumn(new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(42L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(42L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @Test public void fsckTagkUIDHigh() throws Exception { // currently a warning, not an error - storage.addColumn(new byte[] { 0 }, ID_FAMILY, TAGK, Bytes.fromLong(42L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, TAGK, Bytes.fromLong(42L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @@ -150,9 +163,9 @@ public void fsckTagkUIDHigh() throws Exception { public void fsckTagvUIDHigh() throws Exception { // currently a warning, not an error - storage.addColumn(new byte[] { 0 }, ID_FAMILY, TAGV, Bytes.fromLong(42L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, TAGV, Bytes.fromLong(42L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @@ -163,60 +176,60 @@ public void fsckTagvUIDHigh() throws Exception { @Test public void fsckMetricsUIDLow() throws Exception { // currently a warning, not an error - storage.addColumn(new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(1L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(1L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(1, errors); } @Test public void fsckFIXMetricsUIDLow() throws Exception { // currently a warning, not an error - storage.addColumn(new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(0L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(0L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), true, false); + UID_TABLE, true, false); assertEquals(1, errors); errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @Test public void fsckTagkUIDLow() throws Exception { - storage.addColumn(new byte[] { 0 }, ID_FAMILY, TAGK, Bytes.fromLong(1L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, TAGK, Bytes.fromLong(1L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(1, errors); } @Test public void fsckFIXTagkUIDLow() throws Exception { - storage.addColumn(new byte[] { 0 }, ID_FAMILY, TAGK, Bytes.fromLong(1L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, TAGK, Bytes.fromLong(1L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), true, false); + UID_TABLE, true, false); assertEquals(1, errors); errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @Test public void fsckTagvUIDLow() throws Exception { - storage.addColumn(new byte[] { 0 }, ID_FAMILY, TAGV, Bytes.fromLong(1L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, TAGV, Bytes.fromLong(1L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(1, errors); } @Test public void fsckFIXTagvUIDLow() throws Exception { - storage.addColumn(new byte[] { 0 }, ID_FAMILY, TAGV, Bytes.fromLong(1L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, TAGV, Bytes.fromLong(1L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), true, false); + UID_TABLE, true, false); assertEquals(1, errors); errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @@ -227,25 +240,25 @@ public void fsckFIXTagvUIDLow() throws Exception { */ @Test public void fsckMetricsUIDWrongLength() throws Exception { - storage.addColumn(new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromInt(3)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromInt(3)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(2, errors); } @Test public void fsckTagkUIDWrongLength() throws Exception { - storage.addColumn(new byte[] { 0 }, ID_FAMILY, TAGK, Bytes.fromInt(3)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, TAGK, Bytes.fromInt(3)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(2, errors); } @Test public void fsckTagvUIDWrongLength() throws Exception { - storage.addColumn(new byte[] { 0 }, ID_FAMILY, TAGV, Bytes.fromInt(3)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, TAGV, Bytes.fromInt(3)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(2, errors); } @@ -259,64 +272,64 @@ public void fsckTagvUIDWrongLength() throws Exception { */ @Test public void fsckMetricsMissingReverse() throws Exception { - storage.flushColumn(new byte[] {0, 0, 1}, NAME_FAMILY, METRICS); + storage.flushColumn(UID_TABLE, new byte[] {0, 0, 1}, NAME_FAMILY, METRICS); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(1, errors); } @Test public void fsckFIXMetricsMissingReverse() throws Exception { - storage.flushColumn(new byte[] {0, 0, 1}, NAME_FAMILY, METRICS); + storage.flushColumn(UID_TABLE, new byte[] {0, 0, 1}, NAME_FAMILY, METRICS); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), true, false); + UID_TABLE, true, false); assertEquals(1, errors); assertArrayEquals("foo".getBytes(MockBase.ASCII()), - storage.getColumn(new byte [] {0, 0, 1}, NAME_FAMILY, METRICS)); + storage.getColumn(UID_TABLE, new byte [] {0, 0, 1}, NAME_FAMILY, METRICS)); errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @Test public void fsckTagkMissingReverse() throws Exception { - storage.flushColumn(new byte[] {0, 0, 1}, NAME_FAMILY, TAGK); + storage.flushColumn(UID_TABLE, new byte[] {0, 0, 1}, NAME_FAMILY, TAGK); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(1, errors); } @Test public void fsckFIXTagkMissingReverse() throws Exception { - storage.flushColumn(new byte[] {0, 0, 1}, NAME_FAMILY, TAGK); + storage.flushColumn(UID_TABLE, new byte[] {0, 0, 1}, NAME_FAMILY, TAGK); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), true, false); + UID_TABLE, true, false); assertEquals(1, errors); assertArrayEquals("host".getBytes(MockBase.ASCII()), - storage.getColumn(new byte [] {0, 0, 1}, NAME_FAMILY, TAGK)); + storage.getColumn(UID_TABLE, new byte [] {0, 0, 1}, NAME_FAMILY, TAGK)); errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @Test public void fsckTagvMissingReverse() throws Exception { - storage.flushColumn(new byte[] {0, 0, 1}, NAME_FAMILY, TAGV); + storage.flushColumn(UID_TABLE, new byte[] {0, 0, 1}, NAME_FAMILY, TAGV); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(1, errors); } @Test public void fsckFIXTagvMissingReverse() throws Exception { - storage.flushColumn(new byte[] {0, 0, 1}, NAME_FAMILY, TAGV); + storage.flushColumn(UID_TABLE, new byte[] {0, 0, 1}, NAME_FAMILY, TAGV); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), true, false); + UID_TABLE, true, false); assertEquals(1, errors); assertArrayEquals("web01".getBytes(MockBase.ASCII()), - storage.getColumn(new byte [] {0, 0, 1}, NAME_FAMILY, TAGV)); + storage.getColumn(UID_TABLE, new byte [] {0, 0, 1}, NAME_FAMILY, TAGV)); errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @@ -337,88 +350,88 @@ public void fsckFIXTagvMissingReverse() throws Exception { */ @Test public void fsckMetricsInconsistentForward() throws Exception { - storage.addColumn("wtf".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.addColumn(UID_TABLE, "wtf".getBytes(MockBase.ASCII()), ID_FAMILY, METRICS, new byte[] {0, 0, 1}); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(3L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(3L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(2, errors); } @Test public void fsckFIXMetricsInconsistentForward() throws Exception { - storage.addColumn("wtf".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.addColumn(UID_TABLE, "wtf".getBytes(MockBase.ASCII()), ID_FAMILY, METRICS, new byte[] {0, 0, 1}); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(3L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(3L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), true, false); + UID_TABLE, true, false); assertEquals(2, errors); assertArrayEquals("fsck.foo.wtf".getBytes(MockBase.ASCII()), - storage.getColumn(new byte [] {0, 0, 1}, NAME_FAMILY, METRICS)); - assertNull(storage.getColumn("foo".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.getColumn(UID_TABLE, new byte [] {0, 0, 1}, NAME_FAMILY, METRICS)); + assertNull(storage.getColumn(UID_TABLE, "foo".getBytes(MockBase.ASCII()), ID_FAMILY, METRICS)); - assertNull(storage.getColumn("wtf".getBytes(MockBase.ASCII()), ID_FAMILY, + assertNull(storage.getColumn(UID_TABLE, "wtf".getBytes(MockBase.ASCII()), ID_FAMILY, METRICS)); errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @Test public void fsckTagkInconsistentForward() throws Exception { - storage.addColumn("some.other.value".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.addColumn(UID_TABLE, "some.other.value".getBytes(MockBase.ASCII()), ID_FAMILY, TAGK, new byte[] {0, 0, 1}); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, TAGK, Bytes.fromLong(3L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, TAGK, Bytes.fromLong(3L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(2, errors); } @Test public void fsckFIXTagkInconsistentForward() throws Exception { - storage.addColumn("wtf".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.addColumn(UID_TABLE, "wtf".getBytes(MockBase.ASCII()), ID_FAMILY, TAGK, new byte[] {0, 0, 1}); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, TAGK, Bytes.fromLong(3L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, TAGK, Bytes.fromLong(3L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), true, false); + UID_TABLE, true, false); assertEquals(2, errors); assertArrayEquals("fsck.host.wtf".getBytes(MockBase.ASCII()), - storage.getColumn(new byte [] {0, 0, 1}, NAME_FAMILY, TAGK)); - assertNull(storage.getColumn("host".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.getColumn(UID_TABLE, new byte [] {0, 0, 1}, NAME_FAMILY, TAGK)); + assertNull(storage.getColumn(UID_TABLE, "host".getBytes(MockBase.ASCII()), ID_FAMILY, TAGK)); - assertNull(storage.getColumn("wtf".getBytes(MockBase.ASCII()), ID_FAMILY, + assertNull(storage.getColumn(UID_TABLE, "wtf".getBytes(MockBase.ASCII()), ID_FAMILY, TAGK)); errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @Test public void fsckTagvInconsistentForward() throws Exception { - storage.addColumn("some.other.value".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.addColumn(UID_TABLE, "some.other.value".getBytes(MockBase.ASCII()), ID_FAMILY, TAGV, new byte[] {0, 0, 1}); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, TAGV, Bytes.fromLong(3L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, TAGV, Bytes.fromLong(3L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(2, errors); } @Test public void fsckFIXTagvInconsistentForward() throws Exception { - storage.addColumn("wtf".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.addColumn(UID_TABLE, "wtf".getBytes(MockBase.ASCII()), ID_FAMILY, TAGV, new byte[] {0, 0, 1}); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, TAGV, Bytes.fromLong(3L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, TAGV, Bytes.fromLong(3L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), true, false); + UID_TABLE, true, false); assertEquals(2, errors); assertArrayEquals("fsck.web01.wtf".getBytes(MockBase.ASCII()), - storage.getColumn(new byte [] {0, 0, 1}, NAME_FAMILY, TAGV)); - assertNull(storage.getColumn("web01".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.getColumn(UID_TABLE, new byte [] {0, 0, 1}, NAME_FAMILY, TAGV)); + assertNull(storage.getColumn(UID_TABLE, "web01".getBytes(MockBase.ASCII()), ID_FAMILY, TAGV)); - assertNull(storage.getColumn("wtf".getBytes(MockBase.ASCII()), ID_FAMILY, + assertNull(storage.getColumn(UID_TABLE, "wtf".getBytes(MockBase.ASCII()), ID_FAMILY, TAGV)); errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @@ -436,70 +449,70 @@ public void fsckFIXTagvInconsistentForward() throws Exception { */ @Test public void fsckMetricsDuplicateForward() throws Exception { - storage.addColumn(new byte[] {0, 0, 2}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 2}, NAME_FAMILY, METRICS, "wtf".getBytes(MockBase.ASCII())); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(2, errors); } @Test public void fsckFIXMetricsDuplicateForward() throws Exception { - storage.addColumn(new byte[] {0, 0, 2}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 2}, NAME_FAMILY, METRICS, "wtf".getBytes(MockBase.ASCII())); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), true, false); + UID_TABLE, true, false); assertEquals(2, errors); assertArrayEquals("bar".getBytes(MockBase.ASCII()), - storage.getColumn(new byte [] {0, 0, 2}, NAME_FAMILY, METRICS)); + storage.getColumn(UID_TABLE, new byte [] {0, 0, 2}, NAME_FAMILY, METRICS)); errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @Test public void fsckTagkDuplicateForward() throws Exception { - storage.addColumn(new byte[] {0, 0, 2}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 2}, NAME_FAMILY, TAGK, "wtf".getBytes(MockBase.ASCII())); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(2, errors); } @Test public void fsckFIXTagkDuplicateForward() throws Exception { - storage.addColumn(new byte[] {0, 0, 2}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 2}, NAME_FAMILY, TAGK, "wtf".getBytes(MockBase.ASCII())); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), true, false); + UID_TABLE, true, false); assertEquals(2, errors); assertArrayEquals("dc".getBytes(MockBase.ASCII()), - storage.getColumn(new byte [] {0, 0, 2}, NAME_FAMILY, TAGK)); + storage.getColumn(UID_TABLE, new byte [] {0, 0, 2}, NAME_FAMILY, TAGK)); errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @Test public void fsckTagvDuplicateForward() throws Exception { - storage.addColumn(new byte[] {0, 0, 2}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 2}, NAME_FAMILY, TAGV, "wtf".getBytes(MockBase.ASCII())); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(2, errors); } @Test public void fsckFIXTagvDuplicateForward() throws Exception { - storage.addColumn(new byte[] {0, 0, 2}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 2}, NAME_FAMILY, TAGV, "wtf".getBytes(MockBase.ASCII())); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), true, false); + UID_TABLE, true, false); assertEquals(2, errors); assertArrayEquals("web02".getBytes(MockBase.ASCII()), - storage.getColumn(new byte [] {0, 0, 2}, NAME_FAMILY, TAGV)); + storage.getColumn(UID_TABLE, new byte [] {0, 0, 2}, NAME_FAMILY, TAGV)); errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @@ -517,7 +530,7 @@ public void fsckMetricsMissingForward() throws Exception { storage.flushColumn("bar".getBytes(MockBase.ASCII()), ID_FAMILY, METRICS); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @@ -527,7 +540,7 @@ public void fsckFIXMetricsMissingForward() throws Exception { storage.flushColumn("bar".getBytes(MockBase.ASCII()), ID_FAMILY, METRICS); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), true, false); + UID_TABLE, true, false); assertEquals(0, errors); assertNull(storage.getColumn(new byte [] {0, 0, 2}, NAME_FAMILY, METRICS)); } @@ -537,7 +550,7 @@ public void fsckTagkMissingForward() throws Exception { // currently a warning, not an error storage.flushColumn("host".getBytes(MockBase.ASCII()), ID_FAMILY, TAGK); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @@ -546,7 +559,7 @@ public void fsckFIXTagkMissingForward() throws Exception { // currently a warning, not an error storage.flushColumn("host".getBytes(MockBase.ASCII()), ID_FAMILY, TAGK); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), true, false); + UID_TABLE, true, false); assertEquals(0, errors); assertNull(storage.getColumn(new byte [] {0, 0, 1}, NAME_FAMILY, TAGK)); } @@ -556,7 +569,7 @@ public void fsckTagvMissingForward() throws Exception { // currently a warning, not an error storage.flushColumn("web01".getBytes(MockBase.ASCII()), ID_FAMILY, TAGV); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @@ -565,7 +578,7 @@ public void fsckFIXTagvMissingForward() throws Exception { // currently a warning, not an error storage.flushColumn("web01".getBytes(MockBase.ASCII()), ID_FAMILY, TAGV); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), true, false); + UID_TABLE, true, false); assertEquals(0, errors); assertNull(storage.getColumn(new byte [] {0, 0, 1}, NAME_FAMILY, TAGV)); } @@ -581,73 +594,73 @@ public void fsckFIXTagvMissingForward() throws Exception { */ @Test public void fsckMetricsInconsistentReverse() throws Exception { - storage.addColumn(new byte[] {0, 0, 3}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 3}, NAME_FAMILY, METRICS, "foo".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(3L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(3L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(1, errors); } @Test public void fsckFIXMetricsInconsistentReverse() throws Exception { - storage.addColumn(new byte[] {0, 0, 3}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 3}, NAME_FAMILY, METRICS, "foo".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(3L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(3L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), true, false); + UID_TABLE, true, false); assertEquals(1, errors); assertNull(storage.getColumn(new byte [] {0, 0, 3}, NAME_FAMILY, METRICS)); errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @Test public void fsckTagkInconsistentReverse() throws Exception { - storage.addColumn(new byte[] {0, 0, 3}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 3}, NAME_FAMILY, TAGK, "host".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, TAGK, Bytes.fromLong(3L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, TAGK, Bytes.fromLong(3L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(1, errors); } @Test public void fsckFIXTagkInconsistentReverse() throws Exception { - storage.addColumn(new byte[] {0, 0, 3}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 3}, NAME_FAMILY, TAGK, "host".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, TAGK, Bytes.fromLong(3L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, TAGK, Bytes.fromLong(3L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), true, false); + UID_TABLE, true, false); assertEquals(1, errors); assertNull(storage.getColumn(new byte [] {0, 0, 3}, NAME_FAMILY, TAGK)); errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @Test public void fsckTagvInconsistentReverse() throws Exception { - storage.addColumn(new byte[] {0, 0, 3}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 3}, NAME_FAMILY, TAGV, "web01".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, TAGV, Bytes.fromLong(3L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, TAGV, Bytes.fromLong(3L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(1, errors); } @Test public void fsckFIXTagvInconsistentReverse() throws Exception { - storage.addColumn(new byte[] {0, 0, 3}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 3}, NAME_FAMILY, TAGV, "web01".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, TAGV, Bytes.fromLong(3L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, TAGV, Bytes.fromLong(3L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), true, false); + UID_TABLE, true, false); assertEquals(1, errors); assertNull(storage.getColumn(new byte [] {0, 0, 3}, NAME_FAMILY, TAGV)); errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @@ -664,91 +677,91 @@ public void fsckFIXTagvInconsistentReverse() throws Exception { */ @Test public void fsckMetricsDuplicateReverse() throws Exception { - storage.addColumn(new byte[] {0, 0, 3}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 3}, NAME_FAMILY, METRICS, "wtf".getBytes(MockBase.ASCII())); - storage.addColumn("wtf".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.addColumn(UID_TABLE, "wtf".getBytes(MockBase.ASCII()), ID_FAMILY, METRICS, new byte[] {0, 0, 4}); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(4L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(4L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(2, errors); } @Test public void fsckFIXMetricsDuplicateReverse() throws Exception { - storage.addColumn(new byte[] {0, 0, 3}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 3}, NAME_FAMILY, METRICS, "wtf".getBytes(MockBase.ASCII())); - storage.addColumn("wtf".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.addColumn(UID_TABLE, "wtf".getBytes(MockBase.ASCII()), ID_FAMILY, METRICS, new byte[] {0, 0, 4}); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(4L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(4L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), true, false); + UID_TABLE, true, false); assertEquals(2, errors); assertNull(storage.getColumn(new byte [] {0, 0, 3}, NAME_FAMILY, METRICS)); assertArrayEquals("wtf".getBytes(MockBase.ASCII()), - storage.getColumn(new byte [] {0, 0, 4}, NAME_FAMILY, METRICS)); + storage.getColumn(UID_TABLE, new byte [] {0, 0, 4}, NAME_FAMILY, METRICS)); errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @Test public void fsckTagkDuplicateReverse() throws Exception { - storage.addColumn(new byte[] {0, 0, 3}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 3}, NAME_FAMILY, TAGK, "wtf".getBytes(MockBase.ASCII())); - storage.addColumn("wtf".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.addColumn(UID_TABLE, "wtf".getBytes(MockBase.ASCII()), ID_FAMILY, TAGK, new byte[] {0, 0, 4}); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, TAGK, Bytes.fromLong(4L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, TAGK, Bytes.fromLong(4L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(2, errors); } @Test public void fsckFIXTagkDuplicateReverse() throws Exception { - storage.addColumn(new byte[] {0, 0, 3}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 3}, NAME_FAMILY, TAGK, "wtf".getBytes(MockBase.ASCII())); - storage.addColumn("wtf".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.addColumn(UID_TABLE, "wtf".getBytes(MockBase.ASCII()), ID_FAMILY, TAGK, new byte[] {0, 0, 4}); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, TAGK, Bytes.fromLong(4L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, TAGK, Bytes.fromLong(4L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), true, false); + UID_TABLE, true, false); assertEquals(2, errors); assertNull(storage.getColumn(new byte [] {0, 0, 3}, NAME_FAMILY, TAGK)); assertArrayEquals("wtf".getBytes(MockBase.ASCII()), - storage.getColumn(new byte [] {0, 0, 4}, NAME_FAMILY, TAGK)); + storage.getColumn(UID_TABLE, new byte [] {0, 0, 4}, NAME_FAMILY, TAGK)); errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @Test public void fsckTagvDuplicateReverse() throws Exception { - storage.addColumn(new byte[] {0, 0, 3}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 3}, NAME_FAMILY, TAGV, "wtf".getBytes(MockBase.ASCII())); - storage.addColumn("wtf".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.addColumn(UID_TABLE, "wtf".getBytes(MockBase.ASCII()), ID_FAMILY, TAGV, new byte[] {0, 0, 4}); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, TAGV, Bytes.fromLong(4L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, TAGV, Bytes.fromLong(4L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(2, errors); } @Test public void fsckFIXTagvDuplicateReverse() throws Exception { - storage.addColumn(new byte[] {0, 0, 3}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 3}, NAME_FAMILY, TAGV, "wtf".getBytes(MockBase.ASCII())); - storage.addColumn("wtf".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.addColumn(UID_TABLE, "wtf".getBytes(MockBase.ASCII()), ID_FAMILY, TAGV, new byte[] {0, 0, 4}); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, TAGV, Bytes.fromLong(4L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, TAGV, Bytes.fromLong(4L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), true, false); + UID_TABLE, true, false); assertEquals(2, errors); assertNull(storage.getColumn(new byte [] {0, 0, 3}, NAME_FAMILY, TAGV)); assertArrayEquals("wtf".getBytes(MockBase.ASCII()), - storage.getColumn(new byte [] {0, 0, 4}, NAME_FAMILY, TAGV)); + storage.getColumn(UID_TABLE, new byte [] {0, 0, 4}, NAME_FAMILY, TAGV)); errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @@ -771,36 +784,36 @@ public void fsckFIXTagvDuplicateReverse() throws Exception { */ @Test public void fsckMetricsInconsistentFwdAndDupeRev() throws Exception { - storage.addColumn(new byte[] {0, 0, 2}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 2}, NAME_FAMILY, METRICS, "wtf".getBytes(MockBase.ASCII())); - storage.addColumn("wtf".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.addColumn(UID_TABLE, "wtf".getBytes(MockBase.ASCII()), ID_FAMILY, METRICS, new byte[] {0, 0, 1}); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(3L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(3L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(5, errors); } @Test public void fsckFIXMetricsInconsistentFwdAndDupeRev() throws Exception { - storage.addColumn(new byte[] {0, 0, 2}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 2}, NAME_FAMILY, METRICS, "wtf".getBytes(MockBase.ASCII())); - storage.addColumn("wtf".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.addColumn(UID_TABLE, "wtf".getBytes(MockBase.ASCII()), ID_FAMILY, METRICS, new byte[] {0, 0, 1}); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(3L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(3L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), true, false); + UID_TABLE, true, false); assertEquals(4, errors); assertArrayEquals("fsck.foo.wtf".getBytes(MockBase.ASCII()), - storage.getColumn(new byte [] {0, 0, 1}, NAME_FAMILY, METRICS)); + storage.getColumn(UID_TABLE, new byte [] {0, 0, 1}, NAME_FAMILY, METRICS)); assertNull(storage.getColumn("foo".getBytes(MockBase.ASCII()), ID_FAMILY, METRICS)); assertNull(storage.getColumn("wtf".getBytes(MockBase.ASCII()), ID_FAMILY, METRICS)); assertArrayEquals("bar".getBytes(MockBase.ASCII()), - storage.getColumn(new byte [] {0, 0, 2}, NAME_FAMILY, METRICS)); + storage.getColumn(UID_TABLE, new byte [] {0, 0, 2}, NAME_FAMILY, METRICS)); errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @@ -822,40 +835,40 @@ public void fsckFIXMetricsInconsistentFwdAndDupeRev() throws Exception { */ @Test public void fsckMetricsInconsistentFwdAndInconsistentRev() throws Exception { - storage.addColumn(new byte[] {0, 0, 2}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 2}, NAME_FAMILY, METRICS, "wtf".getBytes(MockBase.ASCII())); - storage.addColumn("wtf".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.addColumn(UID_TABLE, "wtf".getBytes(MockBase.ASCII()), ID_FAMILY, METRICS, new byte[] {0, 0, 1}); - storage.addColumn(new byte[] {0, 0, 3}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 3}, NAME_FAMILY, METRICS, "foo".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(3L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(3L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(6, errors); } @Test public void fsckFIXMetricsInconsistentFwdAndInconsistentRev() throws Exception { - storage.addColumn(new byte[] {0, 0, 2}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 2}, NAME_FAMILY, METRICS, "wtf".getBytes(MockBase.ASCII())); - storage.addColumn("wtf".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.addColumn(UID_TABLE, "wtf".getBytes(MockBase.ASCII()), ID_FAMILY, METRICS, new byte[] {0, 0, 1}); - storage.addColumn(new byte[] {0, 0, 3}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 3}, NAME_FAMILY, METRICS, "foo".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(3L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(3L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), true, false); + UID_TABLE, true, false); assertEquals(4, errors); // diff than above since we remove some forwards early assertArrayEquals("fsck.foo.wtf".getBytes(MockBase.ASCII()), - storage.getColumn(new byte [] {0, 0, 1}, NAME_FAMILY, METRICS)); + storage.getColumn(UID_TABLE, new byte [] {0, 0, 1}, NAME_FAMILY, METRICS)); assertNull(storage.getColumn("foo".getBytes(MockBase.ASCII()), ID_FAMILY, METRICS)); assertNull(storage.getColumn("wtf".getBytes(MockBase.ASCII()), ID_FAMILY, METRICS)); assertArrayEquals("bar".getBytes(MockBase.ASCII()), - storage.getColumn(new byte [] {0, 0, 2}, NAME_FAMILY, METRICS)); + storage.getColumn(UID_TABLE, new byte [] {0, 0, 2}, NAME_FAMILY, METRICS)); errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(0, errors); } @@ -872,34 +885,34 @@ public void fsckFIXMetricsInconsistentFwdAndInconsistentRev() throws Exception { */ @Test public void fsckMetricsInconsistentFwdNoDupes() throws Exception { - storage.addColumn(new byte[] {0, 0, 2}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 2}, NAME_FAMILY, METRICS, "wtf".getBytes(MockBase.ASCII())); - storage.addColumn("wtf".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.addColumn(UID_TABLE, "wtf".getBytes(MockBase.ASCII()), ID_FAMILY, METRICS, new byte[] {0, 0, 3}); - storage.addColumn(new byte[] {0, 0, 3}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 3}, NAME_FAMILY, METRICS, "wtf".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(3L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(3L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), false, false); + UID_TABLE, false, false); assertEquals(3, errors); } @Test public void fsckFixMetricsInconsistentFwdNoDupes() throws Exception { - storage.addColumn(new byte[] {0, 0, 2}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 2}, NAME_FAMILY, METRICS, "wtf".getBytes(MockBase.ASCII())); - storage.addColumn("wtf".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.addColumn(UID_TABLE, "wtf".getBytes(MockBase.ASCII()), ID_FAMILY, METRICS, new byte[] {0, 0, 3}); - storage.addColumn(new byte[] {0, 0, 3}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 3}, NAME_FAMILY, METRICS, "wtf".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(3L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(3L)); int errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), true, false); + UID_TABLE, true, false); assertEquals(2, errors); assertArrayEquals("bar".getBytes(MockBase.ASCII()), - storage.getColumn(new byte [] {0, 0, 2}, NAME_FAMILY, METRICS)); + storage.getColumn(UID_TABLE, new byte [] {0, 0, 2}, NAME_FAMILY, METRICS)); errors = (Integer)fsck.invoke(null, client, - "tsdb".getBytes(MockBase.ASCII()), true, false); + UID_TABLE, true, false); assertEquals(0, errors); } @@ -908,37 +921,40 @@ public void fsckFixMetricsInconsistentFwdNoDupes() throws Exception { */ private void setupMockBase() { storage = new MockBase(tsdb, client, true, true, true, true); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, METRICS, Bytes.fromLong(2L)); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, TAGK, Bytes.fromLong(2L)); - storage.addColumn(new byte[] { 0 }, ID_FAMILY, TAGV, Bytes.fromLong(2L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, METRICS, + Bytes.fromLong(2L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, TAGK, + Bytes.fromLong(2L)); + storage.addColumn(UID_TABLE, new byte[] { 0 }, ID_FAMILY, TAGV, + Bytes.fromLong(2L)); // forward mappings - storage.addColumn("foo".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.addColumn(UID_TABLE, "foo".getBytes(MockBase.ASCII()), ID_FAMILY, METRICS, new byte[] {0, 0, 1}); - storage.addColumn("host".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.addColumn(UID_TABLE, "host".getBytes(MockBase.ASCII()), ID_FAMILY, TAGK, new byte[] {0, 0, 1}); - storage.addColumn("web01".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.addColumn(UID_TABLE, "web01".getBytes(MockBase.ASCII()), ID_FAMILY, TAGV, new byte[] {0, 0, 1}); - storage.addColumn("bar".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.addColumn(UID_TABLE, "bar".getBytes(MockBase.ASCII()), ID_FAMILY, METRICS, new byte[] {0, 0, 2}); - storage.addColumn("dc".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.addColumn(UID_TABLE, "dc".getBytes(MockBase.ASCII()), ID_FAMILY, TAGK, new byte[] {0, 0, 2}); - storage.addColumn("web02".getBytes(MockBase.ASCII()), ID_FAMILY, + storage.addColumn(UID_TABLE, "web02".getBytes(MockBase.ASCII()), ID_FAMILY, TAGV, new byte[] {0, 0, 2}); // reverse mappings - storage.addColumn(new byte[] {0, 0, 1}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 1}, NAME_FAMILY, METRICS, "foo".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] {0, 0, 1}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 1}, NAME_FAMILY, TAGK, "host".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] {0, 0, 1}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 1}, NAME_FAMILY, TAGV, "web01".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] {0, 0, 2}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 2}, NAME_FAMILY, METRICS, "bar".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] {0, 0, 2}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 2}, NAME_FAMILY, TAGK, "dc".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] {0, 0, 2}, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] {0, 0, 2}, NAME_FAMILY, TAGV, "web02".getBytes(MockBase.ASCII())); } diff --git a/test/tree/TestBranch.java b/test/tree/TestBranch.java index 22e2eec6e8..7c228df094 100644 --- a/test/tree/TestBranch.java +++ b/test/tree/TestBranch.java @@ -18,11 +18,12 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.anyString; import static org.powermock.api.mockito.PowerMockito.mock; import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -39,7 +40,6 @@ import org.hbase.async.Scanner; import org.junit.Test; import org.junit.runner.RunWith; -import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; @@ -51,7 +51,9 @@ @PrepareForTest({ TSDB.class, HBaseClient.class, GetRequest.class, PutRequest.class, KeyValue.class, Scanner.class, DeleteRequest.class }) public final class TestBranch { - private static byte[] NAME_FAMILY = "name".getBytes(MockBase.ASCII()); + private final static byte[] NAME_FAMILY = "name".getBytes(MockBase.ASCII()); + private final static byte[] TREE_TABLE = "tsdb-tree".getBytes(MockBase.ASCII()); + private final static byte[] UID_TABLE = "tsdb-uid".getBytes(MockBase.ASCII()); private MockBase storage; private Tree tree = TestTree.buildTestTree(); final static private Method toStorageJson; @@ -268,28 +270,28 @@ public void compileBranchIdInvalidId() { public void fetchBranch() throws Exception { setupStorage(); - storage.addColumn(new byte[] { 0, 0, 1 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "metrics".getBytes(MockBase.ASCII()), "sys.cpu.0".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "tagk".getBytes(MockBase.ASCII()), "host".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "tagv".getBytes(MockBase.ASCII()), "web01".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 2 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 2 }, NAME_FAMILY, "metrics".getBytes(MockBase.ASCII()), "sys.cpu.1".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 2 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 2 }, NAME_FAMILY, "tagk".getBytes(MockBase.ASCII()), "owner".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 2 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 2 }, NAME_FAMILY, "tagv".getBytes(MockBase.ASCII()), "ops".getBytes(MockBase.ASCII())); @@ -308,15 +310,15 @@ public void fetchBranch() throws Exception { public void fetchBranchNSU() throws Exception { setupStorage(); - storage.addColumn(new byte[] { 0, 0, 1 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "metrics".getBytes(MockBase.ASCII()), "sys.cpu.0".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "tagk".getBytes(MockBase.ASCII()), "host".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "tagv".getBytes(MockBase.ASCII()), "web01".getBytes(MockBase.ASCII())); @@ -363,11 +365,11 @@ public void storeBranch() throws Exception { setupStorage(); final Branch branch = buildTestBranch(tree); branch.storeBranch(storage.getTSDB(), tree, true); - assertEquals(3, storage.numRows()); - assertEquals(3, storage.numColumns(new byte[] { 0, 1 })); - final Branch parsed = JSON.parseToObject(storage.getColumn( - new byte[] { 0, 1 }, "branch".getBytes(MockBase.ASCII())), - Branch.class); + assertEquals(3, storage.numRows(TREE_TABLE)); + assertEquals(3, storage.numColumns(TREE_TABLE, new byte[] { 0, 1 })); + final Branch parsed = JSON.parseToObject(storage.getColumn(TREE_TABLE, + new byte[] { 0, 1 }, Tree.TREE_FAMILY(), + "branch".getBytes(MockBase.ASCII())), Branch.class); parsed.setTreeId(1); assertEquals("ROOT", parsed.getDisplayName()); } @@ -405,12 +407,12 @@ public void storeBranchExistingLeaf() throws Exception { qualifier, (byte[])LeaftoStorageJson.invoke(leaf)); branch.storeBranch(storage.getTSDB(), tree, true); - assertEquals(3, storage.numRows()); - assertEquals(3, storage.numColumns(new byte[] { 0, 1 })); + assertEquals(3, storage.numRows(TREE_TABLE)); + assertEquals(3, storage.numColumns(TREE_TABLE, new byte[] { 0, 1 })); assertNull(tree.getCollisions()); - final Branch parsed = JSON.parseToObject(storage.getColumn( - new byte[] { 0, 1 }, "branch".getBytes(MockBase.ASCII())), - Branch.class); + final Branch parsed = JSON.parseToObject(storage.getColumn(TREE_TABLE, + new byte[] { 0, 1 }, Tree.TREE_FAMILY(), + "branch".getBytes(MockBase.ASCII())), Branch.class); parsed.setTreeId(1); assertEquals("ROOT", parsed.getDisplayName()); } @@ -421,16 +423,16 @@ public void storeBranchCollision() throws Exception { final Branch branch = buildTestBranch(tree); Leaf leaf = new Leaf("Alarms", "0101"); byte[] qualifier = leaf.columnQualifier(); - storage.addColumn(branch.compileBranchId(), Tree.TREE_FAMILY(), + storage.addColumn(TREE_TABLE, branch.compileBranchId(), Tree.TREE_FAMILY(), qualifier, (byte[])LeaftoStorageJson.invoke(leaf)); branch.storeBranch(storage.getTSDB(), tree, true); - assertEquals(3, storage.numRows()); - assertEquals(3, storage.numColumns(new byte[] { 0, 1 })); + assertEquals(3, storage.numRows(TREE_TABLE)); + assertEquals(3, storage.numColumns(TREE_TABLE, new byte[] { 0, 1 })); assertEquals(1, tree.getCollisions().size()); - final Branch parsed = JSON.parseToObject(storage.getColumn( - new byte[] { 0, 1 }, "branch".getBytes(MockBase.ASCII())), - Branch.class); + final Branch parsed = JSON.parseToObject(storage.getColumn(TREE_TABLE, + new byte[] { 0, 1 }, Tree.TREE_FAMILY(), + "branch".getBytes(MockBase.ASCII())), Branch.class); parsed.setTreeId(1); assertEquals("ROOT", parsed.getDisplayName()); } @@ -565,10 +567,11 @@ public static Branch buildTestBranch(final Tree tree) { private void setupStorage() throws Exception { final HBaseClient client = mock(HBaseClient.class); final Config config = new Config(false); - PowerMockito.whenNew(HBaseClient.class) - .withArguments(anyString(), anyString()).thenReturn(client); - - storage = new MockBase(new TSDB(config), client, true, true, true, true); + storage = new MockBase(new TSDB(client, config), + client, true, true, true, true); + final List<byte[]> families = new ArrayList<byte[]>(); + families.add(Tree.TREE_FAMILY()); + storage.addTable(TREE_TABLE, families); Branch branch = new Branch(1); TreeMap<Integer, String> path = new TreeMap<Integer, String>(); @@ -577,18 +580,18 @@ private void setupStorage() throws Exception { path.put(2, "cpu"); branch.prependParentPath(path); branch.setDisplayName("cpu"); - storage.addColumn(branch.compileBranchId(), Tree.TREE_FAMILY(), + storage.addColumn(TREE_TABLE, branch.compileBranchId(), Tree.TREE_FAMILY(), "branch".getBytes(MockBase.ASCII()), (byte[])toStorageJson.invoke(branch)); Leaf leaf = new Leaf("user", "000001000001000001"); byte[] qualifier = leaf.columnQualifier(); - storage.addColumn(branch.compileBranchId(), Tree.TREE_FAMILY(), + storage.addColumn(TREE_TABLE, branch.compileBranchId(), Tree.TREE_FAMILY(), qualifier, (byte[])LeaftoStorageJson.invoke(leaf)); leaf = new Leaf("nice", "000002000002000002"); qualifier = leaf.columnQualifier(); - storage.addColumn(branch.compileBranchId(), Tree.TREE_FAMILY(), + storage.addColumn(TREE_TABLE, branch.compileBranchId(), Tree.TREE_FAMILY(), qualifier, (byte[])LeaftoStorageJson.invoke(leaf)); // child branch @@ -596,13 +599,13 @@ private void setupStorage() throws Exception { path.put(3, "mboard"); branch.prependParentPath(path); branch.setDisplayName("mboard"); - storage.addColumn(branch.compileBranchId(), Tree.TREE_FAMILY(), + storage.addColumn(TREE_TABLE, branch.compileBranchId(), Tree.TREE_FAMILY(), "branch".getBytes(MockBase.ASCII()), (byte[])toStorageJson.invoke(branch)); leaf = new Leaf("Asus", "000003000003000003"); qualifier = leaf.columnQualifier(); - storage.addColumn(branch.compileBranchId(), Tree.TREE_FAMILY(), + storage.addColumn(TREE_TABLE, branch.compileBranchId(), Tree.TREE_FAMILY(), qualifier, (byte[])LeaftoStorageJson.invoke(leaf)); } } diff --git a/test/tree/TestLeaf.java b/test/tree/TestLeaf.java index 22f8d80579..cb245a7d6e 100644 --- a/test/tree/TestLeaf.java +++ b/test/tree/TestLeaf.java @@ -16,10 +16,12 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.when; import static org.powermock.api.mockito.PowerMockito.mock; +import java.util.ArrayList; +import java.util.List; + import net.opentsdb.core.TSDB; import net.opentsdb.storage.MockBase; import net.opentsdb.uid.NoSuchUniqueId; @@ -35,7 +37,6 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; @@ -50,7 +51,9 @@ GetRequest.class, PutRequest.class, DeleteRequest.class, KeyValue.class, Scanner.class }) public final class TestLeaf { - private static byte[] NAME_FAMILY = "name".getBytes(MockBase.ASCII()); + private final static byte[] NAME_FAMILY = "name".getBytes(MockBase.ASCII()); + private final static byte[] TREE_TABLE = "tsdb-tree".getBytes(MockBase.ASCII()); + private final static byte[] UID_TABLE = "tsdb-uid".getBytes(MockBase.ASCII()); private TSDB tsdb; private HBaseClient client = mock(HBaseClient.class); private MockBase storage; @@ -58,23 +61,24 @@ public final class TestLeaf { @Before public void before() throws Exception { final Config config = new Config(false); - PowerMockito.whenNew(HBaseClient.class) - .withArguments(anyString(), anyString()).thenReturn(client); - tsdb = new TSDB(config); + tsdb = new TSDB(client, config); storage = new MockBase(tsdb, client, true, true, true, true); + final List<byte[]> families = new ArrayList<byte[]>(); + families.add(Tree.TREE_FAMILY()); + storage.addTable(TREE_TABLE, families); - storage.addColumn(new byte[] { 0, 0, 1 }, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "metrics".getBytes(MockBase.ASCII()), "sys.cpu.0".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1 }, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "tagk".getBytes(MockBase.ASCII()), "host".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1 }, NAME_FAMILY, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "tagv".getBytes(MockBase.ASCII()), "web01".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 1 }, Tree.TREE_FAMILY(), + storage.addColumn(TREE_TABLE, new byte[] { 0, 1 }, Tree.TREE_FAMILY(), new Leaf("0", "000001000001000001").columnQualifier(), ("{\"displayName\":\"0\",\"tsuid\":\"000001000001000001\"}") .getBytes(MockBase.ASCII())); @@ -154,7 +158,7 @@ public void storeLeaf() throws Exception { final Tree tree = TestTree.buildTestTree(); assertTrue(leaf.storeLeaf(tsdb, new byte[] { 0, 1 }, tree) .joinUninterruptibly()); - assertEquals(2, storage.numColumns(new byte[] { 0, 1 })); + assertEquals(2, storage.numColumns(TREE_TABLE, new byte[] { 0, 1 })); } @Test @@ -163,7 +167,7 @@ public void storeLeafExistingSame() throws Exception { final Tree tree = TestTree.buildTestTree(); assertTrue(leaf.storeLeaf(tsdb, new byte[] { 0, 1 }, tree) .joinUninterruptibly()); - assertEquals(1, storage.numColumns(new byte[] { 0, 1 })); + assertEquals(1, storage.numColumns(TREE_TABLE, new byte[] { 0, 1 })); } @Test @@ -172,7 +176,7 @@ public void storeLeafCollision() throws Exception { final Tree tree = TestTree.buildTestTree(); assertFalse(leaf.storeLeaf(tsdb, new byte[] { 0, 1 }, tree) .joinUninterruptibly()); - assertEquals(1, storage.numColumns(new byte[] { 0, 1 })); + assertEquals(1, storage.numColumns(TREE_TABLE, new byte[] { 0, 1 })); assertEquals(1, tree.getCollisions().size()); } diff --git a/test/tree/TestTree.java b/test/tree/TestTree.java index e3d10e6334..048a713945 100644 --- a/test/tree/TestTree.java +++ b/test/tree/TestTree.java @@ -18,7 +18,6 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.anyString; import static org.powermock.api.mockito.PowerMockito.mock; import java.lang.reflect.Field; @@ -46,7 +45,6 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; @@ -58,6 +56,7 @@ @PrepareForTest({TSDB.class, HBaseClient.class, GetRequest.class, PutRequest.class, KeyValue.class, Scanner.class, DeleteRequest.class}) public final class TestTree { + private final static byte[] TREE_TABLE = "tsdb-tree".getBytes(); private MockBase storage; private TSDB tsdb; private HBaseClient client = mock(HBaseClient.class); @@ -76,8 +75,6 @@ public final class TestTree { public void before() throws Exception { final Config config = new Config(false); config.overrideConfig("tsd.storage.enable_compaction", "false"); - PowerMockito.whenNew(HBaseClient.class) - .withArguments(anyString(), anyString()).thenReturn(client); tsdb = new TSDB(client, config); } @@ -256,8 +253,8 @@ public void flushCollisions() throws Exception { tree.addCollision("010203", "AABBCCDD"); assertNotNull(tree.flushCollisions(storage.getTSDB()) .joinUninterruptibly()); - assertEquals(4, storage.numRows()); - assertEquals(3, storage.numColumns(new byte[] { 0, 1, 1 })); + assertEquals(4, storage.numRows(TREE_TABLE)); + assertEquals(3, storage.numColumns(TREE_TABLE, new byte[] { 0, 1, 1 })); } @Test @@ -267,8 +264,8 @@ public void flushCollisionsDisabled() throws Exception { tree.addCollision("010203", "AABBCCDD"); assertNotNull(tree.flushCollisions(storage.getTSDB()) .joinUninterruptibly()); - assertEquals(4, storage.numRows()); - assertEquals(2, storage.numColumns(new byte[] { 0, 1, 1 })); + assertEquals(4, storage.numRows(TREE_TABLE)); + assertEquals(2, storage.numColumns(TREE_TABLE, new byte[] { 0, 1, 1 })); } @Test @@ -278,8 +275,8 @@ public void flushCollisionsWCollisionExisting() throws Exception { tree.addCollision("010101", "AAAAAA"); assertNotNull(tree.flushCollisions(storage.getTSDB()) .joinUninterruptibly()); - assertEquals(4, storage.numRows()); - assertEquals(2, storage.numColumns(new byte[] { 0, 1, 1 })); + assertEquals(4, storage.numRows(TREE_TABLE)); + assertEquals(2, storage.numColumns(TREE_TABLE, new byte[] { 0, 1, 1 })); } @Test @@ -290,8 +287,8 @@ public void flushNotMatched() throws Exception { tree.addNotMatched("010203", "Failed rule 2:2"); assertNotNull(tree.flushNotMatched(storage.getTSDB()) .joinUninterruptibly()); - assertEquals(4, storage.numRows()); - assertEquals(3, storage.numColumns(new byte[] { 0, 1, 2 })); + assertEquals(4, storage.numRows(TREE_TABLE)); + assertEquals(3, storage.numColumns(TREE_TABLE, new byte[] { 0, 1, 2 })); } @Test @@ -301,8 +298,8 @@ public void flushNotMatchedDisabled() throws Exception { tree.addNotMatched("010203", "Failed rule 2:2"); assertNotNull(tree.flushNotMatched(storage.getTSDB()) .joinUninterruptibly()); - assertEquals(4, storage.numRows()); - assertEquals(2, storage.numColumns(new byte[] { 0, 1, 2 })); + assertEquals(4, storage.numRows(TREE_TABLE)); + assertEquals(2, storage.numColumns(TREE_TABLE, new byte[] { 0, 1, 2 })); } @Test @@ -312,8 +309,8 @@ public void flushNotMatchedWNotMatchedExisting() throws Exception { tree.addNotMatched("010101", "Failed rule 4:4"); assertNotNull(tree.flushNotMatched(storage.getTSDB()) .joinUninterruptibly()); - assertEquals(4, storage.numRows()); - assertEquals(2, storage.numColumns(new byte[] { 0, 1, 2 })); + assertEquals(4, storage.numRows(TREE_TABLE)); + assertEquals(2, storage.numColumns(TREE_TABLE, new byte[] { 0, 1, 2 })); } @Test @@ -353,8 +350,8 @@ public void createNewTree() throws Exception { final int tree_id = tree.createNewTree(storage.getTSDB()) .joinUninterruptibly(); assertEquals(3, tree_id); - assertEquals(5, storage.numRows()); - assertEquals(1, storage.numColumns(new byte[] { 0, 3 })); + assertEquals(5, storage.numRows(TREE_TABLE)); + assertEquals(1, storage.numColumns(TREE_TABLE, new byte[] { 0, 3 })); } @Test @@ -366,8 +363,8 @@ public void createNewFirstTree() throws Exception { final int tree_id = tree.createNewTree(storage.getTSDB()) .joinUninterruptibly(); assertEquals(1, tree_id); - assertEquals(1, storage.numRows()); - assertEquals(1, storage.numColumns(new byte[] { 0, 1 })); + assertEquals(1, storage.numRows(TREE_TABLE)); + assertEquals(1, storage.numColumns(TREE_TABLE, new byte[] { 0, 1 })); } @Test (expected = IllegalArgumentException.class) @@ -452,7 +449,7 @@ public void fetchAllCollisions() throws Exception { @Test public void fetchAllCollisionsNone() throws Exception { setupStorage(true, true); - storage.flushRow(new byte[] { 0, 1, 1 }); + storage.flushRow(TREE_TABLE, new byte[] { 0, 1, 1 }); Map<String, String> collisions = Tree.fetchCollisions(storage.getTSDB(), 1, null).joinUninterruptibly(); assertNotNull(collisions); @@ -510,7 +507,7 @@ public void fetchAllNotMatched() throws Exception { @Test public void fetchAllNotMatchedNone() throws Exception { setupStorage(true, true); - storage.flushRow(new byte[] { 0, 1, 2 }); + storage.flushRow(TREE_TABLE, new byte[] { 0, 1, 2 }); Map<String, String> not_matched = Tree.fetchNotMatched(storage.getTSDB(), 1, null).joinUninterruptibly(); assertNotNull(not_matched); @@ -557,14 +554,14 @@ public void fetchNotMatchedID655536() throws Exception { public void deleteTree() throws Exception { setupStorage(true, true); - assertEquals(4, storage.numRows()); + assertEquals(4, storage.numRows(TREE_TABLE)); assertNotNull(Tree.deleteTree(storage.getTSDB(), 1, true) .joinUninterruptibly()); byte[] remainingKey = new byte[] {0, 2}; - assertEquals(1, storage.numRows()); - assertNotNull(storage.getColumn( - remainingKey, "tree".getBytes(MockBase.ASCII()))); + assertEquals(1, storage.numRows(TREE_TABLE)); + assertNotNull(storage.getColumn(TREE_TABLE, remainingKey, Tree.TREE_FAMILY(), + "tree".getBytes(MockBase.ASCII()))); } @Test @@ -743,24 +740,28 @@ public static Tree buildTestTree() { private void setupStorage(final boolean default_get, final boolean default_put) throws Exception { storage = new MockBase(tsdb, client, default_get, default_put, true, true); + final List<byte[]> families = new ArrayList<byte[]>(1); + families.add(Tree.TREE_FAMILY()); + storage.addTable(TREE_TABLE, families); byte[] key = new byte[] { 0, 1 }; // set pre-test values - storage.addColumn(key, "tree".getBytes(MockBase.ASCII()), + storage.addColumn(TREE_TABLE, key, Tree.TREE_FAMILY(), + "tree".getBytes(MockBase.ASCII()), (byte[])TreetoStorageJson.invoke(buildTestTree())); TreeRule rule = new TreeRule(1); rule.setField("host"); rule.setType(TreeRuleType.TAGK); - storage.addColumn(key, "tree_rule:0:0".getBytes(MockBase.ASCII()), - JSON.serializeToBytes(rule)); + storage.addColumn(TREE_TABLE, key, Tree.TREE_FAMILY(), + "tree_rule:0:0".getBytes(MockBase.ASCII()), JSON.serializeToBytes(rule)); rule = new TreeRule(1); rule.setField(""); rule.setLevel(1); rule.setType(TreeRuleType.METRIC); - storage.addColumn(key, "tree_rule:1:0".getBytes(MockBase.ASCII()), - JSON.serializeToBytes(rule)); + storage.addColumn(TREE_TABLE, key, Tree.TREE_FAMILY(), + "tree_rule:1:0".getBytes(MockBase.ASCII()), JSON.serializeToBytes(rule)); Branch root = new Branch(1); root.setDisplayName("ROOT"); @@ -770,8 +771,8 @@ private void setupStorage(final boolean default_get, // TODO - static Method branch_json = Branch.class.getDeclaredMethod("toStorageJson"); branch_json.setAccessible(true); - storage.addColumn(key, "branch".getBytes(MockBase.ASCII()), - (byte[])branch_json.invoke(root)); + storage.addColumn(TREE_TABLE, key, Tree.TREE_FAMILY(), + "branch".getBytes(MockBase.ASCII()), (byte[])branch_json.invoke(root)); // tree 2 key = new byte[] { 0, 2 }; @@ -780,29 +781,30 @@ private void setupStorage(final boolean default_get, tree2.setTreeId(2); tree2.setName("2nd Tree"); tree2.setDescription("Other Tree"); - storage.addColumn(key, "tree".getBytes(MockBase.ASCII()), + storage.addColumn(TREE_TABLE, key, Tree.TREE_FAMILY(), + "tree".getBytes(MockBase.ASCII()), (byte[])TreetoStorageJson.invoke(tree2)); rule = new TreeRule(2); rule.setField("host"); rule.setType(TreeRuleType.TAGK); - storage.addColumn(key, "tree_rule:0:0".getBytes(MockBase.ASCII()), - JSON.serializeToBytes(rule)); + storage.addColumn(TREE_TABLE, key, Tree.TREE_FAMILY(), + "tree_rule:0:0".getBytes(MockBase.ASCII()), JSON.serializeToBytes(rule)); rule = new TreeRule(2); rule.setField(""); rule.setLevel(1); rule.setType(TreeRuleType.METRIC); - storage.addColumn(key, "tree_rule:1:0".getBytes(MockBase.ASCII()), - JSON.serializeToBytes(rule)); + storage.addColumn(TREE_TABLE, key, Tree.TREE_FAMILY(), + "tree_rule:1:0".getBytes(MockBase.ASCII()), JSON.serializeToBytes(rule)); root = new Branch(2); root.setDisplayName("ROOT"); root_path = new TreeMap<Integer, String>(); root_path.put(0, "ROOT"); root.prependParentPath(root_path); - storage.addColumn(key, "branch".getBytes(MockBase.ASCII()), - (byte[])branch_json.invoke(root)); + storage.addColumn(TREE_TABLE, key, Tree.TREE_FAMILY(), + "branch".getBytes(MockBase.ASCII()), (byte[])branch_json.invoke(root)); // sprinkle in some collisions and no matches for fun // collisions @@ -815,7 +817,8 @@ private void setupStorage(final boolean default_get, byte[] tsuid_bytes = UniqueId.stringToUid(tsuid); System.arraycopy(tsuid_bytes, 0, qualifier, Tree.COLLISION_PREFIX().length, tsuid_bytes.length); - storage.addColumn(key, qualifier, "AAAAAA".getBytes(MockBase.ASCII())); + storage.addColumn(TREE_TABLE, key, Tree.TREE_FAMILY(), qualifier, + "AAAAAA".getBytes(MockBase.ASCII())); tsuid = "020202"; qualifier = new byte[Tree.COLLISION_PREFIX().length + @@ -825,7 +828,8 @@ private void setupStorage(final boolean default_get, tsuid_bytes = UniqueId.stringToUid(tsuid); System.arraycopy(tsuid_bytes, 0, qualifier, Tree.COLLISION_PREFIX().length, tsuid_bytes.length); - storage.addColumn(key, qualifier, "BBBBBB".getBytes(MockBase.ASCII())); + storage.addColumn(TREE_TABLE, key, Tree.TREE_FAMILY(), qualifier, + "BBBBBB".getBytes(MockBase.ASCII())); // not matched key = new byte[] { 0, 1, 2 }; @@ -837,8 +841,8 @@ private void setupStorage(final boolean default_get, tsuid_bytes = UniqueId.stringToUid(tsuid); System.arraycopy(tsuid_bytes, 0, qualifier, Tree.NOT_MATCHED_PREFIX().length, tsuid_bytes.length); - storage.addColumn(key, qualifier, "Failed rule 0:0" - .getBytes(MockBase.ASCII())); + storage.addColumn(TREE_TABLE, key, Tree.TREE_FAMILY(), qualifier, + "Failed rule 0:0".getBytes(MockBase.ASCII())); tsuid = "020202"; qualifier = new byte[Tree.NOT_MATCHED_PREFIX().length + @@ -848,8 +852,7 @@ private void setupStorage(final boolean default_get, tsuid_bytes = UniqueId.stringToUid(tsuid); System.arraycopy(tsuid_bytes, 0, qualifier, Tree.NOT_MATCHED_PREFIX().length, tsuid_bytes.length); - storage.addColumn(key, qualifier, "Failed rule 1:1" - .getBytes(MockBase.ASCII())); - + storage.addColumn(TREE_TABLE, key, Tree.TREE_FAMILY(), qualifier, + "Failed rule 1:1".getBytes(MockBase.ASCII())); } } diff --git a/test/tree/TestTreeBuilder.java b/test/tree/TestTreeBuilder.java index e657c3ac18..edb5b111bb 100644 --- a/test/tree/TestTreeBuilder.java +++ b/test/tree/TestTreeBuilder.java @@ -16,13 +16,13 @@ import static org.junit.Assert.assertNotNull; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyInt; -import static org.mockito.Matchers.anyString; import static org.powermock.api.mockito.PowerMockito.mock; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.TreeMap; import net.opentsdb.core.TSDB; @@ -59,6 +59,8 @@ HBaseClient.class, Scanner.class, GetRequest.class, KeyValue.class, DeleteRequest.class, Tree.class}) public final class TestTreeBuilder { + private final static byte[] TREE_TABLE = "tsdb-tree".getBytes(MockBase.ASCII()); + private final static byte[] BRANCH = "branch".getBytes(MockBase.ASCII()); private TSDB tsdb; private HBaseClient client = mock(HBaseClient.class); private MockBase storage; @@ -91,11 +93,13 @@ public final class TestTreeBuilder { @Before public void before() throws Exception { final Config config = new Config(false); - PowerMockito.whenNew(HBaseClient.class) - .withArguments(anyString(), anyString()).thenReturn(client); - tsdb = new TSDB(config); + tsdb = new TSDB(client, config); storage = new MockBase(tsdb, client, true, true, true, true); + final List<byte[]> families = new ArrayList<byte[]>(); + families.add(Tree.TREE_FAMILY()); + storage.addTable(TREE_TABLE, families); + treebuilder = new TreeBuilder(storage.getTSDB(), tree); PowerMockito.spy(Tree.class); PowerMockito.doReturn(Deferred.fromResult(tree)).when(Tree.class, @@ -123,25 +127,25 @@ public void before() throws Exception { root.setDisplayName("ROOT"); root_path.put(0, "ROOT"); root.prependParentPath(root_path); - storage.addColumn(root.compileBranchId(), - "branch".getBytes(MockBase.ASCII()), - (byte[])toStorageJson.invoke(root)); + storage.addColumn(TREE_TABLE, root.compileBranchId(), Tree.TREE_FAMILY(), + BRANCH, (byte[])toStorageJson.invoke(root)); } @Test public void processTimeseriesMetaDefaults() throws Exception { treebuilder.processTimeseriesMeta(meta, false).joinUninterruptibly(); - assertEquals(7, storage.numRows()); - assertEquals(2, storage.numColumns(Branch.stringToId( + assertEquals(7, storage.numRows(TREE_TABLE)); + assertEquals(2, storage.numColumns(TREE_TABLE, Branch.stringToId( "00010001A2460001CB54247F72020001BECD000181A800000030"))); final Branch branch = JSON.parseToObject( - storage.getColumn(Branch.stringToId( + storage.getColumn(TREE_TABLE, Branch.stringToId( "00010001A2460001CB54247F72020001BECD000181A800000030"), - "branch".getBytes(MockBase.ASCII())), Branch.class); + Tree.TREE_FAMILY(), + BRANCH), Branch.class); assertNotNull(branch); assertEquals("0", branch.getDisplayName()); - final Leaf leaf = JSON.parseToObject(storage.getColumn(Branch.stringToId( - "00010001A2460001CB54247F72020001BECD000181A800000030"), + final Leaf leaf = JSON.parseToObject(storage.getColumn(TREE_TABLE, Branch.stringToId( + "00010001A2460001CB54247F72020001BECD000181A800000030"), Tree.TREE_FAMILY(), new Leaf("user", "").columnQualifier()), Leaf.class); assertNotNull(leaf); assertEquals("user", leaf.getDisplayName()); @@ -151,8 +155,8 @@ public void processTimeseriesMetaDefaults() throws Exception { public void processTimeseriesMetaNewRoot() throws Exception { storage.flushStorage(); treebuilder.processTimeseriesMeta(meta, false).joinUninterruptibly(); - assertEquals(7, storage.numRows()); - assertEquals(1, storage.numColumns(new byte[] { 0, 1 })); + assertEquals(7, storage.numRows(TREE_TABLE)); + assertEquals(1, storage.numColumns(TREE_TABLE, new byte[] { 0, 1 })); } @Test @@ -194,8 +198,8 @@ public void processTimeseriesMetaMiddleNonMatchedRules() throws Exception { tree.addRule(rule); treebuilder.processTimeseriesMeta(meta, false).joinUninterruptibly(); - assertEquals(5, storage.numRows()); - assertEquals(2, storage.numColumns( + assertEquals(5, storage.numRows(TREE_TABLE)); + assertEquals(2, storage.numColumns(TREE_TABLE, Branch.stringToId("0001247F72020001BECD000181A800000030"))); } @@ -239,8 +243,8 @@ public void processTimeseriesMetaEndNonMatchedRules() throws Exception { treebuilder.setTree(tree); treebuilder.processTimeseriesMeta(meta, false).joinUninterruptibly(); - assertEquals(7, storage.numRows()); - assertEquals(2, storage.numColumns( + assertEquals(7, storage.numRows(TREE_TABLE)); + assertEquals(2, storage.numColumns(TREE_TABLE, Branch.stringToId( "00010001A2460001CB54247F72020001BECD000181A800000030"))); } @@ -280,8 +284,8 @@ public void processTimeseriesMetaNullMetaOddNumTags() throws Exception { tags_field.set(meta, tags); tags_field.setAccessible(false); treebuilder.processTimeseriesMeta(meta, false).joinUninterruptibly(); - assertEquals(5, storage.numRows()); - assertEquals(2, storage.numColumns( + assertEquals(5, storage.numRows(TREE_TABLE)); + assertEquals(2, storage.numColumns(TREE_TABLE, Branch.stringToId( "00010036EBCB0001BECD000181A800000030"))); } @@ -289,15 +293,15 @@ public void processTimeseriesMetaNullMetaOddNumTags() throws Exception { @Test public void processTimeseriesMetaTesting() throws Exception { treebuilder.processTimeseriesMeta(meta, true).joinUninterruptibly(); - assertEquals(1, storage.numRows()); + assertEquals(1, storage.numRows(TREE_TABLE)); } @Test public void processTimeseriesMetaStrict() throws Exception { tree.setStrictMatch(true); treebuilder.processTimeseriesMeta(meta, false).joinUninterruptibly(); - assertEquals(7, storage.numRows()); - assertEquals(2, storage.numColumns( + assertEquals(7, storage.numRows(TREE_TABLE)); + assertEquals(2, storage.numColumns(TREE_TABLE, Branch.stringToId( "00010001A2460001CB54247F72020001BECD000181A800000030"))); } @@ -310,15 +314,15 @@ public void processTimeseriesMetaStrictNoMatch() throws Exception { name.setAccessible(false); tree.setStrictMatch(true); treebuilder.processTimeseriesMeta(meta, false).joinUninterruptibly(); - assertEquals(1, storage.numRows()); + assertEquals(1, storage.numRows(TREE_TABLE)); } @Test public void processTimeseriesMetaNoSplit() throws Exception { tree.getRules().get(3).get(0).setSeparator(""); treebuilder.processTimeseriesMeta(meta, false).joinUninterruptibly(); - assertEquals(5, storage.numRows()); - assertEquals(2, storage.numColumns( + assertEquals(5, storage.numRows(TREE_TABLE)); + assertEquals(2, storage.numColumns(TREE_TABLE, Branch.stringToId("00010001A2460001CB54247F7202CBBF5B09"))); } @@ -326,8 +330,8 @@ public void processTimeseriesMetaNoSplit() throws Exception { public void processTimeseriesMetBadSeparator() throws Exception { tree.getRules().get(3).get(0).setSeparator("."); treebuilder.processTimeseriesMeta(meta, false).joinUninterruptibly(); - assertEquals(4, storage.numRows()); - assertEquals(2, storage.numColumns( + assertEquals(4, storage.numRows(TREE_TABLE)); + assertEquals(2, storage.numColumns(TREE_TABLE, Branch.stringToId("00010001A2460001CB54247F7202"))); } @@ -335,8 +339,8 @@ public void processTimeseriesMetBadSeparator() throws Exception { public void processTimeseriesMetaInvalidRegexIdx() throws Exception { tree.getRules().get(1).get(1).setRegexGroupIdx(42); treebuilder.processTimeseriesMeta(meta, false).joinUninterruptibly(); - assertEquals(6, storage.numRows()); - assertEquals(2, storage.numColumns( + assertEquals(6, storage.numRows(TREE_TABLE)); + assertEquals(2, storage.numColumns(TREE_TABLE, Branch.stringToId("00010001A246247F72020001BECD000181A800000030"))); } @@ -355,8 +359,8 @@ public void processTimeseriesMetaMetricCustom() throws Exception { tree.addRule(rule); treebuilder.processTimeseriesMeta(meta, false).joinUninterruptibly(); - assertEquals(7, storage.numRows()); - assertEquals(2, storage.numColumns( + assertEquals(7, storage.numRows(TREE_TABLE)); + assertEquals(2, storage.numColumns(TREE_TABLE, Branch.stringToId( "0001AE805CA50001CB54247F72020001BECD000181A800000030"))); } @@ -393,8 +397,8 @@ public void processTimeseriesMetaMetricCustomEmptyValue() throws Exception { tree.addRule(rule); treebuilder.processTimeseriesMeta(meta, false).joinUninterruptibly(); - assertEquals(7, storage.numRows()); - assertEquals(2, storage.numColumns( + assertEquals(7, storage.numRows(TREE_TABLE)); + assertEquals(2, storage.numColumns(TREE_TABLE, Branch.stringToId( "00010001A2460001CB54247F72020001BECD000181A800000030"))); } @@ -415,8 +419,8 @@ public void processTimeseriesMetaTagkCustom() throws Exception { tree.addRule(rule); treebuilder.processTimeseriesMeta(meta, false).joinUninterruptibly(); - assertEquals(7, storage.numRows()); - assertEquals(2, storage.numColumns( + assertEquals(7, storage.numRows(TREE_TABLE)); + assertEquals(2, storage.numColumns(TREE_TABLE, Branch.stringToId( "0001AE805CA50001CB54247F72020001BECD000181A800000030"))); } @@ -455,8 +459,8 @@ public void processTimeseriesMetaTagkCustomEmptyValue() throws Exception { tree.addRule(rule); treebuilder.processTimeseriesMeta(meta, false).joinUninterruptibly(); - assertEquals(7, storage.numRows()); - assertEquals(2, storage.numColumns( + assertEquals(7, storage.numRows(TREE_TABLE)); + assertEquals(2, storage.numColumns(TREE_TABLE, Branch.stringToId( "00010001A2460001CB54247F72020001BECD000181A800000030"))); } @@ -477,8 +481,8 @@ public void processTimeseriesMetaTagkCustomNoField() throws Exception { tree.addRule(rule); treebuilder.processTimeseriesMeta(meta, false).joinUninterruptibly(); - assertEquals(7, storage.numRows()); - assertEquals(2, storage.numColumns( + assertEquals(7, storage.numRows(TREE_TABLE)); + assertEquals(2, storage.numColumns(TREE_TABLE, Branch.stringToId( "00010001A2460001CB54247F72020001BECD000181A800000030"))); } @@ -499,8 +503,8 @@ public void processTimeseriesMetaTagvCustom() throws Exception { tree.addRule(rule); treebuilder.processTimeseriesMeta(meta, false).joinUninterruptibly(); - assertEquals(7, storage.numRows()); - assertEquals(2, storage.numColumns( + assertEquals(7, storage.numRows(TREE_TABLE)); + assertEquals(2, storage.numColumns(TREE_TABLE, Branch.stringToId( "0001AE805CA50001CB54247F72020001BECD000181A800000030"))); } @@ -539,8 +543,8 @@ public void processTimeseriesMetaTagvCustomEmptyValue() throws Exception { tree.addRule(rule); treebuilder.processTimeseriesMeta(meta, false).joinUninterruptibly(); - assertEquals(7, storage.numRows()); - assertEquals(2, storage.numColumns( + assertEquals(7, storage.numRows(TREE_TABLE)); + assertEquals(2, storage.numColumns(TREE_TABLE, Branch.stringToId( "00010001A2460001CB54247F72020001BECD000181A800000030"))); } @@ -561,8 +565,8 @@ public void processTimeseriesMetaTagvCustomNoField() throws Exception { tree.addRule(rule); treebuilder.processTimeseriesMeta(meta, false).joinUninterruptibly(); - assertEquals(7, storage.numRows()); - assertEquals(2, storage.numColumns( + assertEquals(7, storage.numRows(TREE_TABLE)); + assertEquals(2, storage.numColumns(TREE_TABLE, Branch.stringToId( "00010001A2460001CB54247F72020001BECD000181A800000030"))); } @@ -571,10 +575,10 @@ public void processTimeseriesMetaTagvCustomNoField() throws Exception { public void processTimeseriesMetaFormatOvalue() throws Exception { tree.getRules().get(1).get(1).setDisplayFormat("OV: {ovalue}"); treebuilder.processTimeseriesMeta(meta, false).joinUninterruptibly(); - assertEquals(7, storage.numRows()); + assertEquals(7, storage.numRows(TREE_TABLE)); final Branch branch = JSON.parseToObject( - storage.getColumn(Branch.stringToId("00010001A24637E140D5"), - "branch".getBytes(MockBase.ASCII())), Branch.class); + storage.getColumn(TREE_TABLE, Branch.stringToId("00010001A24637E140D5"), + Tree.TREE_FAMILY(), BRANCH), Branch.class); assertEquals("OV: web-01.lga.mysite.com", branch.getDisplayName()); } @@ -582,10 +586,10 @@ public void processTimeseriesMetaFormatOvalue() throws Exception { public void processTimeseriesMetaFormatValue() throws Exception { tree.getRules().get(1).get(1).setDisplayFormat("V: {value}"); treebuilder.processTimeseriesMeta(meta, false).joinUninterruptibly(); - assertEquals(7, storage.numRows()); + assertEquals(7, storage.numRows(TREE_TABLE)); final Branch branch = JSON.parseToObject( - storage.getColumn(Branch.stringToId("00010001A24696026FD8"), - "branch".getBytes(MockBase.ASCII())), Branch.class); + storage.getColumn(TREE_TABLE, Branch.stringToId("00010001A24696026FD8"), + Tree.TREE_FAMILY(), BRANCH), Branch.class); assertEquals("V: web", branch.getDisplayName()); } @@ -593,10 +597,10 @@ public void processTimeseriesMetaFormatValue() throws Exception { public void processTimeseriesMetaFormatTSUID() throws Exception { tree.getRules().get(1).get(1).setDisplayFormat("TSUID: {tsuid}"); treebuilder.processTimeseriesMeta(meta, false).joinUninterruptibly(); - assertEquals(7, storage.numRows()); + assertEquals(7, storage.numRows(TREE_TABLE)); final Branch branch = JSON.parseToObject( - storage.getColumn(Branch.stringToId("00010001A246E0A07086"), - "branch".getBytes(MockBase.ASCII())), Branch.class); + storage.getColumn(TREE_TABLE, Branch.stringToId("00010001A246E0A07086"), + Tree.TREE_FAMILY(), BRANCH), Branch.class); assertEquals("TSUID: " + tsuid, branch.getDisplayName()); } @@ -604,10 +608,10 @@ public void processTimeseriesMetaFormatTSUID() throws Exception { public void processTimeseriesMetaFormatTagName() throws Exception { tree.getRules().get(1).get(1).setDisplayFormat("TAGNAME: {tag_name}"); treebuilder.processTimeseriesMeta(meta, false).joinUninterruptibly(); - assertEquals(7, storage.numRows()); + assertEquals(7, storage.numRows(TREE_TABLE)); final Branch branch = JSON.parseToObject( - storage.getColumn(Branch.stringToId("00010001A2467BFCCB13"), - "branch".getBytes(MockBase.ASCII())), Branch.class); + storage.getColumn(TREE_TABLE, Branch.stringToId("00010001A2467BFCCB13"), + Tree.TREE_FAMILY(), BRANCH), Branch.class); assertEquals("TAGNAME: host", branch.getDisplayName()); } @@ -616,10 +620,10 @@ public void processTimeseriesMetaFormatMulti() throws Exception { tree.getRules().get(1).get(1).setDisplayFormat( "{ovalue}:{value}:{tag_name}:{tsuid}"); treebuilder.processTimeseriesMeta(meta, false).joinUninterruptibly(); - assertEquals(7, storage.numRows()); + assertEquals(7, storage.numRows(TREE_TABLE)); final Branch branch = JSON.parseToObject( - storage.getColumn(Branch.stringToId("00010001A246E4592083"), - "branch".getBytes(MockBase.ASCII())), Branch.class); + storage.getColumn(TREE_TABLE, Branch.stringToId("00010001A246E4592083"), + Tree.TREE_FAMILY(), BRANCH), Branch.class); assertEquals("web-01.lga.mysite.com:web:host:0102030405", branch.getDisplayName()); } @@ -628,11 +632,11 @@ public void processTimeseriesMetaFormatMulti() throws Exception { public void processTimeseriesMetaFormatBadType() throws Exception { tree.getRules().get(3).get(0).setDisplayFormat("Wrong: {tag_name}"); treebuilder.processTimeseriesMeta(meta, false).joinUninterruptibly(); - assertEquals(5, storage.numRows()); + assertEquals(5, storage.numRows(TREE_TABLE)); final Branch branch = JSON.parseToObject( - storage.getColumn(Branch.stringToId( + storage.getColumn(TREE_TABLE, Branch.stringToId( "00010001A2460001CB54247F7202C3165573"), - "branch".getBytes(MockBase.ASCII())), Branch.class); + Tree.TREE_FAMILY(), BRANCH), Branch.class); assertEquals("Wrong: ", branch.getDisplayName()); } @@ -640,11 +644,11 @@ public void processTimeseriesMetaFormatBadType() throws Exception { public void processTimeseriesMetaFormatOverride() throws Exception { tree.getRules().get(3).get(0).setDisplayFormat("OVERRIDE"); treebuilder.processTimeseriesMeta(meta, false).joinUninterruptibly(); - assertEquals(5, storage.numRows()); + assertEquals(5, storage.numRows(TREE_TABLE)); final Branch branch = JSON.parseToObject( - storage.getColumn(Branch.stringToId( + storage.getColumn(TREE_TABLE, Branch.stringToId( "00010001A2460001CB54247F72024E3D0BCC"), - "branch".getBytes(MockBase.ASCII())), Branch.class); + Tree.TREE_FAMILY(), BRANCH), Branch.class); assertEquals("OVERRIDE", branch.getDisplayName()); } } diff --git a/test/tree/TestTreeRule.java b/test/tree/TestTreeRule.java index 6aa7ccdd1f..3c3010e405 100644 --- a/test/tree/TestTreeRule.java +++ b/test/tree/TestTreeRule.java @@ -16,9 +16,10 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.anyString; import static org.powermock.api.mockito.PowerMockito.mock; +import java.util.ArrayList; +import java.util.List; import java.util.regex.PatternSyntaxException; import net.opentsdb.core.TSDB; @@ -37,7 +38,6 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; @@ -50,6 +50,7 @@ PutRequest.class, KeyValue.class, Scanner.class, DeleteRequest.class, Tree.class}) public final class TestTreeRule { + private final static byte[] TREE_TABLE = "tsdb-tree".getBytes(MockBase.ASCII()); private TSDB tsdb; private HBaseClient client = mock(HBaseClient.class); private MockBase storage; @@ -58,9 +59,7 @@ public final class TestTreeRule { @Before public void before() throws Exception { final Config config = new Config(false); - PowerMockito.whenNew(HBaseClient.class) - .withArguments(anyString(), anyString()).thenReturn(client); - tsdb = new TSDB(config); + tsdb = new TSDB(client, config); rule = new TreeRule(); } @@ -243,7 +242,7 @@ public void storeRule() throws Exception { rule.setType(TreeRuleType.METRIC); rule.setNotes("Just some notes"); assertTrue(rule.syncToStorage(storage.getTSDB(), false).joinUninterruptibly()); - assertEquals(3, storage.numColumns(new byte[] { 0, 1 })); + assertEquals(3, storage.numColumns(TREE_TABLE, new byte[] { 0, 1 })); } @Test @@ -254,9 +253,9 @@ public void storeRuleMege() throws Exception { rule.setOrder(1); rule.setNotes("Just some notes"); assertTrue(rule.syncToStorage(storage.getTSDB(), false).joinUninterruptibly()); - assertEquals(2, storage.numColumns(new byte[] { 0, 1 })); + assertEquals(2, storage.numColumns(TREE_TABLE, new byte[] { 0, 1 })); final TreeRule stored = JSON.parseToObject( - storage.getColumn(new byte[] { 0, 1 }, + storage.getColumn(TREE_TABLE, new byte[] { 0, 1 }, Tree.TREE_FAMILY(), "tree_rule:2:1".getBytes(MockBase.ASCII())), TreeRule.class); assertEquals("Host owner", stored.getDescription()); assertEquals("Just some notes", stored.getNotes()); @@ -390,14 +389,14 @@ public void storeRuleInvalidRegexIdx() throws Exception { public void deleteRule() throws Exception { setupStorage(); assertNotNull(TreeRule.deleteRule(storage.getTSDB(), 1, 2, 1)); - assertEquals(1, storage.numColumns(new byte[] { 0, 1 })); + assertEquals(1, storage.numColumns(TREE_TABLE, new byte[] { 0, 1 })); } @Test public void deleteAllRules() throws Exception { setupStorage(); TreeRule.deleteAllRules(storage.getTSDB(), 1); - assertEquals(1, storage.numColumns(new byte[] { 0, 1 })); + assertEquals(1, storage.numColumns(TREE_TABLE, new byte[] { 0, 1 })); } @Test @@ -417,7 +416,10 @@ public void getQualifier() throws Exception { */ private void setupStorage() throws Exception { storage = new MockBase(tsdb, client, true, true, true, true); - + final List<byte[]> families = new ArrayList<byte[]>(); + families.add(Tree.TREE_FAMILY()); + storage.addTable(TREE_TABLE, families); + final TreeRule stored_rule = new TreeRule(1); stored_rule.setLevel(2); stored_rule.setOrder(1); @@ -428,11 +430,11 @@ private void setupStorage() throws Exception { stored_rule.setNotes("Owner of the host machine"); // pretend there's a tree definition in the storage row - storage.addColumn(new byte[] { 0, 1 }, "tree".getBytes(MockBase.ASCII()), - new byte[] { 1 }); + storage.addColumn(TREE_TABLE, new byte[] { 0, 1 }, Tree.TREE_FAMILY(), + "tree".getBytes(MockBase.ASCII()), new byte[] { 1 }); // add a rule to the row - storage.addColumn(new byte[] { 0, 1 }, + storage.addColumn(TREE_TABLE, new byte[] { 0, 1 }, Tree.TREE_FAMILY(), "tree_rule:2:1".getBytes(MockBase.ASCII()), JSON.serializeToBytes(stored_rule)); } diff --git a/test/tsd/BaseTestPutRpc.java b/test/tsd/BaseTestPutRpc.java new file mode 100644 index 0000000000..b6bc2eb514 --- /dev/null +++ b/test/tsd/BaseTestPutRpc.java @@ -0,0 +1,161 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.tsd; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.util.concurrent.atomic.AtomicLong; + +import org.hbase.async.HBaseClient; +import org.hbase.async.PleaseThrottleException; +import org.hbase.async.Scanner; +import org.jboss.netty.util.HashedWheelTimer; +import org.junit.Before; +import org.junit.Ignore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.reflect.Whitebox; + +import com.stumbleupon.async.TimeoutException; + +import net.opentsdb.core.TSDB; +import net.opentsdb.uid.UniqueId; +import net.opentsdb.core.BaseTsdbTest; +import net.opentsdb.core.Const; +import net.opentsdb.core.IncomingDataPoint; +import net.opentsdb.utils.Config; +import net.opentsdb.utils.Threads; + +@Ignore +@PrepareForTest({ TSDB.class, Config.class, HttpQuery.class, UniqueId.class, + HBaseClient.class, HashedWheelTimer.class, Scanner.class, Const.class, Threads.class, + TimeoutException.class, StorageExceptionHandler.class, PutDataPointRpc.class, + PleaseThrottleException.class, RollupDataPointRpc.class }) +public class BaseTestPutRpc extends BaseTsdbTest { + protected AtomicLong telnet_requests = new AtomicLong(); + protected AtomicLong http_requests = new AtomicLong(); + protected AtomicLong raw_dps = new AtomicLong(); + protected AtomicLong rollup_dps = new AtomicLong(); + protected AtomicLong raw_stored = new AtomicLong(); + protected AtomicLong rollup_stored = new AtomicLong(); + protected AtomicLong hbase_errors = new AtomicLong(); + protected AtomicLong unknown_errors = new AtomicLong(); + protected AtomicLong invalid_values = new AtomicLong(); + protected AtomicLong illegal_arguments = new AtomicLong(); + protected AtomicLong unknown_metrics = new AtomicLong(); + protected AtomicLong inflight_exceeded = new AtomicLong(); + protected AtomicLong writes_blocked = new AtomicLong(); + protected AtomicLong writes_timedout = new AtomicLong(); + protected AtomicLong requests_timedout = new AtomicLong(); + protected StorageExceptionHandler handler; + + @Before + public void beforeCounters() throws Exception { + telnet_requests = Whitebox.getInternalState(PutDataPointRpc.class, "telnet_requests"); + telnet_requests.set(0); + http_requests = Whitebox.getInternalState(PutDataPointRpc.class, "http_requests"); + http_requests.set(0); + raw_dps = Whitebox.getInternalState(PutDataPointRpc.class, "raw_dps"); + raw_dps.set(0); + rollup_dps = Whitebox.getInternalState(PutDataPointRpc.class, "rollup_dps"); + rollup_dps.set(0); + hbase_errors = Whitebox.getInternalState(PutDataPointRpc.class, "hbase_errors"); + hbase_errors.set(0); + unknown_errors = Whitebox.getInternalState(PutDataPointRpc.class, "unknown_errors"); + unknown_errors.set(0); + raw_stored = Whitebox.getInternalState(PutDataPointRpc.class, "raw_stored"); + raw_stored.set(0); + rollup_stored = Whitebox.getInternalState(PutDataPointRpc.class, "rollup_stored"); + rollup_stored.set(0); + invalid_values = Whitebox.getInternalState(PutDataPointRpc.class, "invalid_values"); + invalid_values.set(0); + illegal_arguments = Whitebox.getInternalState(PutDataPointRpc.class, "illegal_arguments"); + illegal_arguments.set(0); + unknown_metrics = Whitebox.getInternalState(PutDataPointRpc.class, "unknown_metrics"); + unknown_metrics.set(0); + inflight_exceeded = Whitebox.getInternalState(PutDataPointRpc.class, "inflight_exceeded"); + inflight_exceeded.set(0); + writes_blocked = Whitebox.getInternalState(PutDataPointRpc.class, "writes_blocked"); + writes_blocked.set(0); + writes_timedout = Whitebox.getInternalState(PutDataPointRpc.class, "writes_timedout"); + writes_timedout.set(0); + requests_timedout = Whitebox.getInternalState(PutDataPointRpc.class, "requests_timedout"); + requests_timedout.set(0); + } + + /** + * Helper to set the storage exception handler in the TSDB under test. + */ + protected void setStorageExceptionHandler() { + handler = mock(StorageExceptionHandler.class); + Whitebox.setInternalState(tsdb, "storage_exception_handler", handler); + } + + /** + * Helper to validate calls to the storage exception handler. + * @param called Whether or not SEH should have been called. + */ + protected void validateSEH(final boolean called) { + if (called) { + verify(tsdb, times(1)).getStorageExceptionHandler(); + if (handler != null) { + verify(handler, times(1)).handleError((IncomingDataPoint)any(), + (Exception)any()); + } + } else { + verify(tsdb, never()).getStorageExceptionHandler(); + if (handler != null) { + verify(handler, never()).handleError((IncomingDataPoint)any(), + (Exception)any()); + } + } + } + + // Helper to validate all the counters per call. + protected void validateCounters( + final long telnet_requests, + final long http_requests, + final long raw_dps, + final long rollup_dps, + final long raw_stored, + final long rollup_stored, + final long hbase_errors, + final long unknown_errors, + final long invalid_values, + final long illegal_arguments, + final long unknown_metrics, + final long inflight_exceeded, + final long writes_blocked, + final long writes_timedout, + final long requests_timedout) { + assertEquals(telnet_requests, this.telnet_requests.get()); + assertEquals(http_requests, this.http_requests.get()); + assertEquals(raw_dps, this.raw_dps.get()); + assertEquals(rollup_dps, this.rollup_dps.get()); + assertEquals(raw_stored, this.raw_stored.get()); + assertEquals(rollup_stored, this.rollup_stored.get()); + assertEquals(hbase_errors, this.hbase_errors.get()); + assertEquals(unknown_errors, this.unknown_errors.get()); + assertEquals(invalid_values, this.invalid_values.get()); + assertEquals(illegal_arguments, this.illegal_arguments.get()); + assertEquals(unknown_metrics, this.unknown_metrics.get()); + assertEquals(inflight_exceeded, this.inflight_exceeded.get()); + assertEquals(writes_blocked, this.writes_blocked.get()); + assertEquals(writes_timedout, this.writes_timedout.get()); + assertEquals(requests_timedout, this.requests_timedout.get()); + } +} diff --git a/test/tsd/DummyHttpRpcPlugin.java b/test/tsd/DummyHttpRpcPlugin.java new file mode 100644 index 0000000000..8115ba1232 --- /dev/null +++ b/test/tsd/DummyHttpRpcPlugin.java @@ -0,0 +1,56 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2014 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.tsd; + +import java.io.IOException; + +import com.google.common.base.Preconditions; +import com.stumbleupon.async.Deferred; + +import net.opentsdb.core.TSDB; +import net.opentsdb.stats.StatsCollector; + +/** + * This is a dummy HTTP RPC plugin implementation for unit test purposes. + * @since 2.1 + */ +public class DummyHttpRpcPlugin extends HttpRpcPlugin { + @Override + public void initialize(TSDB tsdb) { + Preconditions.checkNotNull(tsdb); + } + + @Override + public Deferred<Object> shutdown() { + return Deferred.fromResult(null); + } + + @Override + public String version() { + return "2.0.0"; + } + + @Override + public void collectStats(StatsCollector collector) { + collector.record("http_rpcplugin.dummy.value", 1); + } + + @Override + public String getPath() { + return "/dummy/test"; + } + + @Override + public void execute(TSDB tsdb, HttpRpcPluginQuery query) throws IOException { + } +} diff --git a/test/tsd/DummySEHPlugin.java b/test/tsd/DummySEHPlugin.java new file mode 100644 index 0000000000..56c2af2fb2 --- /dev/null +++ b/test/tsd/DummySEHPlugin.java @@ -0,0 +1,61 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.tsd; + +import net.opentsdb.core.IllegalDataException; +import net.opentsdb.core.IncomingDataPoint; +import net.opentsdb.core.TSDB; +import net.opentsdb.stats.StatsCollector; + +import com.stumbleupon.async.Deferred; + +public class DummySEHPlugin extends StorageExceptionHandler { + + @Override + public void initialize(TSDB tsdb) { + if (tsdb == null) { + throw new IllegalArgumentException("The TSDB object was null"); + } + // a dummy config to check for throwing exceptions + if (!tsdb.getConfig().hasProperty( + "tsd.core.storage_exception_handler.DummySEHPlugin.hosts")) { + throw new IllegalArgumentException("Missing hosts config"); + } + } + + @Override + public Deferred<Object> shutdown() { + return Deferred.fromResult(new Object()); + } + + @Override + public String version() { + return "2.2.0"; + } + + @Override + public void collectStats(StatsCollector collector) { + collector.record("seh.dummy.retries", 1); + } + + @Override + public void handleError(IncomingDataPoint dp, Exception exception) { + if (dp == null) { + throw new IllegalArgumentException("Missing Data Point"); + } + if (dp.getValue().equals("42")) { + throw new IllegalDataException("Testing"); + } + } + +} diff --git a/test/tsd/NettyMocks.java b/test/tsd/NettyMocks.java index c881462f9e..0c384af64b 100644 --- a/test/tsd/NettyMocks.java +++ b/test/tsd/NettyMocks.java @@ -12,17 +12,28 @@ // see <http://www.gnu.org/licenses/>. package net.opentsdb.tsd; +import static org.mockito.Matchers.any; import static org.mockito.Mockito.when; import static org.powermock.api.mockito.PowerMockito.mock; +import java.net.SocketAddress; import java.nio.charset.Charset; -import java.util.HashMap; +import com.stumbleupon.async.Deferred; +import net.opentsdb.auth.AllowAllAuthenticatingAuthorizer; +import net.opentsdb.auth.AuthState; +import net.opentsdb.auth.Authentication; +import net.opentsdb.auth.Authorization; +import net.opentsdb.core.DataPoints; import net.opentsdb.core.TSDB; +import net.opentsdb.core.TSQuery; import net.opentsdb.utils.Config; +import org.jboss.netty.buffer.ChannelBuffer; import org.jboss.netty.buffer.ChannelBuffers; import org.jboss.netty.channel.Channel; +import org.jboss.netty.channel.ChannelFuture; +import org.jboss.netty.channel.DefaultChannelFuture; import org.jboss.netty.channel.DefaultChannelPipeline; import org.jboss.netty.handler.codec.http.DefaultHttpRequest; import org.jboss.netty.handler.codec.http.HttpMethod; @@ -31,7 +42,6 @@ import org.jboss.netty.handler.codec.http.HttpResponseEncoder; import org.jboss.netty.handler.codec.http.HttpVersion; import org.junit.Ignore; -import org.powermock.reflect.Whitebox; /** * Helper class that provides mockups for testing any OpenTSDB processes that @@ -44,16 +54,46 @@ public final class NettyMocks { * Sets up a TSDB object for HTTP RPC tests that has a Config object * @return A TSDB mock */ - public static TSDB getMockedHTTPTSDB() { + public static TSDB getMockedHTTPTSDB() throws Exception { final TSDB tsdb = mock(TSDB.class); - final Config config = mock(Config.class); - HashMap<String, String> properties = new HashMap<String, String>(); - properties.put("tsd.http.show_stack_trace", "true"); - Whitebox.setInternalState(config, "properties", properties); + + final Config config = new Config(false); + config.overrideConfig("tsd.http.show_stack_trace", "true"); + config.overrideConfig("tsd.core.authentication.enable", "false"); when(tsdb.getConfig()).thenReturn(config); + + final Authentication authentication = mock(Authentication.class); + when(authentication.isReady(any(TSDB.class), any(Channel.class))).thenReturn(false); + when(tsdb.getAuth()).thenReturn(authentication); + return tsdb; } - + + /** + * Sets up a TSDB object for HTTP RPC tests that has a Config object + * @return A TSDB mock + */ + public static TSDB getMockedHTTPTSDBWithAuthEnabled(final AuthState.AuthStatus authStatus) throws Exception { + final TSDB tsdb = mock(TSDB.class); + + final Config config = new Config(false); + config.overrideConfig("tsd.http.show_stack_trace", "true"); + config.overrideConfig("tsd.core.authentication.enable", "true"); + when(tsdb.getConfig()).thenReturn(config); + + final AuthState state = mock(AuthState.class); + when(state.getStatus()).thenReturn(authStatus); + + final Authorization authorization = mock(Authorization.class); + when(authorization.allowQuery(any(AuthState.class), any(TSQuery.class))).thenReturn(state); + + final Authentication authentication = mock(Authentication.class); + when(authentication.authorization()).thenReturn(authorization); + when(authentication.isReady(any(TSDB.class), any(Channel.class))).thenReturn(true); + when(tsdb.getAuth()).thenReturn(authentication); + return tsdb; + } + /** * Returns a mocked Channel object that simply sets the name to * [fake channel] @@ -63,6 +103,11 @@ public static Channel fakeChannel() { final Channel chan = mock(Channel.class); when(chan.toString()).thenReturn("[fake channel]"); when(chan.isConnected()).thenReturn(true); + when(chan.isWritable()).thenReturn(true); + + final SocketAddress socket = mock(SocketAddress.class); + when(socket.toString()).thenReturn("192.168.1.1:4243"); + when(chan.getRemoteAddress()).thenReturn(socket); return chan; } @@ -192,6 +237,15 @@ public static HttpQuery contentQuery(final TSDB tsdb, final String uri, req.headers().set("Content-Type", type); return new HttpQuery(tsdb, req, channelMock); } + + /** + * @param query the query to mock a future callback for + */ + public static void mockChannelFuture(final HttpQuery query) { + final ChannelFuture future = new DefaultChannelFuture(query.channel(), false); + when(query.channel().write(any(ChannelBuffer.class))).thenReturn(future); + future.setSuccess(); + } /** * Returns a simple pipeline with an HttpRequestDecoder and an diff --git a/test/tsd/TestAnnotationRpc.java b/test/tsd/TestAnnotationRpc.java index 299e58be1e..fc80a399d5 100644 --- a/test/tsd/TestAnnotationRpc.java +++ b/test/tsd/TestAnnotationRpc.java @@ -14,7 +14,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.anyString; import static org.powermock.api.mockito.PowerMockito.mock; import java.nio.charset.Charset; @@ -37,7 +36,6 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; @@ -63,23 +61,21 @@ public final class TestAnnotationRpc { @Before public void before() throws Exception { final Config config = new Config(false); - PowerMockito.whenNew(HBaseClient.class) - .withArguments(anyString(), anyString()).thenReturn(client); - tsdb = new TSDB(config); + tsdb = new TSDB(client, config); storage = new MockBase(tsdb, client, true, true, true, true); - // add a global + // add a global storage.addColumn(global_row_key, new byte[] { 1, 0, 0 }, ("{\"startTime\":1328140800,\"endTime\":1328140801,\"description\":" + "\"Description\",\"notes\":\"Notes\",\"custom\":{\"owner\":" + - "\"ops\"}}").getBytes(MockBase.ASCII())); + "\"ops\"}}").getBytes(MockBase.ASCII()), 1328140799972L); storage.addColumn(global_row_key, new byte[] { 1, 0, 1 }, ("{\"startTime\":1328140801,\"endTime\":1328140803,\"description\":" + - "\"Global 2\",\"notes\":\"Nothing\"}").getBytes(MockBase.ASCII())); + "\"Global 2\",\"notes\":\"Nothing\"}").getBytes(MockBase.ASCII()), 1328140799973L); // add a local storage.addColumn(tsuid_row_key, @@ -87,21 +83,21 @@ public void before() throws Exception { ("{\"tsuid\":\"000001000001000001\",\"startTime\":1388450562," + "\"endTime\":1419984000,\"description\":\"Hello!\",\"notes\":" + "\"My Notes\",\"custom\":{\"owner\":\"ops\"}}") - .getBytes(MockBase.ASCII())); + .getBytes(MockBase.ASCII()), 1388448000003L); storage.addColumn(tsuid_row_key, new byte[] { 1, 0x0A, 0x03 }, ("{\"tsuid\":\"000001000001000001\",\"startTime\":1388450563," + "\"endTime\":1419984000,\"description\":\"Note2\",\"notes\":" + "\"Nothing\"}") - .getBytes(MockBase.ASCII())); + .getBytes(MockBase.ASCII()), 1388448000004L); // add some data points too storage.addColumn(tsuid_row_key, - new byte[] { 0x50, 0x10 }, new byte[] { 1 }); + new byte[] { 0x50, 0x10 }, new byte[] { 1 }, 1388448000005L); storage.addColumn(tsuid_row_key, - new byte[] { 0x50, 0x18 }, new byte[] { 2 }); + new byte[] { 0x50, 0x18 }, new byte[] { 2 }, 1388448000006L); } @Test @@ -120,6 +116,7 @@ public void badMethod() throws Exception { @Test public void get() throws Exception { + storage.dumpToSystemOut(); HttpQuery query = NettyMocks.getQuery(tsdb, "/api/annotation?tsuid=000001000001000001&start_time=1388450562"); rpc.execute(tsdb, query); @@ -134,6 +131,14 @@ public void getGlobal() throws Exception { assertEquals(HttpResponseStatus.OK, query.response().getStatus()); } + @Test + public void getGlobals() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/annotations?start_time=1328140800"); + rpc.execute(tsdb, query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + } + @Test (expected = BadRequestException.class) public void getNotFound() throws Exception { HttpQuery query = NettyMocks.getQuery(tsdb, @@ -148,6 +153,13 @@ public void getGlobalNotFound() throws Exception { rpc.execute(tsdb, query); } + @Test (expected = BadRequestException.class) + public void getGlobalsNotFound() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/annotation?start_time=1388450563"); + rpc.execute(tsdb, query); + } + @Test (expected = BadRequestException.class) public void getMissingStart() throws Exception { HttpQuery query = NettyMocks.getQuery(tsdb, diff --git a/test/tsd/TestGraphHandler.java b/test/tsd/TestGraphHandler.java index e8c49d5ac4..d2600f8df6 100644 --- a/test/tsd/TestGraphHandler.java +++ b/test/tsd/TestGraphHandler.java @@ -1,5 +1,5 @@ // This file is part of OpenTSDB. -// Copyright (C) 2011-2012 The OpenTSDB Authors. +// Copyright (C) 2011-2021 The OpenTSDB Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU Lesser General Public License as published by @@ -13,14 +13,21 @@ package net.opentsdb.tsd; import java.io.File; +import java.lang.reflect.Method; +import java.util.List; +import java.util.Map; import org.jboss.netty.channel.Channel; - -import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; + +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -30,6 +37,12 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; import org.powermock.reflect.Whitebox; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import net.opentsdb.graph.Plot; + import static org.powermock.api.mockito.PowerMockito.mock; @RunWith(PowerMockRunner.class) @@ -38,14 +51,141 @@ @PowerMockIgnore({"javax.management.*", "javax.xml.*", "ch.qos.*", "org.slf4j.*", "com.sum.*", "org.xml.*"}) -@PrepareForTest({ GraphHandler.class, HttpQuery.class }) +@PrepareForTest({ GraphHandler.class, HttpQuery.class, Plot.class }) public final class TestGraphHandler { - @BeforeClass - public static void setUpClass() throws Exception { - // This is pure voodoo. It ensures the GraphHelper is initialized on time - // for all other tests. - staleCacheFile(null, 0, 10, fakeFile("voodoo")); + private final static Method sm; + static { + try { + sm = GraphHandler.class.getDeclaredMethod("staleCacheFile", + HttpQuery.class, long.class, long.class, File.class); + sm.setAccessible(true); + } catch (Exception e) { + throw new RuntimeException("Failed in static initializer", e); + } + } + + @Test + public void setYRangeParams() throws Exception { + assertPlotParam("yrange","[0:1]"); + assertPlotParam("yrange", "[:]"); + assertPlotParam("yrange", "[:0]"); + assertPlotParam("yrange", "[:42]"); + assertPlotParam("yrange", "[:-42]"); + assertPlotParam("yrange", "[:0.8]"); + assertPlotParam("yrange", "[:-0.8]"); + assertPlotParam("yrange", "[:42.4]"); + assertPlotParam("yrange", "[:-42.4]"); + assertPlotParam("yrange", "[:4e4]"); + assertPlotParam("yrange", "[:-4e4]"); + assertPlotParam("yrange", "[:4e-4]"); + assertPlotParam("yrange", "[:-4e-4]"); + assertPlotParam("yrange", "[:4.2e4]"); + assertPlotParam("yrange", "[:-4.2e4]"); + assertPlotParam("yrange", "[0:]"); + assertPlotParam("yrange", "[5:]"); + assertPlotParam("yrange", "[-5:]"); + assertPlotParam("yrange", "[0.5:]"); + assertPlotParam("yrange", "[-0.5:]"); + assertPlotParam("yrange", "[10.5:]"); + assertPlotParam("yrange", "[-10.5:]"); + assertPlotParam("yrange", "[10e5:]"); + assertPlotParam("yrange", "[-10e5:]"); + assertPlotParam("yrange", "[10e-5:]"); + assertPlotParam("yrange", "[-10e-5:]"); + assertPlotParam("yrange", "[10.1e-5:]"); + assertPlotParam("yrange", "[-10.1e-5:]"); + assertPlotParam("yrange", "[-10.1e-5:-10.1e-6]"); + assertInvalidPlotParam("yrange", "[33:system('touch /tmp/poc.txt')]"); + assertInvalidPlotParam("y2range", "[42:%0a[33:system('touch /tmp/poc.txt')]"); + } + + @Test + public void setKeyParams() throws Exception { + assertPlotParam("key", "out"); + assertPlotParam("key", "left"); + assertPlotParam("key", "top"); + assertPlotParam("key", "center"); + assertPlotParam("key", "right"); + assertPlotParam("key", "horiz"); + assertPlotParam("key", "box"); + assertPlotParam("key", "bottom"); + assertInvalidPlotParam("key", "out%20right%20top%0aset%20yrange%20[33:system(%20"); + assertInvalidPlotParam("key", "%3Bsystem%20%22cat%20/home/ubuntuvm/secret.txt%20%3E/tmp/secret.txt%22%20%22"); + } + + @Test + public void setStyleParams() throws Exception { + assertPlotParam("style", "linespoint"); + assertPlotParam("style", "points"); + assertPlotParam("style", "circles"); + assertPlotParam("style", "dots"); + assertInvalidPlotParam("style", "dots%20%0a[33:system(%20"); + assertInvalidPlotParam("style", "%3Bsystem%20%22cat%20/home/ubuntuvm/secret.txt%20%3E/tmp/secret.txt%22%20%22\""); + } + + @Test + public void setLabelParams() throws Exception { + assertPlotParam("ylabel", "This is good"); + assertPlotParam("ylabel", " and so Is this - _ yay"); + assertInvalidPlotParam("ylabel", "system(%20no%0anewlines"); + assertInvalidPlotParam("title", "system(%20no%0anewlines"); + assertInvalidPlotParam("y2label", "system(%20no%0anewlines"); + } + + @Test + public void setWXH() throws Exception { + assertPlotDimension("wxh", "720x640"); + assertInvalidPlotDimension("wxh", "720%0ax640"); + } + + @Test + public void setColorParams() throws Exception { + assertPlotParam("bgcolor", "x000000"); + assertPlotParam("bgcolor", "XDEADBE"); + assertPlotParam("bgcolor", "%58DEADBE"); + assertInvalidPlotParam("bgcolor", "XDEADBEF"); + assertInvalidPlotParam("bgcolor", "%5BDEADBE"); + assertInvalidPlotParam("bgcolor", "xBDE%0AAD"); + + assertPlotParam("fgcolor", "x000000"); + assertPlotParam("fgcolor", "XDEADBE"); + assertPlotParam("fgcolor", "%58DEADBE"); + assertInvalidPlotParam("fgcolor", "XDEADBEF"); + assertInvalidPlotParam("fgcolor", "%5BDEADBE"); + assertInvalidPlotParam("fgcolor", "xBDE%0AAD"); + } + + @Test + public void setSmoothParams() throws Exception { + assertPlotParam("smooth", "unique"); + assertPlotParam("smooth", "frequency"); + assertPlotParam("smooth", "fnormal"); + assertPlotParam("smooth", "cumulative"); + assertPlotParam("smooth", "cnormal"); + assertPlotParam("smooth", "bins"); + assertPlotParam("smooth", "csplines"); + assertPlotParam("smooth", "acsplines"); + assertPlotParam("smooth", "mcsplines"); + assertPlotParam("smooth", "bezier"); + assertPlotParam("smooth", "sbezier"); + assertPlotParam("smooth", "unwrap"); + assertPlotParam("smooth", "zsort"); + assertInvalidPlotParam("smooth", "bezier%20system(%20"); + assertInvalidPlotParam("smooth", "fnormal%0asystem(%20"); + } + + @Test + public void setFormatParams() throws Exception { + assertPlotParam("yformat", "%25.2f"); + assertPlotParam("y2format", "%25.2f"); + assertPlotParam("xformat", "%25.2f"); + assertPlotParam("yformat", "%253.0em"); + assertPlotParam("yformat", "%253.0em%25%25"); + assertPlotParam("yformat", "%25.2f seconds"); + assertPlotParam("yformat", "%25.0f ms"); + assertInvalidPlotParam("yformat", "%252.system(%20"); + assertInvalidPlotParam("yformat", "%252.%0asystem(%20"); } @Test // If the file doesn't exist, we don't use it, obviously. @@ -80,56 +220,56 @@ public void staleCacheFileInTheFuture() throws Exception { System.currentTimeMillis(); // ... this was called only once. } - @Test // End time in the future => OK to serve stale file up to max_age. - public void staleCacheFileEndTimeInFuture() throws Exception { - PowerMockito.mockStatic(System.class); - - final HttpQuery query = fakeHttpQuery(); - final File cachedfile = fakeFile("/cache/fake-file"); - - final long end_time = 20000L; - when(System.currentTimeMillis()).thenReturn(10000L); - when(cachedfile.lastModified()).thenReturn(8000L); - - assertFalse("File is not more than 3s stale", - staleCacheFile(query, end_time, 3, cachedfile)); - assertFalse("File is more than 2s stale", - staleCacheFile(query, end_time, 2, cachedfile)); - assertTrue("File is more than 1s stale", - staleCacheFile(query, end_time, 1, cachedfile)); - - // Ensure that we stat() the file and look at the current time once per - // invocation of staleCacheFile(). - verify(cachedfile, times(3)).lastModified(); - PowerMockito.verifyStatic(times(3)); - System.currentTimeMillis(); - } - - @Test // No end time = end time is now. - public void staleCacheFileEndTimeIsNow() throws Exception { - PowerMockito.mockStatic(System.class); - - final HttpQuery query = fakeHttpQuery(); - final File cachedfile = fakeFile("/cache/fake-file"); - - final long now = 10000L; - final long end_time = now; - when(System.currentTimeMillis()).thenReturn(now); - when(cachedfile.lastModified()).thenReturn(8000L); - - assertFalse("File is not more than 3s stale", - staleCacheFile(query, end_time, 3, cachedfile)); - assertFalse("File is more than 2s stale", - staleCacheFile(query, end_time, 2, cachedfile)); - assertTrue("File is more than 1s stale", - staleCacheFile(query, end_time, 1, cachedfile)); - - // Ensure that we stat() the file and look at the current time once per - // invocation of staleCacheFile(). - verify(cachedfile, times(3)).lastModified(); - PowerMockito.verifyStatic(times(3)); - System.currentTimeMillis(); - } +// @Test // End time in the future => OK to serve stale file up to max_age. +// public void staleCacheFileEndTimeInFuture() throws Exception { +// PowerMockito.mockStatic(System.class); +// +// final HttpQuery query = fakeHttpQuery(); +// final File cachedfile = fakeFile("/cache/fake-file"); +// +// final long end_time = 20000L; +// when(System.currentTimeMillis()).thenReturn(10000L); +// when(cachedfile.lastModified()).thenReturn(8000L); +// +// assertFalse("File is not more than 3s stale", +// staleCacheFile(query, end_time, 3, cachedfile)); +// assertFalse("File is more than 2s stale", +// staleCacheFile(query, end_time, 2, cachedfile)); +// assertTrue("File is more than 1s stale", +// staleCacheFile(query, end_time, 1, cachedfile)); +// +// // Ensure that we stat() the file and look at the current time once per +// // invocation of staleCacheFile(). +// verify(cachedfile, times(3)).lastModified(); +// PowerMockito.verifyStatic(times(3)); +// System.currentTimeMillis(); +// } + +// @Test // No end time = end time is now. +// public void staleCacheFileEndTimeIsNow() throws Exception { +// PowerMockito.mockStatic(System.class); +// +// final HttpQuery query = fakeHttpQuery(); +// final File cachedfile = fakeFile("/cache/fake-file"); +// +// final long now = 10000L; +// final long end_time = now; +// when(System.currentTimeMillis()).thenReturn(now); +// when(cachedfile.lastModified()).thenReturn(8000L); +// +// assertFalse("File is not more than 3s stale", +// staleCacheFile(query, end_time, 3, cachedfile)); +// assertFalse("File is more than 2s stale", +// staleCacheFile(query, end_time, 2, cachedfile)); +// assertTrue("File is more than 1s stale", +// staleCacheFile(query, end_time, 1, cachedfile)); +// +// // Ensure that we stat() the file and look at the current time once per +// // invocation of staleCacheFile(). +// verify(cachedfile, times(3)).lastModified(); +// PowerMockito.verifyStatic(times(3)); +// System.currentTimeMillis(); +// } @Test // End time in the past, file's mtime predates it. public void staleCacheFileEndTimeInPastOlderFile() throws Exception { @@ -151,25 +291,25 @@ public void staleCacheFileEndTimeInPastOlderFile() throws Exception { System.currentTimeMillis(); // ... this was called only once. } - @Test // End time in the past, file's mtime is after it. - public void staleCacheFileEndTimeInPastCacheableFile() throws Exception { - PowerMockito.mockStatic(System.class); - - final HttpQuery query = fakeHttpQuery(); - final File cachedfile = fakeFile("/cache/fake-file"); - - final long end_time = 8000L; - final long now = end_time + 2000L; - when(System.currentTimeMillis()).thenReturn(now); - when(cachedfile.lastModified()).thenReturn(end_time + 1000L); - - assertFalse("File was created after end-time and can be re-used", - staleCacheFile(query, end_time, 1, cachedfile)); - - verify(cachedfile).lastModified(); // Ensure we do a single stat() call. - PowerMockito.verifyStatic(); // Verify that ... - System.currentTimeMillis(); // ... this was called only once. - } +// @Test // End time in the past, file's mtime is after it. +// public void staleCacheFileEndTimeInPastCacheableFile() throws Exception { +// PowerMockito.mockStatic(System.class); +// +// final HttpQuery query = fakeHttpQuery(); +// final File cachedfile = fakeFile("/cache/fake-file"); +// +// final long end_time = 8000L; +// final long now = end_time + 2000L; +// when(System.currentTimeMillis()).thenReturn(now); +// when(cachedfile.lastModified()).thenReturn(end_time + 1000L); +// +// assertFalse("File was created after end-time and can be re-used", +// staleCacheFile(query, end_time, 1, cachedfile)); +// +// verify(cachedfile).lastModified(); // Ensure we do a single stat() call. +// PowerMockito.verifyStatic(); // Verify that ... +// System.currentTimeMillis(); // ... this was called only once. +// } /** * Helper to call private static method. @@ -180,9 +320,17 @@ private static boolean staleCacheFile(final HttpQuery query, final long end_time, final long max_age, final File cachedfile) throws Exception { + PowerMockito.mockStatic(System.class); + PowerMockito.when(System.getProperty(anyString(), anyString())).thenReturn(""); + PowerMockito.when(System.getProperty(anyString())).thenReturn(""); + PowerMockito.spy(GraphHandler.class); + PowerMockito.doReturn("").when(GraphHandler.class, "findGnuplotHelperScript"); + return Whitebox.<Boolean>invokeMethod(GraphHandler.class, "staleCacheFile", query, end_time / 1000, max_age, cachedfile); + + //return (Boolean)sm.invoke(null, query, end_time / 1000, max_age, cachedfile); } private static HttpQuery fakeHttpQuery() { @@ -199,4 +347,44 @@ private static File fakeFile(final String path) { return file; } + private static void assertPlotParam(String param, String value) { + Plot plot = mock(Plot.class); + HttpQuery query = mock(HttpQuery.class); + Map<String, List<String>> params = Maps.newHashMap(); + when(query.getQueryString()).thenReturn(params); + + params.put(param, Lists.newArrayList(value)); + GraphHandler.setPlotParams(query, plot); + } + + private static void assertPlotDimension(String param, String value) { + Plot plot = mock(Plot.class); + HttpQuery query = mock(HttpQuery.class); + when(query.getQueryStringParam(param)).thenReturn(value); + GraphHandler.setPlotParams(query, plot); + } + + private static void assertInvalidPlotParam(String param, String value) { + Plot plot = mock(Plot.class); + HttpQuery query = mock(HttpQuery.class); + Map<String, List<String>> params = Maps.newHashMap(); + when(query.getQueryString()).thenReturn(params); + + params.put(param, Lists.newArrayList(value)); + try { + GraphHandler.setPlotParams(query, plot); + fail("Expected BadRequestException"); + } catch (BadRequestException e) { } + } + + private static void assertInvalidPlotDimension(String param, String value) { + Plot plot = mock(Plot.class); + HttpQuery query = mock(HttpQuery.class); + when(query.getQueryStringParam(param)).thenReturn(value); + try { + GraphHandler.setPlotDimensions(query, plot); + fail("Expected BadRequestException"); + } catch (BadRequestException e) { } + } + } diff --git a/test/tsd/TestHistogramDataPointRpc.java b/test/tsd/TestHistogramDataPointRpc.java new file mode 100644 index 0000000000..8254126a65 --- /dev/null +++ b/test/tsd/TestHistogramDataPointRpc.java @@ -0,0 +1,620 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2017 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. + +package net.opentsdb.tsd; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.HashMap; +import java.util.concurrent.atomic.AtomicLong; + +import org.hbase.async.HBaseClient; +import org.hbase.async.HBaseException; +import org.hbase.async.PleaseThrottleException; +import org.jboss.netty.channel.Channel; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.jboss.netty.util.HashedWheelTimer; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +import com.google.common.collect.Maps; + +import net.opentsdb.core.HistogramCodecManager; +import net.opentsdb.core.HistogramPojo; +import net.opentsdb.core.IncomingDataPoint; +import net.opentsdb.core.SimpleHistogram; +import net.opentsdb.core.TSDB; +import net.opentsdb.storage.MockBase; +import net.opentsdb.uid.UniqueId.UniqueIdType; +import net.opentsdb.utils.Config; +import net.opentsdb.utils.Threads; + +@RunWith(PowerMockRunner.class) +//"Classloader hell"... It's real. Tell PowerMock to ignore these classes +//because they fiddle with the class loader. We don't test them anyway. +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +public class TestHistogramDataPointRpc extends BaseTestPutRpc { + + protected AtomicLong raw_histograms = new AtomicLong(); + protected AtomicLong raw_histograms_stored = new AtomicLong(); + + private HistogramCodecManager manager; + private SimpleHistogram test_histo; + + @Before + public void before() throws Exception { + uid_map = Maps.newHashMap(); + PowerMockito.mockStatic(Threads.class); + timer = new FakeTaskTimer(); + PowerMockito.when(Threads.newTimer(anyString())).thenReturn(timer); + PowerMockito.when(Threads.newTimer(anyInt(), anyString())).thenReturn(timer); + + PowerMockito.whenNew(HashedWheelTimer.class).withNoArguments() + .thenReturn(timer); + PowerMockito.whenNew(HBaseClient.class).withAnyArguments() + .thenReturn(client); + + config = new Config(false); + config.overrideConfig("tsd.storage.enable_compaction", "false"); + config.overrideConfig("tsd.core.histograms.config", + "{\"net.opentsdb.core.SimpleHistogramDecoder\": 42}"); + tsdb = new TSDB(config); + + config.setAutoMetric(true); + + Whitebox.setInternalState(tsdb, "metrics", metrics); + Whitebox.setInternalState(tsdb, "tag_names", tag_names); + Whitebox.setInternalState(tsdb, "tag_values", tag_values); + + setupMetricMaps(); + setupTagkMaps(); + setupTagvMaps(); + + mockUID(UniqueIdType.METRIC, HISTOGRAM_METRIC_STRING, HISTOGRAM_METRIC_BYTES); + + // add metrics and tags to the UIDs list for other functions to share + uid_map.put(METRIC_STRING, METRIC_BYTES); + uid_map.put(METRIC_B_STRING, METRIC_B_BYTES); + uid_map.put(NSUN_METRIC, NSUI_METRIC); + uid_map.put(HISTOGRAM_METRIC_STRING, HISTOGRAM_METRIC_BYTES); + + uid_map.put(TAGK_STRING, TAGK_BYTES); + uid_map.put(TAGK_B_STRING, TAGK_B_BYTES); + uid_map.put(NSUN_TAGK, NSUI_TAGK); + + uid_map.put(TAGV_STRING, TAGV_BYTES); + uid_map.put(TAGV_B_STRING, TAGV_B_BYTES); + uid_map.put(NSUN_TAGV, NSUI_TAGV); + + uid_map.putAll(UIDS); + + when(metrics.width()).thenReturn((short)3); + when(tag_names.width()).thenReturn((short)3); + when(tag_values.width()).thenReturn((short)3); + + tags = new HashMap<String, String>(1); + tags.put(TAGK_STRING, TAGV_STRING); + + + manager = new HistogramCodecManager(tsdb); + Whitebox.setInternalState(tsdb, "histogram_manager", manager); + + storage = new MockBase(tsdb, client, true, true, true, true); + + test_histo = new SimpleHistogram(42); + test_histo.addBucket(0F, 1F, 42L); + test_histo.addBucket(1F, 5F, 24L); + test_histo.setOverflow(1L); + + // counters + raw_histograms = Whitebox.getInternalState(PutDataPointRpc.class, "raw_histograms"); + raw_histograms.set(0); + raw_histograms_stored = Whitebox.getInternalState(PutDataPointRpc.class, "raw_histograms_stored"); + raw_histograms_stored.set(0); + } + + @Test + public void constructor() { + HistogramDataPointRpc rpc = new HistogramDataPointRpc(tsdb.getConfig()); + assertTrue(rpc.enabled()); + + config.overrideConfig("tsd.core.histograms.config", null); + rpc = new HistogramDataPointRpc(tsdb.getConfig()); + assertFalse(rpc.enabled()); + } + + @Test + public void executeTelnet() throws Exception { + final HistogramDataPointRpc rpc = new HistogramDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + + rpc.execute(tsdb, chan, new String[] { "histogram", METRIC_STRING, + "1356998400", "u=0:o=1:0,1=42:1,5=24", + TAGK_STRING + "=" + TAGV_STRING }) + .join(); + validateCounters(1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1); + verify(chan, never()).write(any()); + verify(chan, never()).isConnected(); + validateSEH(false); + + byte[] qualifier = new byte[] {0x06, 0, 0}; + byte[] value = storage.getColumn(getRowKey(METRIC_STRING, 1356998400, + TAGK_STRING, TAGV_STRING), qualifier); + assertArrayEquals(test_histo.histogram(true), value); + } + + @Test + public void executeTelnetBinary() throws Exception { + final HistogramDataPointRpc rpc = new HistogramDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + + rpc.execute(tsdb, chan, new String[] { "histogram", METRIC_STRING, + "1356998400", "42", + HistogramPojo.bytesToBase64String(test_histo.histogram(false)), + TAGK_STRING + "=" + TAGV_STRING }) + .join(); + validateCounters(1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1); + verify(chan, never()).write(any()); + verify(chan, never()).isConnected(); + validateSEH(false); + + byte[] qualifier = new byte[] {0x06, 0, 0}; + byte[] value = storage.getColumn(getRowKey(METRIC_STRING, 1356998400, + TAGK_STRING, TAGV_STRING), qualifier); + assertArrayEquals(test_histo.histogram(true), value); + } + + @Test + public void executeTelnetHistosDisabled() throws Exception { + config.overrideConfig("tsd.core.histograms.config", null); + final HistogramDataPointRpc rpc = new HistogramDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + + rpc.execute(tsdb, chan, new String[] { "histogram", METRIC_STRING, + "1356998400", "42", + HistogramPojo.bytesToBase64String(test_histo.histogram(false)), + TAGK_STRING + "=" + TAGV_STRING }) + .join(); + validateCounters(1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(false); + + byte[] qualifier = new byte[] {0x06, 0, 0}; + assertNull(storage.getColumn(getRowKey(METRIC_STRING, 1356998400, + TAGK_STRING, TAGV_STRING), qualifier)); + } + + @Test + public void executeTelnetBinaryValueTooShort() throws Exception { + final HistogramDataPointRpc rpc = new HistogramDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + + rpc.execute(tsdb, chan, new String[] { "histogram", METRIC_STRING, + "1356998400", "42", + HistogramPojo.bytesToBase64String(test_histo.histogram(false)) + .substring(0, 4), + TAGK_STRING + "=" + TAGV_STRING }) + .join(); + validateCounters(1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(false); + + byte[] qualifier = new byte[] {0x06, 0, 0}; + assertNull(storage.getColumn(getRowKey(METRIC_STRING, 1356998400, + TAGK_STRING, TAGV_STRING), qualifier)); + } + + @Test + public void executeTelnetBinaryCorruptValue() throws Exception { + final HistogramDataPointRpc rpc = new HistogramDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + + rpc.execute(tsdb, chan, new String[] { "histogram", METRIC_STRING, + "1356998400", "42", + HistogramPojo.bytesToBase64String(test_histo.histogram(false)) + .substring(0, 8), + TAGK_STRING + "=" + TAGV_STRING }) + .join(); + validateCounters(1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(false); + + byte[] qualifier = new byte[] {0x06, 0, 0}; + assertNull(storage.getColumn(getRowKey(METRIC_STRING, 1356998400, + TAGK_STRING, TAGV_STRING), qualifier)); + } + + @Test + public void executeTelnetHBaseError() throws Exception { + storage.throwException(getRowKey(METRIC_STRING, 1356998400, + TAGK_STRING, TAGV_STRING), mock(HBaseException.class)); + final HistogramDataPointRpc rpc = new HistogramDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + + rpc.execute(tsdb, chan, new String[] { "histogram", METRIC_STRING, + "1356998400", "42", + HistogramPojo.bytesToBase64String(test_histo.histogram(false)), + TAGK_STRING + "=" + TAGV_STRING }) + .join(); + validateCounters(1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(true); + + byte[] qualifier = new byte[] {0x06, 0, 0}; + assertNull(storage.getColumn(getRowKey(METRIC_STRING, 1356998400, + TAGK_STRING, TAGV_STRING), qualifier)); + } + + @Test + public void executeTelnetePleaseThrottle() throws Exception { + storage.throwException(getRowKey(METRIC_STRING, 1356998400, + TAGK_STRING, TAGV_STRING), mock(PleaseThrottleException.class)); + final HistogramDataPointRpc rpc = new HistogramDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + + rpc.execute(tsdb, chan, new String[] { "histogram", METRIC_STRING, + "1356998400", "42", + HistogramPojo.bytesToBase64String(test_histo.histogram(false)), + TAGK_STRING + "=" + TAGV_STRING }) + .join(); + validateCounters(1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(true); + + byte[] qualifier = new byte[] {0x06, 0, 0}; + assertNull(storage.getColumn(getRowKey(METRIC_STRING, 1356998400, + TAGK_STRING, TAGV_STRING), qualifier)); + } + + @Test (expected = NullPointerException.class) + public void executeTelnetNullTSDB() throws Exception { + final HistogramDataPointRpc rpc = new HistogramDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + + rpc.execute(null, chan, new String[] { "histogram", METRIC_STRING, + "1356998400", "42", + HistogramPojo.bytesToBase64String(test_histo.histogram(false)), + TAGK_STRING + "=" + TAGV_STRING }) + .join(); + } + + @Test + public void executeTelnetNullChannelOK() throws Exception { + final HistogramDataPointRpc rpc = new HistogramDataPointRpc(tsdb.getConfig()); + rpc.execute(tsdb, null, new String[] { "histogram", METRIC_STRING, + "1356998400", "42", + HistogramPojo.bytesToBase64String(test_histo.histogram(false)), + TAGK_STRING + "=" + TAGV_STRING }) + .join(); + validateCounters(1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1); + validateSEH(false); + } + + @Test (expected = NullPointerException.class) + public void executeNullChannelError() throws Exception { + final HistogramDataPointRpc rpc = new HistogramDataPointRpc(tsdb.getConfig()); + rpc.execute(tsdb, null, new String[] { "histogram" }) + .joinUninterruptibly(); + } + + @Test (expected = NullPointerException.class) + public void executeNullArray() throws Exception { + final HistogramDataPointRpc rpc = new HistogramDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + rpc.execute(tsdb, chan, null).joinUninterruptibly(); + } + + @Test (expected = ArrayIndexOutOfBoundsException.class) + public void executeEmptyArray() throws Exception { + final HistogramDataPointRpc rpc = new HistogramDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + rpc.execute(tsdb, chan, new String[0]).joinUninterruptibly(); + } + + @Test + public void executeShortArray() throws Exception { + final HistogramDataPointRpc rpc = new HistogramDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + rpc.execute(tsdb, chan, new String[] { "histogram", METRIC_STRING, + "1356998400", "42" }).joinUninterruptibly(); + validateCounters(1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(false); + } + + @Test + public void executeMissingTags() throws Exception { + final HistogramDataPointRpc rpc = new HistogramDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + rpc.execute(tsdb, chan, new String[] { "histogram", METRIC_STRING, + "1356998400", "42", + HistogramPojo.bytesToBase64String(test_histo.histogram(false)), + "" }).joinUninterruptibly(); + validateCounters(1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(false); + } + + @Test + public void executeNSUNTagk() throws Exception { + final HistogramDataPointRpc rpc = new HistogramDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + rpc.execute(tsdb, chan, new String[] { "histogram", METRIC_STRING, + "1356998400", "42", + HistogramPojo.bytesToBase64String(test_histo.histogram(false)), + NSUN_TAGK + "=" + TAGV_STRING }).joinUninterruptibly(); + validateCounters(1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(false); + } + + @Test + public void executeNSUNTagV() throws Exception { + final HistogramDataPointRpc rpc = new HistogramDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + rpc.execute(tsdb, chan, new String[] { "histogram", METRIC_STRING, + "1356998400", "42", + HistogramPojo.bytesToBase64String(test_histo.histogram(false)), + TAGK_STRING + "=" + NSUN_TAGV }).joinUninterruptibly(); + validateCounters(1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(false); + } + + @Test + public void executeHttpSingle() throws Exception { + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/histogram", + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1356998400," + + "\"overflow\":1,\"buckets\":{\"0,1\":42,\"1,5\":24}" + + ",\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + final HistogramDataPointRpc rpc = new HistogramDataPointRpc(tsdb.getConfig()); + rpc.execute(tsdb, query); + validateCounters(0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1); + assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateSEH(false); + + byte[] qualifier = new byte[] {0x06, 0, 0}; + byte[] value = storage.getColumn(getRowKey(METRIC_STRING, 1356998400, + TAGK_STRING, TAGV_STRING), qualifier); + assertArrayEquals(test_histo.histogram(true), value); + } + + @Test + public void executeHttpSingleBinary() throws Exception { + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/histogram", + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1356998400," + + "\"id\":42,\"value\":\"" + + HistogramPojo.bytesToBase64String(test_histo.histogram(false)) + + "\",\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + final HistogramDataPointRpc rpc = new HistogramDataPointRpc(tsdb.getConfig()); + rpc.execute(tsdb, query); + validateCounters(0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1); + assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateSEH(false); + + byte[] qualifier = new byte[] {0x06, 0, 0}; + byte[] value = storage.getColumn(getRowKey(METRIC_STRING, 1356998400, + TAGK_STRING, TAGV_STRING), qualifier); + assertArrayEquals(test_histo.histogram(true), value); + } + + @Test + public void executeHttpTwoBinary() throws Exception { + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/histogram", + "[{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1356998400," + + "\"id\":42,\"value\":\"" + + HistogramPojo.bytesToBase64String(test_histo.histogram(false)) + + "\",\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}," + + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1356998460," + + "\"id\":42,\"value\":\"" + + HistogramPojo.bytesToBase64String(test_histo.histogram(false)) + + "\",\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}]"); + final HistogramDataPointRpc rpc = new HistogramDataPointRpc(tsdb.getConfig()); + rpc.execute(tsdb, query); + validateCounters(0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2); + assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateSEH(false); + + byte[] qualifier = new byte[] {0x06, 0, 0}; + byte[] value = storage.getColumn(getRowKey(METRIC_STRING, 1356998400, + TAGK_STRING, TAGV_STRING), qualifier); + assertArrayEquals(test_histo.histogram(true), value); + + qualifier = new byte[] { 0x06, 0, 0x3C }; + value = storage.getColumn(getRowKey(METRIC_STRING, 1356998400, + TAGK_STRING, TAGV_STRING), qualifier); + assertArrayEquals(test_histo.histogram(true), value); + } + + @Test + public void executeHttpTwoOneGoodOneBadBinary() throws Exception { + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/histogram", + "[{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1356998400," + + "\"id\":42,\"value\":\"" + + HistogramPojo.bytesToBase64String(test_histo.histogram(false)) + + "\",\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}," + + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1356998460," + + "\"id\":42,\"value\":\"" + + HistogramPojo.bytesToBase64String(test_histo.histogram(false)) + .substring(0, 4) + + "\",\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}]"); + final HistogramDataPointRpc rpc = new HistogramDataPointRpc(tsdb.getConfig()); + rpc.execute(tsdb, query); + validateCounters(0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 1); + assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); + validateSEH(false); + + byte[] qualifier = new byte[] {0x06, 0, 0}; + byte[] value = storage.getColumn(getRowKey(METRIC_STRING, 1356998400, + TAGK_STRING, TAGV_STRING), qualifier); + assertArrayEquals(test_histo.histogram(true), value); + + qualifier = new byte[] { 0x06, 0, 0x3C }; + assertNull(storage.getColumn(getRowKey(METRIC_STRING, 1356998400, + TAGK_STRING, TAGV_STRING), qualifier)); + } + + @Test + public void httpNSUNMetric() throws Exception { + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/histogram", + "{\"metric\":\"" + NSUN_METRIC + "\",\"timestamp\":1356998400," + + "\"id\":42,\"value\":\"" + + HistogramPojo.bytesToBase64String(test_histo.histogram(false)) + + "\",\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + final HistogramDataPointRpc rpc = new HistogramDataPointRpc(tsdb.getConfig()); + rpc.execute(tsdb, query); + assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); + validateCounters(0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0); + validateSEH(false); + } + + @Test + public void httpNSUNTagk() throws Exception { + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/histogram", + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1356998400," + + "\"id\":42,\"value\":\"" + + HistogramPojo.bytesToBase64String(test_histo.histogram(false)) + + "\",\"tags\":{\"" + NSUN_TAGK + "\":\"" + TAGV_STRING + "\"}}"); + final HistogramDataPointRpc rpc = new HistogramDataPointRpc(tsdb.getConfig()); + rpc.execute(tsdb, query); + assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); + validateCounters(0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0); + validateSEH(false); + } + + @Test + public void httpNSUNTagv() throws Exception { + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/histogram", + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1356998400," + + "\"id\":42,\"value\":\"" + + HistogramPojo.bytesToBase64String(test_histo.histogram(false)) + + "\",\"tags\":{\"" + TAGK_STRING + "\":\"" + NSUN_TAGV + "\"}}"); + final HistogramDataPointRpc rpc = new HistogramDataPointRpc(tsdb.getConfig()); + rpc.execute(tsdb, query); + assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); + validateCounters(0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0); + validateSEH(false); + } + + @Test + public void httpHBaseError() throws Exception { + storage.throwException(getRowKey(METRIC_STRING, 1356998400, + TAGK_STRING, TAGV_STRING), mock(HBaseException.class)); + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/histogram", + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1356998400," + + "\"id\":42,\"value\":\"" + + HistogramPojo.bytesToBase64String(test_histo.histogram(false)) + + "\",\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + final HistogramDataPointRpc rpc = new HistogramDataPointRpc(tsdb.getConfig()); + rpc.execute(tsdb, query); + assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateCounters(0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0); + validateSEH(true); + } + + @Test + public void httpPleaseThrottleError() throws Exception { + storage.throwException(getRowKey(METRIC_STRING, 1356998400, + TAGK_STRING, TAGV_STRING), mock(PleaseThrottleException.class)); + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/histogram", + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1356998400," + + "\"id\":42,\"value\":\"" + + HistogramPojo.bytesToBase64String(test_histo.histogram(false)) + + "\",\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + final HistogramDataPointRpc rpc = new HistogramDataPointRpc(tsdb.getConfig()); + rpc.execute(tsdb, query); + assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateCounters(0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0); + validateSEH(true); + } + + @Override + protected void validateSEH(final boolean called) { + if (called) { + if (handler != null) { + verify(handler, times(1)).handleError((IncomingDataPoint)any(), + (Exception)any()); + } + } else { + if (handler != null) { + verify(handler, never()).handleError((IncomingDataPoint)any(), + (Exception)any()); + } + } + } + + protected void validateCounters( + final long telnet_requests, + final long http_requests, + final long raw_dps, + final long rollup_dps, + final long raw_stored, + final long rollup_stored, + final long hbase_errors, + final long unknown_errors, + final long invalid_values, + final long illegal_arguments, + final long unknown_metrics, + final long inflight_exceeded, + final long writes_blocked, + final long writes_timedout, + final long requests_timedout, + final long raw_histograms, + final long raw_histograms_stored) { + assertEquals(telnet_requests, this.telnet_requests.get()); + assertEquals(http_requests, this.http_requests.get()); + assertEquals(raw_dps, this.raw_dps.get()); + assertEquals(rollup_dps, this.rollup_dps.get()); + assertEquals(raw_stored, this.raw_stored.get()); + assertEquals(rollup_stored, this.rollup_stored.get()); + assertEquals(hbase_errors, this.hbase_errors.get()); + assertEquals(unknown_errors, this.unknown_errors.get()); + assertEquals(invalid_values, this.invalid_values.get()); + assertEquals(illegal_arguments, this.illegal_arguments.get()); + assertEquals(unknown_metrics, this.unknown_metrics.get()); + assertEquals(inflight_exceeded, this.inflight_exceeded.get()); + assertEquals(writes_blocked, this.writes_blocked.get()); + assertEquals(writes_timedout, this.writes_timedout.get()); + assertEquals(requests_timedout, this.requests_timedout.get()); + assertEquals(raw_histograms, this.raw_histograms.get()); + assertEquals(raw_histograms_stored, this.raw_histograms_stored.get()); + } +} diff --git a/test/tsd/TestHttpJsonSerializer.java b/test/tsd/TestHttpJsonSerializer.java index 4d4c355d62..fc8acad09d 100644 --- a/test/tsd/TestHttpJsonSerializer.java +++ b/test/tsd/TestHttpJsonSerializer.java @@ -13,32 +13,78 @@ package net.opentsdb.tsd; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.when; +import java.lang.Thread.State; +import java.lang.reflect.Field; import java.nio.charset.Charset; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import net.opentsdb.core.DataPoints; import net.opentsdb.core.TSDB; +import net.opentsdb.core.TSQuery; +import net.opentsdb.core.TSSubQuery; +import net.opentsdb.meta.Annotation; +import net.opentsdb.stats.QueryStats; +import net.opentsdb.storage.MockDataPoints; +import net.opentsdb.uid.NoSuchUniqueId; import net.opentsdb.utils.Config; +import net.opentsdb.utils.DateTime; import org.jboss.netty.buffer.ChannelBuffer; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import com.google.common.cache.CacheBuilder; +import com.stumbleupon.async.Deferred; +import com.stumbleupon.async.DeferredGroupException; + /** * Unit tests for the JSON serializer. * <b>Note:</b> Tests for the default error handlers are in the TestHttpQuery * class */ @RunWith(PowerMockRunner.class) -@PrepareForTest({TSDB.class, Config.class, HttpQuery.class}) +@PrepareForTest({ HttpJsonSerializer.class, TSDB.class, Config.class, + HttpQuery.class, TSQuery.class, TSSubQuery.class, QueryStats.class, + DateTime.class }) public final class TestHttpJsonSerializer { private TSDB tsdb = null; + private final List<Long> timestamp = new ArrayList<Long>(1); + private static String remote = "192.168.1.1:4242"; + private static Field running_queries; + static { + try { + running_queries = QueryStats.class.getDeclaredField("running_queries"); + running_queries.setAccessible(true); + } catch (Exception e) { + throw new RuntimeException("Failed in static initializer", e); + } + } + private static Field completed_queries; + static { + try { + completed_queries = QueryStats.class.getDeclaredField("completed_queries"); + completed_queries.setAccessible(true); + } catch (Exception e) { + throw new RuntimeException("Failed in static initializer", e); + } + } @Before public void before() throws Exception { @@ -117,6 +163,37 @@ public void parseSuggestV1NotJSON() throws Exception { serdes.parseSuggestV1(); } + @Test + public void parseUidRenameV1() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "", + "{\"metric\":\"sys.cpu.1\",\"name\":\"sys.cpu.2\"}", ""); + HttpJsonSerializer serdes = new HttpJsonSerializer(query); + HashMap<String, String> map = serdes.parseUidRenameV1(); + assertNotNull(map); + assertEquals("sys.cpu.1", map.get("metric")); + } + + @Test (expected = BadRequestException.class) + public void parseUidRenameV1NoContent() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "", null, ""); + HttpJsonSerializer serdes = new HttpJsonSerializer(query); + serdes.parseUidRenameV1(); + } + + @Test (expected = BadRequestException.class) + public void parseUidRenameV1EmptyContent() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "", "", ""); + HttpJsonSerializer serdes = new HttpJsonSerializer(query); + serdes.parseUidRenameV1(); + } + + @Test (expected = BadRequestException.class) + public void parseUidRenameV1NotJSON() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "", "NOT JSON", ""); + HttpJsonSerializer serdes = new HttpJsonSerializer(query); + serdes.parseUidRenameV1(); + } + @Test public void formatSuggestV1() throws Exception { HttpQuery query = NettyMocks.getQuery(tsdb, ""); @@ -148,13 +225,445 @@ public void formatSuggestV1Null() throws Exception { serdes.formatSuggestV1(null); } + @Test + public void formatUidRenameV1Success() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, ""); + HttpJsonSerializer serdes = new HttpJsonSerializer(query); + final HashMap<String, String> map = new HashMap<String, String>(2); + map.put("result", "true"); + ChannelBuffer cb = serdes.formatUidRenameV1(map); + assertNotNull(cb); + assertEquals("{\"result\":\"true\"}", + cb.toString(Charset.forName("UTF-8"))); + } + + @Test + public void formatUidRenameV1Failed() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, ""); + HttpJsonSerializer serdes = new HttpJsonSerializer(query); + final HashMap<String, String> map = new HashMap<String, String>(2); + map.put("result", "false"); + map.put("error", "known"); + ChannelBuffer cb = serdes.formatUidRenameV1(map); + assertNotNull(cb); + final String json = cb.toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"error\":\"known\"")); + assertTrue(json.contains("\"result\":\"false\"")); + } + + @Test (expected = IllegalArgumentException.class) + public void formatUidRenameV1Null() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, ""); + HttpJsonSerializer serdes = new HttpJsonSerializer(query); + serdes.formatUidRenameV1(null); + } + @Test public void formatSerializersV1() throws Exception { HttpQuery.initializeSerializerMaps(tsdb); HttpQuery query = NettyMocks.getQuery(tsdb, ""); HttpJsonSerializer serdes = new HttpJsonSerializer(query); - assertEquals("[{\"formatters\":", - serdes.formatSerializersV1().toString(Charset.forName("UTF-8")) - .substring(0, 15)); + + String json = serdes.formatSerializersV1().toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"request_content_type\":\"application/json\"")); + assertTrue(json.contains("\"formatters\":[")); + assertTrue(json.contains("\"response_content_type\":\"application/json; " + + "charset=UTF-8\"")); + assertTrue(json.contains("\"parsers\":[")); + assertTrue(json.contains("\"serializer\":\"json\"")); + assertTrue(json.contains("\"class\":\"net.opentsdb.tsd.HttpJsonSerializer\"")); + } + + @Test + public void formatQueryAsyncV1() throws Exception { + setupFormatQuery(); + HttpQuery query = NettyMocks.getQuery(tsdb, ""); + HttpJsonSerializer serdes = new HttpJsonSerializer(query); + final TSQuery data_query = getTestQuery(false); + validateTestQuery(data_query); + final List<DataPoints[]> results = new ArrayList<DataPoints[]>(1); + results.add(new DataPoints[] { new MockDataPoints().getMock() }); + + ChannelBuffer cb = serdes.formatQueryAsyncV1(data_query, results, + Collections.<Annotation> emptyList()).joinUninterruptibly(); + assertNotNull(cb); + final String json = cb.toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"metric\":\"system.cpu.user\",")); + assertTrue(json.contains("\"1356998700\":1,")); + assertTrue(json.contains("\"1357058700\":201")); + assertFalse(json.contains("\"timeTotal\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertFalse(json.contains("\"query\":")); + } + + @Test + public void formatQueryAsyncV1wQuery() throws Exception { + setupFormatQuery(); + HttpQuery query = NettyMocks.getQuery(tsdb, ""); + HttpJsonSerializer serdes = new HttpJsonSerializer(query); + final TSQuery data_query = getTestQuery(false); + data_query.setShowQuery(true); + validateTestQuery(data_query); + final List<DataPoints[]> results = new ArrayList<DataPoints[]>(1); + results.add(new DataPoints[] { new MockDataPoints().getMock() }); + + ChannelBuffer cb = serdes.formatQueryAsyncV1(data_query, results, + Collections.<Annotation> emptyList()).joinUninterruptibly(); + assertNotNull(cb); + final String json = cb.toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"metric\":\"system.cpu.user\",")); + assertTrue(json.contains("\"1356998700\":1,")); + assertTrue(json.contains("\"1357058700\":201")); + assertFalse(json.contains("\"timeTotal\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"query\":")); } + + @Test + public void formatQueryAsyncV1wStatsSummary() throws Exception { + setupFormatQuery(); + final HttpQuery query = NettyMocks.getQuery(tsdb, ""); + final HttpJsonSerializer serdes = new HttpJsonSerializer(query); + final TSQuery data_query = getTestQuery(true, true); + validateTestQuery(data_query); + final List<DataPoints[]> results = new ArrayList<DataPoints[]>(1); + results.add(new DataPoints[] { new MockDataPoints().getMock() }); + + final ChannelBuffer cb = serdes.formatQueryAsyncV1(data_query, results, + Collections.<Annotation> emptyList()).joinUninterruptibly(); + assertNotNull(cb); + final String json = cb.toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"metric\":\"system.cpu.user\",")); + assertTrue(json.contains("\"1356998700\":1,")); + assertTrue(json.contains("\"1357058700\":201")); + //assert stats + assertTrue(json.contains("\"stats\":{")); + assertTrue(json.contains("\"emittedDPs\":401")); + System.out.println(json); + //assert stats summary + assertTrue(json.contains("{\"statsSummary\":{")); + assertTrue(json.contains("\"serializationTime\":")); + assertTrue(json.contains("\"processingPreWriteTime\":")); + assertTrue(json.contains("\"queryIdx_00\":")); + } + + @Test + public void formatQueryAsyncV1wStatsWoSummary() throws Exception { + setupFormatQuery(); + final HttpQuery query = NettyMocks.getQuery(tsdb, ""); + final HttpJsonSerializer serdes = new HttpJsonSerializer(query); + final TSQuery data_query = getTestQuery(true, false); + validateTestQuery(data_query); + final List<DataPoints[]> results = new ArrayList<DataPoints[]>(1); + results.add(new DataPoints[] { new MockDataPoints().getMock() }); + + final ChannelBuffer cb = serdes.formatQueryAsyncV1(data_query, results, + Collections.<Annotation> emptyList()).joinUninterruptibly(); + assertNotNull(cb); + final String json = cb.toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"metric\":\"system.cpu.user\",")); + assertTrue(json.contains("\"stats\":{")); + assertTrue(json.contains("\"1356998700\":1,")); + assertTrue(json.contains("\"1357058700\":201")); + + //assert stats + assertTrue(json.contains("\"stats\":{")); + assertTrue(json.contains("\"emittedDPs\":401")); + + //assert stats summary + assertFalse(json.contains("{\"statsSummary\":{")); + } + + @Test + public void formatQueryAsyncV1woStatsWSummary() throws Exception { + setupFormatQuery(); + final HttpQuery query = NettyMocks.getQuery(tsdb, ""); + final HttpJsonSerializer serdes = new HttpJsonSerializer(query); + final TSQuery data_query = getTestQuery(false, true); + validateTestQuery(data_query); + final List<DataPoints[]> results = new ArrayList<DataPoints[]>(1); + results.add(new DataPoints[] { new MockDataPoints().getMock() }); + + final ChannelBuffer cb = serdes.formatQueryAsyncV1(data_query, results, + Collections.<Annotation> emptyList()).joinUninterruptibly(); + assertNotNull(cb); + final String json = cb.toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"metric\":\"system.cpu.user\",")); + assertTrue(json.contains("\"1356998700\":1,")); + assertTrue(json.contains("\"1357058700\":201")); + + //assert stats + assertFalse(json.contains("\"stats\":{")); + + //assert stats summary + assertTrue(json.contains("{\"statsSummary\":{")); + assertTrue(json.contains("\"serializationTime\":")); + assertTrue(json.contains("\"processingPreWriteTime\":")); + assertTrue(json.contains("\"emittedDPs\":401")); + assertTrue(json.contains("\"queryIdx_00\":")); + } + + @Test + public void formatQueryAsyncV1woStatsWoSummary() throws Exception { + setupFormatQuery(); + final HttpQuery query = NettyMocks.getQuery(tsdb, ""); + final HttpJsonSerializer serdes = new HttpJsonSerializer(query); + final TSQuery data_query = getTestQuery(false, false); + validateTestQuery(data_query); + final List<DataPoints[]> results = new ArrayList<DataPoints[]>(1); + results.add(new DataPoints[] { new MockDataPoints().getMock() }); + + final ChannelBuffer cb = serdes.formatQueryAsyncV1(data_query, results, + Collections.<Annotation> emptyList()).joinUninterruptibly(); + assertNotNull(cb); + final String json = cb.toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"metric\":\"system.cpu.user\",")); + assertTrue(json.contains("\"1356998700\":1,")); + assertTrue(json.contains("\"1357058700\":201")); + + //assert stats + assertFalse(json.contains("\"stats\":{")); + + //assert stats summary + assertFalse(json.contains("{\"statsSummary\":{")); + } + + @Test + public void formatQueryAsyncTimeFilterV1() throws Exception { + setupFormatQuery(); + HttpQuery query = NettyMocks.getQuery(tsdb, ""); + HttpJsonSerializer serdes = new HttpJsonSerializer(query); + final TSQuery data_query = getTestQuery(false, false); + final List<DataPoints[]> results = new ArrayList<DataPoints[]>(1); + results.add(new DataPoints[] { new MockDataPoints().getMock() }); + + data_query.setEnd("1357000500"); + validateTestQuery(data_query); + + ChannelBuffer cb = serdes.formatQueryAsyncV1(data_query, results, + Collections.<Annotation> emptyList()).joinUninterruptibly(); + assertNotNull(cb); + final String json = cb.toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"metric\":\"system.cpu.user\",")); + assertTrue(json.contains("\"1356998700\":1,")); + assertTrue(json.contains("\"1357000500\":7")); + } + + @Test + public void formatQueryAsyncV1EmptyDPs() throws Exception { + setupFormatQuery(); + HttpQuery query = NettyMocks.getQuery(tsdb, ""); + HttpJsonSerializer serdes = new HttpJsonSerializer(query); + final TSQuery data_query = getTestQuery(false); + validateTestQuery(data_query); + final List<DataPoints[]> results = new ArrayList<DataPoints[]>(1); + + ChannelBuffer cb = serdes.formatQueryAsyncV1(data_query, results, + Collections.<Annotation> emptyList()).joinUninterruptibly(); + assertNotNull(cb); + final String json = cb.toString(Charset.forName("UTF-8")); + assertEquals("[]", json); + } + + @Test (expected = DeferredGroupException.class) + public void formatQueryAsyncV1NoSuchMetricId() throws Exception { + setupFormatQuery(); + HttpQuery query = NettyMocks.getQuery(tsdb, ""); + HttpJsonSerializer serdes = new HttpJsonSerializer(query); + final TSQuery data_query = getTestQuery(false); + validateTestQuery(data_query); + + final DataPoints dps = new MockDataPoints().getMock(); + final List<DataPoints[]> results = new ArrayList<DataPoints[]>(1); + results.add(new DataPoints[] { dps }); + + when(dps.metricNameAsync()) + .thenReturn(Deferred.<String>fromError( + new NoSuchUniqueId("No such metric", new byte[] { 0, 0, 1 }))); + + serdes.formatQueryAsyncV1(data_query, results, + Collections.<Annotation> emptyList()).joinUninterruptibly(); + } + + @Test (expected = DeferredGroupException.class) + public void formatQueryAsyncV1NoSuchTagId() throws Exception { + setupFormatQuery(); + HttpQuery query = NettyMocks.getQuery(tsdb, ""); + HttpJsonSerializer serdes = new HttpJsonSerializer(query); + final TSQuery data_query = getTestQuery(false); + validateTestQuery(data_query); + + final DataPoints dps = new MockDataPoints().getMock(); + final List<DataPoints[]> results = new ArrayList<DataPoints[]>(1); + results.add(new DataPoints[] { dps }); + + when(dps.getTagsAsync()) + .thenReturn(Deferred.<Map<String, String>>fromError( + new NoSuchUniqueId("No such tagv", new byte[] { 0, 0, 1 }))); + + serdes.formatQueryAsyncV1(data_query, results, + Collections.<Annotation> emptyList()).joinUninterruptibly(); + } + + @Test (expected = DeferredGroupException.class) + public void formatQueryAsyncV1NoSuchAggTagId() throws Exception { + setupFormatQuery(); + HttpQuery query = NettyMocks.getQuery(tsdb, ""); + HttpJsonSerializer serdes = new HttpJsonSerializer(query); + final TSQuery data_query = getTestQuery(false); + validateTestQuery(data_query); + + final DataPoints dps = new MockDataPoints().getMock(); + final List<DataPoints[]> results = new ArrayList<DataPoints[]>(1); + results.add(new DataPoints[] { dps }); + + when(dps.getAggregatedTagsAsync()) + .thenReturn(Deferred.<List<String>>fromError( + new NoSuchUniqueId("No such tagk", new byte[] { 0, 0, 1 }))); + + serdes.formatQueryAsyncV1(data_query, results, + Collections.<Annotation> emptyList()).joinUninterruptibly(); + } + + @Test (expected = NullPointerException.class) + public void formatQueryAsyncV1NullIterator() throws Exception { + setupFormatQuery(); + HttpQuery query = NettyMocks.getQuery(tsdb, ""); + HttpJsonSerializer serdes = new HttpJsonSerializer(query); + final TSQuery data_query = getTestQuery(false); + validateTestQuery(data_query); + + final DataPoints dps = new MockDataPoints().getMock(); + final List<DataPoints[]> results = new ArrayList<DataPoints[]>(1); + results.add(new DataPoints[] { dps }); + + when(dps.iterator()).thenReturn(null); + + serdes.formatQueryAsyncV1(data_query, results, + Collections.<Annotation> emptyList()).joinUninterruptibly(); + } + + @Test (expected = RuntimeException.class) + public void formatQueryAsyncV1UnexpectedAggException() throws Exception { + setupFormatQuery(); + HttpQuery query = NettyMocks.getQuery(tsdb, ""); + HttpJsonSerializer serdes = new HttpJsonSerializer(query); + final TSQuery data_query = getTestQuery(false); + validateTestQuery(data_query); + + final MockDataPoints mdps = new MockDataPoints(); + final List<DataPoints[]> results = new ArrayList<DataPoints[]>(1); + results.add(new DataPoints[] { mdps.getMock() }); + + when(mdps.getMockDP().timestamp()).thenThrow( + new RuntimeException("Unexpected error")); + + serdes.formatQueryAsyncV1(data_query, results, + Collections.<Annotation> emptyList()).joinUninterruptibly(); + } + + @Test + public void formatThreadStats() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, ""); + HttpJsonSerializer serdes = new HttpJsonSerializer(query); + + final List<Map<String, Object>> output = + new ArrayList<Map<String, Object>>(1); + Map<String, Object> status = new HashMap<String, Object>(); + status.put("threadID", 1); + status.put("name", "Test Thread 1"); + status.put("state", State.RUNNABLE); + status.put("interrupted", false); + status.put("priority", 1); + + List<String> stack = new ArrayList<String>(2); + stack.add("net.opentsdb.tsd(TestHttpJsonSerializer.java:0)"); + stack.add("java.lang.Thread.run(Thread.java:695)"); + status.put("stack", stack); + output.add(status); + + ChannelBuffer cb = serdes.formatThreadStatsV1(output); + assertNotNull(cb); + final String json = cb.toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"threadID\":1")); + assertTrue(json.contains("\"name\":\"Test Thread 1\"")); + } + + @Test (expected = IllegalArgumentException.class) + public void formatThreadStatsNull() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, ""); + HttpJsonSerializer serdes = new HttpJsonSerializer(query); + serdes.formatThreadStatsV1(null); + } + + /** + * Helper to reset the query stats and mock the time calls before each + * data point query. + */ + private void setupFormatQuery() throws Exception { + mockTime(); + running_queries.set(null, new ConcurrentHashMap<Integer, QueryStats>()); + completed_queries.set(null, CacheBuilder.newBuilder().maximumSize(2).build()); + } + + /** @return Returns a test TSQuery object to pass on to the serializer */ + private TSQuery getTestQuery(final boolean show_stats) { + return getTestQuery(show_stats, false); + } + + /** @return Returns a test TSQuery object to pass on to the serializer */ + private TSQuery getTestQuery(final boolean show_stats, final boolean show_summary) { + final TSQuery data_query = new TSQuery(); + data_query.setStart("1356998400"); + data_query.setEnd("1388534400"); + data_query.setShowStats(show_stats); + data_query.setShowSummary(show_summary); + + final TSSubQuery sub_query = new TSSubQuery(); + sub_query.setMetric("sys.cpu.user"); + sub_query.setAggregator("sum"); + final ArrayList<TSSubQuery> sub_queries = new ArrayList<TSSubQuery>(1); + sub_queries.add(sub_query); + data_query.setQueries(sub_queries); + + return data_query; + } + + /** + * Helper to validate (set) the time series query + * @param data_query The query to validate + */ + private void validateTestQuery(final TSQuery data_query) { + data_query.validateAndSetQuery(); + data_query.setQueryStats(new QueryStats(remote, data_query, null)); + } + + /** + * Mocks out the DateTime class and increments the timestamp by 500ms every + * time it's called. + */ + private void mockTime() { + timestamp.add(1388534400000L); + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.parseDateTimeString(anyString(), anyString())) + .thenCallRealMethod(); + PowerMockito.when(DateTime.currentTimeMillis()) + .thenAnswer(new Answer<Long> () { + public Long answer(InvocationOnMock invocation) throws Throwable { + long ts = timestamp.get(0); + timestamp.set(0, ts + 500); + return ts; + } + }); + + PowerMockito.when(DateTime.nanoTime()) + .thenAnswer(new Answer<Long> () { + public Long answer(InvocationOnMock invocation) throws Throwable { + long ts = timestamp.get(0); + timestamp.set(0, ts + 500); + return ts * 1000000; + } + }); + } + } diff --git a/test/tsd/TestHttpQuery.java b/test/tsd/TestHttpQuery.java index ab1a64800a..1efa626145 100644 --- a/test/tsd/TestHttpQuery.java +++ b/test/tsd/TestHttpQuery.java @@ -20,6 +20,8 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.when; import java.lang.reflect.Method; import java.nio.charset.Charset; @@ -34,6 +36,8 @@ import org.jboss.netty.buffer.ChannelBuffer; import org.jboss.netty.buffer.ChannelBuffers; import org.jboss.netty.channel.Channel; +import org.jboss.netty.channel.ChannelFuture; +import org.jboss.netty.channel.DefaultChannelFuture; import org.jboss.netty.handler.codec.http.DefaultHttpRequest; import org.jboss.netty.handler.codec.http.HttpMethod; import org.jboss.netty.handler.codec.http.HttpRequest; @@ -787,9 +791,21 @@ public void internalErrorDeprecated() { assertEquals(HttpResponseStatus.INTERNAL_SERVER_ERROR, query.response().getStatus()); assertEquals( - "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\">", + "<!DOCTYPE html>", query.response().getContent().toString(Charset.forName("UTF-8")) - .substring(0, 63)); + .substring(0, 15)); + } + + @Test + public void internalErrorDeprecatedHTMLEscaped() { + HttpQuery query = NettyMocks.getQuery(tsdb, ""); + query.internalError(new Exception("<script>alert(document.cookie)</script>")); + + assertEquals(HttpResponseStatus.INTERNAL_SERVER_ERROR, + query.response().getStatus()); + assertTrue(query.response().getContent().toString(Charset.forName("UTF-8")).contains( + "<script>alert(document.cookie)</script>" + )); } @Test @@ -841,9 +857,20 @@ public void badRequestDeprecated() { } assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); assertEquals( - "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\">", + "<!DOCTYPE html>", query.response().getContent().toString(Charset.forName("UTF-8")) - .substring(0, 63)); + .substring(0, 15)); + } + + @Test + public void badRequestDeprecatedHTMLEscaped() { + HttpQuery query = NettyMocks.getQuery(tsdb, "/"); + query.badRequest(new BadRequestException("<script>alert(document.cookie)</script>")); + + assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); + assertTrue(query.response().getContent().toString(Charset.forName("UTF-8")).contains( + "The reason provided was:<blockquote><script>alert(document.cookie)</script>" + )); } @Test @@ -922,9 +949,9 @@ public void badRequestDeprecatedString() { query.badRequest("Bad user error"); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); assertEquals( - "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\">", + "<!DOCTYPE html>", query.response().getContent().toString(Charset.forName("UTF-8")) - .substring(0, 63)); + .substring(0, 15)); } @Test @@ -963,9 +990,9 @@ public void notFoundDeprecated() { query.notFound(); assertEquals(HttpResponseStatus.NOT_FOUND, query.response().getStatus()); assertEquals( - "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\">", + "<!DOCTYPE html>", query.response().getContent().toString(Charset.forName("UTF-8")) - .substring(0, 63)); + .substring(0, 15)); } @Test @@ -1205,5 +1232,12 @@ public void getSerializerStatus() throws Exception { HttpQuery.initializeSerializerMaps(tsdb); assertNotNull(HttpQuery.getSerializerStatus()); } - + + /** @param the query to mock a future callback for */ + public static void mockChannelFuture(final HttpQuery query) { + final ChannelFuture future = new DefaultChannelFuture(query.channel(), false); + when(query.channel().write(any(ChannelBuffer.class))).thenReturn(future); + future.setSuccess(); + } + } diff --git a/test/tsd/TestHttpRpcPluginQuery.java b/test/tsd/TestHttpRpcPluginQuery.java new file mode 100644 index 0000000000..278f7da520 --- /dev/null +++ b/test/tsd/TestHttpRpcPluginQuery.java @@ -0,0 +1,73 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2011-2014 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.tsd; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; + +import org.jboss.netty.channel.Channel; +import org.jboss.netty.handler.codec.http.DefaultHttpRequest; +import org.jboss.netty.handler.codec.http.HttpMethod; +import org.jboss.netty.handler.codec.http.HttpRequest; +import org.jboss.netty.handler.codec.http.HttpVersion; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import net.opentsdb.core.TSDB; +import net.opentsdb.utils.Config; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({TSDB.class, Config.class, HttpRpcPluginQuery.class}) +public final class TestHttpRpcPluginQuery { + private TSDB mockTsdb; + private Channel mockChannel; + + @Before + public void before() { + mockTsdb = mock(TSDB.class); + mockChannel = NettyMocks.fakeChannel(); + } + + @Test + public void getQueryBaseRoute() { + assertEquals("test", makeQuery("/plugin/test/this/path").getQueryBaseRoute()); + assertEquals("test", makeQuery("/plugin/test/").getQueryBaseRoute()); + assertEquals("test", makeQuery("/plugin/test?some=else&this=that").getQueryBaseRoute()); + } + + @Test(expected=BadRequestException.class) + public void getQueryBaseRouteNoSlash() { + makeQuery("plugin/test?some=else&this=that").getQueryBaseRoute(); + } + + @Test(expected=BadRequestException.class) + public void getQueryBaseRouteNoPluginBase() { + makeQuery("/test?some=else&this=that").getQueryBaseRoute(); + } + + @Test(expected=BadRequestException.class) + public void getQueryBaseRouteNoPath() { + makeQuery("/plugin?some=else&this=that").getQueryBaseRoute(); + } + + private HttpRpcPluginQuery makeQuery(final String uriString) { + HttpRequest req = new DefaultHttpRequest( + HttpVersion.HTTP_1_1, + HttpMethod.GET, + uriString); + return new HttpRpcPluginQuery(mockTsdb, req, mockChannel); + } +} \ No newline at end of file diff --git a/test/tsd/TestPutRpc.java b/test/tsd/TestPutRpc.java index 983594cf49..13a409b7a0 100644 --- a/test/tsd/TestPutRpc.java +++ b/test/tsd/TestPutRpc.java @@ -1,5 +1,5 @@ // This file is part of OpenTSDB. -// Copyright (C) 2013 The OpenTSDB Authors. +// Copyright (C) 2013-2016 The OpenTSDB Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU Lesser General Public License as published by @@ -12,62 +12,281 @@ // see <http://www.gnu.org/licenses/>. package net.opentsdb.tsd; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.nio.charset.Charset; import java.util.HashMap; -import net.opentsdb.core.TSDB; -import net.opentsdb.uid.NoSuchUniqueName; -import net.opentsdb.utils.Config; - +import org.hbase.async.HBaseException; +import org.hbase.async.PleaseThrottleException; +import org.hbase.async.PutRequest; +import org.jboss.netty.channel.Channel; import org.jboss.netty.handler.codec.http.HttpResponseStatus; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.modules.junit4.PowerMockRunner; import com.stumbleupon.async.Deferred; @RunWith(PowerMockRunner.class) -@PrepareForTest({TSDB.class, Config.class, HttpQuery.class}) -public final class TestPutRpc { - private TSDB tsdb = null; - - @Before - public void before() throws Exception { - tsdb = NettyMocks.getMockedHTTPTSDB(); - final HashMap<String, String> tags1 = new HashMap<String, String>(); - tags1.put("host", "web01"); - when(tsdb.addPoint("sys.cpu.nice", 1365465600, 42, tags1)) - .thenReturn(Deferred.fromResult(new Object())); - when(tsdb.addPoint("sys.cpu.nice", 1365465600, -42, tags1)) - .thenReturn(Deferred.fromResult(new Object())); - when(tsdb.addPoint("sys.cpu.nice", 1365465600, 42.2f, tags1)) - .thenReturn(Deferred.fromResult(new Object())); - when(tsdb.addPoint("sys.cpu.nice", 1365465600, -42.2f, tags1)) - .thenReturn(Deferred.fromResult(new Object())); - when(tsdb.addPoint("sys.cpu.nice", 1365465600, 4220.0f, tags1)) - .thenReturn(Deferred.fromResult(new Object())); - when(tsdb.addPoint("sys.cpu.nice", 1365465600, -4220.0f, tags1)) - .thenReturn(Deferred.fromResult(new Object())); - when(tsdb.addPoint("sys.cpu.nice", 1365465600, .0042f, tags1)) - .thenReturn(Deferred.fromResult(new Object())); - when(tsdb.addPoint("sys.cpu.nice", 1365465600, -0.0042f, tags1)) - .thenReturn(Deferred.fromResult(new Object())); - when(tsdb.addPoint("sys.cpu.system", 1365465600, 24, tags1)) - .thenReturn(Deferred.fromResult(new Object())); - when(tsdb.addPoint("doesnotexist", 1365465600, 42, tags1)) - .thenThrow(new NoSuchUniqueName("metric", "doesnotexist")); - } +//"Classloader hell"... It's real. Tell PowerMock to ignore these classes +//because they fiddle with the class loader. We don't test them anyway. +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +public final class TestPutRpc extends BaseTestPutRpc { @Test public void constructor() { - assertNotNull(new PutDataPointRpc()); + assertNotNull(new PutDataPointRpc(tsdb.getConfig())); + } + + // Socket RPC Tests ------------------------------------ + + @Test + public void execute() throws Exception { + final PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + put.execute(tsdb, chan, new String[] { "put", METRIC_STRING, + "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + verify(chan, never()).write(any()); + verify(chan, never()).isConnected(); + validateSEH(false); + } + + @Test + public void executeBadValue() throws Exception { + final PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + put.execute(tsdb, chan, new String[] { "put", METRIC_STRING, + "1365465600", "notanum", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(false); + } + + @Test + public void executeMissingMetric() throws Exception { + final PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + put.execute(tsdb, chan, new String[] { "put", "", + "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(false); + } + + @Test + public void executeMissingMetricNotWriteable() throws Exception { + final PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + when(chan.isWritable()).thenReturn(false); + put.execute(tsdb, chan, new String[] { "put", "", + "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0); + verify(chan, never()).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(false); + } + + @Test + public void executeUnknownMetric() throws Exception { + final PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + put.execute(tsdb, chan, new String[] { "put", NSUN_METRIC, + "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(false); + } + + @SuppressWarnings("unchecked") + @Test (expected = RuntimeException.class) + public void executeRuntimeException() throws Exception { + when(tsdb.addPoint(anyString(), anyLong(), anyLong(), + (HashMap<String, String>)any())) + .thenThrow(new RuntimeException("Fail!")); + + final PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + put.execute(tsdb, chan, new String[] { "put", METRIC_STRING, + "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }); + validateCounters(1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0); + validateSEH(true); + } + + @Test + public void executeHBaseError() throws Exception { + when(client.put(any(PutRequest.class))) + .thenReturn(Deferred.fromError(mock(HBaseException.class))); + + final PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + put.execute(tsdb, chan, new String[] { "put", METRIC_STRING, + "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(true); + } + + @Test + public void executeHBaseErrorNotWriteable() throws Exception { + when(client.put(any(PutRequest.class))) + .thenReturn(Deferred.fromError(mock(HBaseException.class))); + + final PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + when(chan.isWritable()).thenReturn(false); + put.execute(tsdb, chan, new String[] { "put", METRIC_STRING, + "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0); + verify(chan, never()).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(true); + } + + @Test + public void executeHBaseErrorHandler() throws Exception { + when(client.put(any(PutRequest.class))) + .thenReturn(Deferred.fromError(mock(HBaseException.class))); + setStorageExceptionHandler(); + + final PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + put.execute(tsdb, chan, new String[] { "put", METRIC_STRING, + "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(true); + } + + @Test + public void executePleaseThrottle() throws Exception { + when(client.put(any(PutRequest.class))) + .thenReturn(Deferred.fromError(mock(PleaseThrottleException.class))); + final PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + put.execute(tsdb, chan, new String[] { "put", METRIC_STRING, + "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(true); + } + + @Test + public void executePleaseThrottleNotWriteable() throws Exception { + when(client.put(any(PutRequest.class))) + .thenReturn(Deferred.fromError(mock(PleaseThrottleException.class))); + final PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + when(chan.isWritable()).thenReturn(false); + put.execute(tsdb, chan, new String[] { "put", METRIC_STRING, + "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0); + verify(chan, never()).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(true); + } + + @Test + public void executePleaseThrottleHandler() throws Exception { + setStorageExceptionHandler(); + when(client.put(any(PutRequest.class))) + .thenReturn(Deferred.fromError(mock(PleaseThrottleException.class))); + final PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + put.execute(tsdb, chan, new String[] { "put", METRIC_STRING, + "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(true); + } + + @Test (expected = NullPointerException.class) + public void executeNullTSDB() throws Exception { + final PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + put.execute(null, chan, new String[] { "put", METRIC_STRING, + "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }); + } + + @Test + public void executeNullChannelOK() throws Exception { + // we can pass in a null channel but since we only write when an error occurs + // then we won't fail. + final PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + put.execute(tsdb, null, new String[] { "put", METRIC_STRING, + "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + assertEquals(1, telnet_requests.get()); + assertEquals(0, invalid_values.get()); + validateCounters(1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); + } + + @Test (expected = NullPointerException.class) + public void executeNullChannelError() throws Exception { + final PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + put.execute(tsdb, null, new String[] { "put", METRIC_STRING, + "1365465600", "notanumber", TAGK_STRING + "=" + TAGV_STRING }); + } + + @Test (expected = NullPointerException.class) + public void executeNullArray() throws Exception { + final PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + put.execute(tsdb, chan, null); + } + + @Test (expected = ArrayIndexOutOfBoundsException.class) + public void executeEmptyArray() throws Exception { + final PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + put.execute(tsdb, chan, new String[0]); + } + + @Test + public void executeShortArray() throws Exception { + final PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + put.execute(tsdb, chan, new String[] { "put", METRIC_STRING, + "1365465600", "42" }).joinUninterruptibly(); + validateCounters(1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(false); } // HTTP RPC Tests -------------------------------------- @@ -75,45 +294,52 @@ public void constructor() { @Test public void putSingle() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":42,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateCounters(0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); } - + @Test - public void putDouble() throws Exception { + public void putTwo() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put", - "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - + ":42,\"tags\":{\"host\":\"web01\"}},{\"metric\":\"sys.cpu.system\"," + "[{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + + ":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + + "\"}},{\"metric\":\"" + METRIC_B_STRING + "\"," + "\"timestamp\":1365465600,\"value\":24,\"tags\":" - + "{\"host\":\"web01\"}}]"); - PutDataPointRpc put = new PutDataPointRpc(); + + "{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}]"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateCounters(0, 1, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void putSingleSummary() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?summary", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":42,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); final String response = query.response().getContent().toString(Charset.forName("UTF-8")); assertTrue(response.contains("\"failed\":0")); assertTrue(response.contains("\"success\":1")); + validateCounters(0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void putSingleDetails() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":42,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); final String response = @@ -121,14 +347,16 @@ public void putSingleDetails() throws Exception { assertTrue(response.contains("\"failed\":0")); assertTrue(response.contains("\"success\":1")); assertTrue(response.contains("\"errors\":[]")); + validateCounters(0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void putSingleSummaryAndDetails() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?summary&details", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":42,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); final String response = @@ -136,173 +364,302 @@ public void putSingleSummaryAndDetails() throws Exception { assertTrue(response.contains("\"failed\":0")); assertTrue(response.contains("\"success\":1")); assertTrue(response.contains("\"errors\":[]")); + validateCounters(0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); } @Test - public void putDoubleSummary() throws Exception { + public void putTwoSummary() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?summary", - "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - + ":42,\"tags\":{\"host\":\"web01\"}},{\"metric\":\"sys.cpu.system\"," - + "\"timestamp\":1365465600,\"value\":24,\"tags\":" - + "{\"host\":\"web01\"}}]"); - PutDataPointRpc put = new PutDataPointRpc(); + "[{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + + ":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + + "\"}},{\"metric\":\"" + METRIC_B_STRING + "\"," + + "\"timestamp\":1365465600,\"value\":24,\"tags\":" + + "{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}]"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); final String response = query.response().getContent().toString(Charset.forName("UTF-8")); assertTrue(response.contains("\"failed\":0")); assertTrue(response.contains("\"success\":2")); + validateCounters(0, 1, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void putNegativeInt() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":-42,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":-42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateCounters(0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void putFloat() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":42.2,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":42.2,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateCounters(0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); + } + + @Test + public void putDoublePrecisionFloatingPoint() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put", + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + + ":123456.123456789,\"tags\":{\"" + TAGK_STRING + "\":\"" + + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + put.execute(tsdb, query); + assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateCounters(0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); + } + + @Test + public void putNegativeDoublePrecisionFloatingPoint() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put", + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + + ":-123456.123456789,\"tags\":{\"" + TAGK_STRING + "\":\"" + + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + put.execute(tsdb, query); + assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateCounters(0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void putNegativeFloat() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":-42.2,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":-42.2,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateCounters(0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void putSEBig() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":4.22e3,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":42.22e3,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateCounters(0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void putSECaseBig() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":4.22E3,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":42.22E3,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateCounters(0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void putNegativeSEBig() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":-4.22e3,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":-42.22e3,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateCounters(0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void putNegativeSECaseBig() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":-4.22E3,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":-42.22E3,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateCounters(0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void putSETiny() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":4.2e-3,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":42.22e-3,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateCounters(0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void putSECaseTiny() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":4.2E-3,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":42.22E-3,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateCounters(0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void putNegativeSETiny() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":-4.2e-3,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":-4.2e-3,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateCounters(0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void putNegativeSECaseTiny() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":-4.2E-3,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":-4.2E-3,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateCounters(0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); } - @Test (expected = BadRequestException.class) + @Test public void badMethod() throws Exception { HttpQuery query = NettyMocks.getQuery(tsdb, "/api/put"); - PutDataPointRpc put = new PutDataPointRpc(); - put.execute(tsdb, query); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + try { + put.execute(tsdb, query); + fail("Expected BadRequestException"); + } catch (BadRequestException e) { } + validateCounters(0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); } - - @Test (expected = BadRequestException.class) + + @Test public void badJSON() throws Exception { // missing a quotation mark HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put", - "{\"metric\":\"sys.cpu.nice\",\"timestamp:1365465600,\"value\"" - +":42,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); - put.execute(tsdb, query); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp:1365465600,\"value\"" + +":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + try { + put.execute(tsdb, query); + fail("Expected BadRequestException"); + } catch (BadRequestException e) { } + validateCounters(0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + validateSEH(false); } - @Test (expected = BadRequestException.class) + @Test public void notJSON() throws Exception { // missing a quotation mark HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put", "Hello World"); - PutDataPointRpc put = new PutDataPointRpc(); - put.execute(tsdb, query); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + try { + put.execute(tsdb, query); + fail("Expected BadRequestException"); + } catch (BadRequestException e) { } + validateCounters(0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + validateSEH(false); } - @Test (expected = BadRequestException.class) + @Test public void noContent() throws Exception { // missing a quotation mark HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put", ""); - PutDataPointRpc put = new PutDataPointRpc(); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + try { + put.execute(tsdb, query); + fail("Expected BadRequestException"); + } catch (BadRequestException e) { } + validateCounters(0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + validateSEH(false); + } + + @Test + public void inFlightExceeded() throws Exception { + when(client.put(any(PutRequest.class))) + .thenReturn(Deferred.fromError(mock(PleaseThrottleException.class))); + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + put.execute(tsdb, query); + validateCounters(0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0); + validateSEH(true); + } + + @Test + public void inFlightExceededHandler() throws Exception { + setStorageExceptionHandler(); + when(client.put(any(PutRequest.class))) + .thenReturn(Deferred.fromError(mock(PleaseThrottleException.class))); + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); + validateCounters(0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0); + validateSEH(true); + } + + @Test + public void hbaseError() throws Exception { + when(client.put(any(PutRequest.class))) + .thenReturn(Deferred.fromError(mock(HBaseException.class))); + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + put.execute(tsdb, query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + validateCounters(0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(true); } + @Test + public void hbaseErrorHandler() throws Exception { + setStorageExceptionHandler(); + when(client.put(any(PutRequest.class))) + .thenReturn(Deferred.fromError(mock(HBaseException.class))); + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + put.execute(tsdb, query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + validateCounters(0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(true); + } + @Test public void noSuchUniqueName() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", - "{\"metric\":\"doesnotexist\",\"timestamp\":1365465600,\"value\"" - +":42,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + NSUN_METRIC + "\",\"timestamp\":1365465600,\"value\"" + +":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); final String response = @@ -310,14 +667,16 @@ public void noSuchUniqueName() throws Exception { assertTrue(response.contains("\"error\":\"Unknown metric\"")); assertTrue(response.contains("\"failed\":1")); assertTrue(response.contains("\"success\":0")); + validateCounters(0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0); + validateSEH(false); } @Test public void missingMetric() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", "{\"timestamp\":1365465600,\"value\"" - +":42,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + +":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); final String response = @@ -325,14 +684,16 @@ public void missingMetric() throws Exception { assertTrue(response.contains("\"error\":\"Metric name was empty\"")); assertTrue(response.contains("\"failed\":1")); assertTrue(response.contains("\"success\":0")); + validateCounters(0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void nullMetric() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", "{\"metric\":null,\"timestamp\":1365465600,\"value\"" - +":42,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + +":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); final String response = @@ -340,14 +701,16 @@ public void nullMetric() throws Exception { assertTrue(response.contains("\"error\":\"Metric name was empty\"")); assertTrue(response.contains("\"failed\":1")); assertTrue(response.contains("\"success\":0")); + validateCounters(0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void missingTimestamp() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", - "{\"metric\":\"sys.cpu.nice\",\"value\"" - +":42,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"value\"" + +":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); final String response = @@ -355,14 +718,16 @@ public void missingTimestamp() throws Exception { assertTrue(response.contains("\"error\":\"Invalid timestamp\"")); assertTrue(response.contains("\"failed\":1")); assertTrue(response.contains("\"success\":0")); + validateCounters(0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void nullTimestamp() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":null,\"value\"" - +":42,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":null,\"value\"" + +":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); final String response = @@ -370,14 +735,16 @@ public void nullTimestamp() throws Exception { assertTrue(response.contains("\"error\":\"Invalid timestamp\"")); assertTrue(response.contains("\"failed\":1")); assertTrue(response.contains("\"success\":0")); + validateCounters(0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void invalidTimestamp() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":-1,\"value\"" - +":42,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":-1,\"value\"" + +":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); final String response = @@ -385,14 +752,16 @@ public void invalidTimestamp() throws Exception { assertTrue(response.contains("\"error\":\"Invalid timestamp\"")); assertTrue(response.contains("\"failed\":1")); assertTrue(response.contains("\"success\":0")); + validateCounters(0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void missingValue() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"tags\":" - + "{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"tags\":" + + "{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); final String response = @@ -400,14 +769,16 @@ public void missingValue() throws Exception { assertTrue(response.contains("\"error\":\"Empty value\"")); assertTrue(response.contains("\"failed\":1")); assertTrue(response.contains("\"success\":0")); + validateCounters(0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void nullValue() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":null,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":null,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); final String response = @@ -415,14 +786,16 @@ public void nullValue() throws Exception { assertTrue(response.contains("\"error\":\"Empty value\"")); assertTrue(response.contains("\"failed\":1")); assertTrue(response.contains("\"success\":0")); + validateCounters(0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void emptyValue() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":\"\",\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":\"\",\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); final String response = @@ -430,14 +803,16 @@ public void emptyValue() throws Exception { assertTrue(response.contains("\"error\":\"Empty value\"")); assertTrue(response.contains("\"failed\":1")); assertTrue(response.contains("\"success\":0")); + validateCounters(0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void badValue() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":\"notanumber\",\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":\"notanumber\",\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); final String response = @@ -445,14 +820,16 @@ public void badValue() throws Exception { assertTrue(response.contains("\"error\":\"Unable to parse value to a number\"")); assertTrue(response.contains("\"failed\":1")); assertTrue(response.contains("\"success\":0")); + validateCounters(0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void ValueNaN() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":NaN,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":NaN,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); final String response = @@ -460,23 +837,30 @@ public void ValueNaN() throws Exception { assertTrue(response.contains("\"error\":\"Unable to parse value to a number\"")); assertTrue(response.contains("\"failed\":1")); assertTrue(response.contains("\"success\":0")); + validateCounters(0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0); + validateSEH(false); } - @Test (expected = BadRequestException.class) + @Test public void ValueNaNCase() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":Nan,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); - put.execute(tsdb, query); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":Nan,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + try { + put.execute(tsdb, query); + fail("Expected BadRequestException"); + } catch (BadRequestException e) { } + validateCounters(0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void ValueINF() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":+INF,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":+INF,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); final String response = @@ -484,14 +868,16 @@ public void ValueINF() throws Exception { assertTrue(response.contains("\"error\":\"Unable to parse value to a number\"")); assertTrue(response.contains("\"failed\":1")); assertTrue(response.contains("\"success\":0")); + validateCounters(0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void ValueNINF() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":-INF,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":-INF,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); final String response = @@ -499,32 +885,44 @@ public void ValueNINF() throws Exception { assertTrue(response.contains("\"error\":\"Unable to parse value to a number\"")); assertTrue(response.contains("\"failed\":1")); assertTrue(response.contains("\"success\":0")); + validateCounters(0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0); + validateSEH(false); } - @Test (expected = BadRequestException.class) + @Test public void ValueINFUnsigned() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":INF,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); - put.execute(tsdb, query); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":INF,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + try { + put.execute(tsdb, query); + fail("Expected BadRequestException"); + } catch (BadRequestException e) { } + validateCounters(0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + validateSEH(false); } - @Test (expected = BadRequestException.class) + @Test public void ValueINFCase() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":+inf,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); - put.execute(tsdb, query); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":+inf,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + try { + put.execute(tsdb, query); + fail("Expected BadRequestException"); + } catch (BadRequestException e) { } + validateCounters(0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + validateSEH(false); } @Test - public void ValueInfiniy() throws Exception { + public void ValueInfinity() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":+Infinity,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":+Infinity,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); final String response = @@ -532,14 +930,16 @@ public void ValueInfiniy() throws Exception { assertTrue(response.contains("\"error\":\"Unable to parse value to a number\"")); assertTrue(response.contains("\"failed\":1")); assertTrue(response.contains("\"success\":0")); + validateCounters(0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0); + validateSEH(false); } @Test - public void ValueNInfiniy() throws Exception { + public void ValueNInfinity() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":-Infinity,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":-Infinity,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); final String response = @@ -547,14 +947,16 @@ public void ValueNInfiniy() throws Exception { assertTrue(response.contains("\"error\":\"Unable to parse value to a number\"")); assertTrue(response.contains("\"failed\":1")); assertTrue(response.contains("\"success\":0")); + validateCounters(0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void ValueInfinityUnsigned() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" - +":Infinity,\"tags\":{\"host\":\"web01\"}}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + +":Infinity,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); final String response = @@ -562,14 +964,16 @@ public void ValueInfinityUnsigned() throws Exception { assertTrue(response.contains("\"error\":\"Unable to parse value to a number\"")); assertTrue(response.contains("\"failed\":1")); assertTrue(response.contains("\"success\":0")); + validateCounters(0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void missingTags() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\":42" - + "}"); - PutDataPointRpc put = new PutDataPointRpc(); + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + + ":42}"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); final String response = @@ -577,14 +981,16 @@ public void missingTags() throws Exception { assertTrue(response.contains("\"error\":\"Missing tags\"")); assertTrue(response.contains("\"failed\":1")); assertTrue(response.contains("\"success\":0")); + validateCounters(0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void nullTags() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" +":42,\"tags\":null}"); - PutDataPointRpc put = new PutDataPointRpc(); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); final String response = @@ -592,14 +998,16 @@ public void nullTags() throws Exception { assertTrue(response.contains("\"error\":\"Missing tags\"")); assertTrue(response.contains("\"failed\":1")); assertTrue(response.contains("\"success\":0")); + validateCounters(0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + validateSEH(false); } @Test public void emptyTags() throws Exception { HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", - "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\"" + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" +":42,\"tags\":{}}"); - PutDataPointRpc put = new PutDataPointRpc(); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); put.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); final String response = @@ -607,5 +1015,370 @@ public void emptyTags() throws Exception { assertTrue(response.contains("\"error\":\"Missing tags\"")); assertTrue(response.contains("\"failed\":1")); assertTrue(response.contains("\"success\":0")); + validateCounters(0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + validateSEH(false); } + + @Test + public void syncOKNoDetails() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?sync=true", + "[{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + + ":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}," + + "{\"metric\":\"" + METRIC_B_STRING + "\"," + + "\"timestamp\":1365465600,\"value\":24,\"tags\":" + + "{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}]"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + put.execute(tsdb, query); + assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateCounters(0, 1, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); + verify(tsdb, never()).getTimer(); + } + + @Test + public void syncOKSummary() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?sync=true&summary", + "[{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + + ":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}," + + "{\"metric\":\"" + METRIC_B_STRING + "\"," + + "\"timestamp\":1365465600,\"value\":24,\"tags\":" + + "{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}]"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + put.execute(tsdb, query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"failed\":0")); + assertTrue(response.contains("\"success\":2")); + assertFalse(response.contains("\"errors\":[]")); + validateCounters(0, 1, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); + verify(tsdb, never()).getTimer(); + } + + @Test + public void syncOKSummaryDetails() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/put?sync=true&summary&details", + "[{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + + ":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}," + + "{\"metric\":\"" + METRIC_B_STRING + "\"," + + "\"timestamp\":1365465600,\"value\":24,\"tags\":" + + "{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}]"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + put.execute(tsdb, query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + + assertTrue(response.contains("\"failed\":0")); + assertTrue(response.contains("\"success\":2")); + assertTrue(response.contains("\"errors\":[]")); + validateCounters(0, 1, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); + verify(tsdb, never()).getTimer(); + } + + @Test + public void syncOKDetails() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?sync=true&details", + "[{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + + ":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}," + + "{\"metric\":\"" + METRIC_B_STRING + "\"," + + "\"timestamp\":1365465600,\"value\":24,\"tags\":" + + "{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}]"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + put.execute(tsdb, query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"failed\":0")); + assertTrue(response.contains("\"success\":2")); + assertTrue(response.contains("\"errors\":[]")); + validateCounters(0, 1, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); + verify(tsdb, never()).getTimer(); + } + + @Test + public void syncOneFailed() throws Exception { + when(client.put(any(PutRequest.class))) + .thenReturn(Deferred.fromResult(null)) + .thenReturn(Deferred.fromError(mock(HBaseException.class))); + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?sync", + "[{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + + ":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + + "\"}},{\"metric\":\"" + METRIC_B_STRING + "\"," + + "\"timestamp\":1365465600,\"value\":1,\"tags\":" + + "{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}]"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + put.execute(tsdb, query); + assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"code\":400")); + assertTrue(response.contains("\"message\":")); + validateCounters(0, 1, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(true); + verify(tsdb, never()).getTimer(); + } + + @Test + public void syncOneFailedSummary() throws Exception { + when(client.put(any(PutRequest.class))) + .thenReturn(Deferred.fromResult(null)) + .thenReturn(Deferred.fromError(mock(HBaseException.class))); + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?sync&summary", + "[{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + + ":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + + "\"}},{\"metric\":\"" + METRIC_B_STRING + "\"," + + "\"timestamp\":1365465600,\"value\":1,\"tags\":" + + "{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}]"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + put.execute(tsdb, query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"failed\":1")); + assertTrue(response.contains("\"success\":1")); + assertFalse(response.contains("\"errors\":[]")); + validateCounters(0, 1, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(true); + verify(tsdb, never()).getTimer(); + } + + @Test + public void syncOneFailedDetails() throws Exception { + when(client.put(any(PutRequest.class))) + .thenReturn(Deferred.fromResult(null)) + .thenReturn(Deferred.fromError(mock(HBaseException.class))); + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?sync&details", + "[{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + + ":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + + "\"}},{\"metric\":\"" + METRIC_B_STRING + "\"," + + "\"timestamp\":1365465600,\"value\":1,\"tags\":" + + "{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}]"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + put.execute(tsdb, query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"failed\":1")); + assertTrue(response.contains("\"success\":1")); + assertTrue(response.contains("\"errors\":[{")); + validateCounters(0, 1, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(true); + verify(tsdb, never()).getTimer(); + } + + @Test + public void syncTwoFailedDetails() throws Exception { + when(client.put(any(PutRequest.class))) + .thenReturn(Deferred.fromError(mock(HBaseException.class))); + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?sync&details", + "[{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + + ":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + + "\"}},{\"metric\":\"" + NSUN_METRIC + "\"," + + "\"timestamp\":1365465600,\"value\":1,\"tags\":" + + "{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}]"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + put.execute(tsdb, query); + assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + System.out.println(response); + assertTrue(response.contains("\"success\":0")); + assertTrue(response.contains("\"failed\":2")); + assertTrue(response.contains("\"errors\":[{")); + validateCounters(0, 1, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0); + validateSEH(true); + verify(tsdb, never()).getTimer(); + } + + @Test + public void syncOKTimeoutNoDetails() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?sync&sync_timeout=30000", + "[{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + + ":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + + "\"}},{\"metric\":\"" + METRIC_B_STRING + "\"," + + "\"timestamp\":1365465600,\"value\":1,\"tags\":" + + "{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}]"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + put.execute(tsdb, query); + assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + assertEquals(1, http_requests.get()); + assertEquals(0, invalid_values.get()); + validateCounters(0, 1, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); + verify(tsdb, times(1)).getTimer(); + verify(timer.timeout, times(1)).cancel(); + } + + @Test + public void syncOKTimeoutSummary() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/put?sync&sync_timeout=30000&summary", + "[{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + + ":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + + "\"}},{\"metric\":\"" + METRIC_B_STRING + "\"," + + "\"timestamp\":1365465600,\"value\":1,\"tags\":" + + "{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}]"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + put.execute(tsdb, query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"failed\":0")); + assertTrue(response.contains("\"success\":2")); + assertTrue(response.contains("\"timeouts\":0")); + assertFalse(response.contains("\"errors\":[]")); + validateCounters(0, 1, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); + verify(tsdb, times(1)).getTimer(); + verify(timer.timeout, times(1)).cancel(); + } + + @Test + public void syncTimeoutOneFailed() throws Exception { + when(client.put(any(PutRequest.class))) + .thenReturn(Deferred.fromResult(null)) + .thenReturn(Deferred.fromError(mock(HBaseException.class))); + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?sync&sync_timeout=30000", + "[{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + + ":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + + "\"}},{\"metric\":\"" + METRIC_B_STRING + "\"," + + "\"timestamp\":1365465600,\"value\":1,\"tags\":" + + "{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}]"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + put.execute(tsdb, query); + assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"code\":400")); + assertTrue(response.contains("\"message\":")); + validateCounters(0, 1, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(true); + verify(tsdb, times(1)).getTimer(); + verify(timer.timeout, times(1)).cancel(); + } + + @Test + public void syncTimeoutOneFailedSummary() throws Exception { + when(client.put(any(PutRequest.class))) + .thenReturn(Deferred.fromResult(null)) + .thenReturn(Deferred.fromError(mock(HBaseException.class))); + HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/put?sync&summary&sync_timeout=30000", + "[{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + + ":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + + "\"}},{\"metric\":\"" + METRIC_B_STRING + "\"," + + "\"timestamp\":1365465600,\"value\":1,\"tags\":" + + "{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}]"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + put.execute(tsdb, query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"failed\":1")); + assertTrue(response.contains("\"success\":1")); + assertTrue(response.contains("\"timeouts\":0")); + assertFalse(response.contains("\"errors\":[]")); + validateCounters(0, 1, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(true); + verify(tsdb, times(1)).getTimer(); + verify(timer.timeout, times(1)).cancel(); + } + + @Test + public void syncTimeoutTwoFailedDetails() throws Exception { + when(client.put(any(PutRequest.class))) + .thenReturn(Deferred.fromError(mock(HBaseException.class))); + HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/put?sync&details&sync_timeout=30000", + "[{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + + ":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + + "\"}},{\"metric\":\"" + NSUN_METRIC + "\"," + + "\"timestamp\":1365465600,\"value\":1,\"tags\":" + + "{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}]"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + put.execute(tsdb, query); + assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + System.out.println(response); + assertTrue(response.contains("\"success\":0")); + assertTrue(response.contains("\"failed\":2")); + assertTrue(response.contains("\"timeouts\":0")); + assertTrue(response.contains("\"errors\":[{")); + validateCounters(0, 1, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0); + validateSEH(true); + verify(tsdb, times(1)).getTimer(); + verify(timer.timeout, times(1)).cancel(); + } + + @Test + public void syncTimeoutOneFailedTimedoutSummary() throws Exception { + when(client.put(any(PutRequest.class))) + .thenReturn(new Deferred<Object>()) + .thenReturn(Deferred.fromError(mock(HBaseException.class))); + HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/put?sync&summary&sync_timeout=30000", + "[{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + + ":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + + "\"}},{\"metric\":\"" + METRIC_B_STRING + "\"," + + "\"timestamp\":1365465600,\"value\":1,\"tags\":" + + "{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}]"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + put.execute(tsdb, query); + verify(tsdb, times(1)).getTimer(); + verify(timer.timeout, never()).cancel(); + + timer.continuePausedTask(); + + assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"failed\":1")); + assertTrue(response.contains("\"success\":0")); + assertTrue(response.contains("\"timeouts\":1")); + assertFalse(response.contains("\"errors\":[]")); + validateCounters(0, 1, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0); + validateSEH(true); + verify(tsdb, times(1)).getTimer(); + verify(timer.timeout, never()).cancel(); + } + + @Test + public void syncTimeoutOneFailedTimedoutDetails() throws Exception { + when(client.put(any(PutRequest.class))) + .thenReturn(new Deferred<Object>()) + .thenReturn(Deferred.fromError(mock(HBaseException.class))); + HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/put?sync&details&sync_timeout=30000", + "[{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\"" + + ":42,\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + + "\"}},{\"metric\":\"" + METRIC_B_STRING + "\"," + + "\"timestamp\":1365465600,\"value\":1,\"tags\":" + + "{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}]"); + PutDataPointRpc put = new PutDataPointRpc(tsdb.getConfig()); + put.execute(tsdb, query); + verify(tsdb, times(1)).getTimer(); + verify(timer.timeout, never()).cancel(); + + timer.continuePausedTask(); + + assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"failed\":1")); + assertTrue(response.contains("\"success\":0")); + assertTrue(response.contains("\"timeouts\":1")); + assertTrue(response.contains("\"errors\":[{")); + assertTrue(response.contains("Write timedout")); + validateCounters(0, 1, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0); + validateSEH(true); + verify(tsdb, times(1)).getStorageExceptionHandler(); + verify(timer.timeout, never()).cancel(); + } + } diff --git a/test/tsd/TestQueryExecutor.java b/test/tsd/TestQueryExecutor.java new file mode 100644 index 0000000000..b5c1f5c903 --- /dev/null +++ b/test/tsd/TestQueryExecutor.java @@ -0,0 +1,720 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2010-2016 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.tsd; + +import static org.junit.Assert.assertTrue; + +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import net.opentsdb.core.FillPolicy; +import net.opentsdb.core.TSDB; +import net.opentsdb.core.TSQuery; +import net.opentsdb.query.expression.NumericFillPolicy; +import net.opentsdb.query.expression.BaseTimeSyncedIteratorTest; +import net.opentsdb.query.expression.VariableIterator.SetOperator; +import net.opentsdb.query.filter.TagVFilter; +import net.opentsdb.query.pojo.Downsampler; +import net.opentsdb.query.pojo.Expression; +import net.opentsdb.query.pojo.Filter; +import net.opentsdb.query.pojo.Join; +import net.opentsdb.query.pojo.Metric; +import net.opentsdb.query.pojo.Output; +import net.opentsdb.query.pojo.Query; +import net.opentsdb.query.pojo.Timespan; +import net.opentsdb.storage.MockBase; +import net.opentsdb.utils.Config; +import net.opentsdb.utils.DateTime; +import net.opentsdb.utils.JSON; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.stumbleupon.async.Deferred; +import com.stumbleupon.async.DeferredGroupException; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({TSDB.class, Config.class, HttpQuery.class, + Deferred.class, TSQuery.class, DateTime.class, DeferredGroupException.class }) +public class TestQueryExecutor extends BaseTimeSyncedIteratorTest { + + private Timespan time; + private List<TagVFilter> tags; + private List<Filter> filters; + private List<Metric> metrics; + private List<Expression> expressions; + private List<Output> outputs; + private Join intersection; + + @Before + public void setup() { + intersection = Join.Builder().setOperator(SetOperator.INTERSECTION).build(); + time = Timespan.Builder().setStart("1431561600") + .setAggregator("sum").build(); + + tags = Arrays.asList(new TagVFilter.Builder().setFilter("*").setGroupBy(true) + .setTagk("D").setType("wildcard").build()); + + filters = Arrays.asList(Filter.Builder().setId("f1") + .setTags(tags).build()); + final Metric metric1 = Metric.Builder().setMetric("A").setId("a") + .setFilter("f1").build(); + final Metric metric2 = Metric.Builder().setMetric("B").setId("b") + .setFilter("f1").build(); + metrics = Arrays.asList(metric1, metric2); + expressions = Arrays.asList(Expression.Builder().setId("e") + .setExpression("a + b").setJoin(intersection).build()); + outputs = Arrays.asList(Output.Builder().setId("e").setAlias("A plus B") + .build()); + } + + @Test + public void oneExpressionWithOutputAlias() throws Exception { + oneExtraSameE(); + final String json = JSON.serializeToString(getDefaultQueryBuilder().build()); + final QueryRpc rpc = new QueryRpc(); + final HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/query/exp", json); + NettyMocks.mockChannelFuture(query); + rpc.execute(tsdb, query); + + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"alias\":\"A plus B\"")); + assertTrue(response.contains("\"dps\":[[1431561600000,12.0,18.0]")); + assertTrue(response.contains("[1431561660000,14.0,20.0]")); + assertTrue(response.contains("[1431561720000,16.0,22.0]")); + assertTrue(response.contains("\"firstTimestamp\":1431561600000")); + assertTrue(response.contains("\"index\":1")); + assertTrue(response.contains("\"metrics\":[\"A\",\"B\"]")); + assertTrue(response.contains("\"index\":2")); + } + + @Test + public void oneExpressionDefaultOutput() throws Exception { + oneExtraSameE(); + final Query q = Query.Builder().setExpressions(expressions) + .setFilters(filters).setMetrics(metrics).setName("q1") + .setTime(time).build(); + final String json = JSON.serializeToString(q); + final QueryRpc rpc = new QueryRpc(); + final HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/query/exp", json); + NettyMocks.mockChannelFuture(query); + + rpc.execute(tsdb, query); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"id\":\"e\"")); + assertTrue(response.contains("\"dps\":[[1431561600000,12.0,18.0]")); + assertTrue(response.contains("[1431561660000,14.0,20.0]")); + assertTrue(response.contains("[1431561720000,16.0,22.0]")); + assertTrue(response.contains("\"firstTimestamp\":1431561600000")); + assertTrue(response.contains("\"index\":1")); + assertTrue(response.contains("\"metrics\":[\"A\",\"B\"]")); + assertTrue(response.contains("\"index\":2")); + // TODO - more asserts once we settle on names + } + + @Test + public void oneExpressionOutputAndBAlso() throws Exception { + oneExtraSameE(); + + outputs = new ArrayList<Output>(3); + outputs.add(Output.Builder().setId("e").setAlias("A plus B").build()); + outputs.add(Output.Builder().setId("a").build()); + outputs.add(Output.Builder().setId("b").build()); + + final String json = JSON.serializeToString(getDefaultQueryBuilder().build()); + + final QueryRpc rpc = new QueryRpc(); + final HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/query/exp", json); + NettyMocks.mockChannelFuture(query); + + rpc.execute(tsdb, query); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"alias\":\"A plus B\"")); + assertTrue(response.contains("\"dps\":[[1431561600000,12.0,18.0]")); + assertTrue(response.contains("[1431561660000,14.0,20.0]")); + assertTrue(response.contains("[1431561720000,16.0,22.0]")); + assertTrue(response.contains("\"firstTimestamp\":1431561600000")); + assertTrue(response.contains("\"index\":1")); + assertTrue(response.contains("\"metrics\":[\"A\",\"B\"]")); + assertTrue(response.contains("\"index\":2")); + + assertTrue(response.contains("\"id\":\"a\"")); + assertTrue(response.contains("\"dps\":[[1431561600000,1.0,4.0]")); + assertTrue(response.contains("\"metrics\":[\"A\"]")); + assertTrue(response.contains("\"id\":\"b\"")); + assertTrue(response.contains("\"dps\":[[1431561600000,11.0,14.0,17.0]")); + assertTrue(response.contains("\"metrics\":[\"B\"]")); + } + + @Test + public void oneExpressionDefaultFill() throws Exception { + threeSameEGaps(); + String json = JSON.serializeToString(getDefaultQueryBuilder()); + final QueryRpc rpc = new QueryRpc(); + final HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/query/exp", json); + query.getQueryBaseRoute(); // to the correct serializer + NettyMocks.mockChannelFuture(query); + + rpc.execute(tsdb, query); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"alias\":\"A plus B\"")); + assertTrue(response.contains("\"dps\":[[1431561600000,1.0,4.0,0.0]")); + assertTrue(response.contains("[1431561660000,0.0,20.0,8.0]")); + assertTrue(response.contains("[1431561720000,16.0,0.0,28.0]")); + assertTrue(response.contains("\"firstTimestamp\":1431561600000")); + assertTrue(response.contains("\"index\":1")); + assertTrue(response.contains("\"metrics\":[\"A\",\"B\"]")); + assertTrue(response.contains("\"index\":2")); + assertTrue(response.contains("\"index\":3")); + } + + @Test + public void oneExpressionDownsamplingMissingTimestampNoFill() throws Exception { + threeSameEGaps(); + final Downsampler downsampler = Downsampler.Builder() + .setAggregator("sum") + .setInterval("1m") + .build(); + time = Timespan.Builder().setStart("1431561600") + .setAggregator("sum") + .setDownsampler(downsampler).build(); + String json = JSON.serializeToString(getDefaultQueryBuilder()); + final QueryRpc rpc = new QueryRpc(); + final HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/query/exp", json); + query.getQueryBaseRoute(); // to the correct serializer + NettyMocks.mockChannelFuture(query); + + rpc.execute(tsdb, query); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"alias\":\"A plus B\"")); + assertTrue(response.contains("\"dps\":[[1431561600000,1.0,4.0,0.0]")); + assertTrue(response.contains("[1431561660000,0.0,20.0,8.0]")); + assertTrue(response.contains("[1431561720000,16.0,0.0,28.0]")); + assertTrue(response.contains("\"firstTimestamp\":1431561600000")); + assertTrue(response.contains("\"index\":1")); + assertTrue(response.contains("\"metrics\":[\"A\",\"B\"]")); + assertTrue(response.contains("\"index\":2")); + assertTrue(response.contains("\"index\":3")); + } + +// @Test +// public void oneExpressionDownsamplingMissingTimestampZeroFill() throws Exception { +// threeSameEGaps(); +// final Downsampler downsampler = Downsampler.Builder() +// .setAggregator("sum") +// .setInterval("1m") +// .setFillPolicy(new NumericFillPolicy(FillPolicy.ZERO)) +// .build(); +// time = Timespan.Builder().setStart("1431561540") +// .setEnd("1431561780") +// .setAggregator("sum") +// .setDownsampler(downsampler).build(); +// String json = JSON.serializeToString(getDefaultQueryBuilder()); +// final QueryRpc rpc = new QueryRpc(); +// final HttpQuery query = NettyMocks.postQuery(tsdb, +// "/api/query/exp", json); +// query.getQueryBaseRoute(); // to the correct serializer +// NettyMocks.mockChannelFuture(query); +// +// rpc.execute(tsdb, query); +// final String response = +// query.response().getContent().toString(Charset.forName("UTF-8")); +// assertTrue(response.contains("\"alias\":\"A plus B\"")); +// assertTrue(response.contains("\"dps\":[[1431561540000,0.0,0.0,0.0]")); +// assertTrue(response.contains("[1431561600000,1.0,4.0,0.0]")); +// assertTrue(response.contains("[1431561660000,0.0,20.0,8.0]")); +// assertTrue(response.contains("[1431561720000,16.0,0.0,28.0]")); +// assertTrue(response.contains("[1431561780000,0.0,0.0,0.0]")); +// assertTrue(response.contains("\"firstTimestamp\":1431561540000")); +// assertTrue(response.contains("\"index\":1")); +// assertTrue(response.contains("\"metrics\":[\"A\",\"B\"]")); +// assertTrue(response.contains("\"index\":2")); +// assertTrue(response.contains("\"index\":3")); +// } + + @Test + public void oneExpressionNoFilter() throws Exception { + oneExtraSameE(); + final Metric metric1 = Metric.Builder().setMetric("A").setId("a") + .build(); + final Metric metric2 = Metric.Builder().setMetric("B").setId("b") + .build(); + metrics = Arrays.asList(metric1, metric2); + + final String json = JSON.serializeToString(getDefaultQueryBuilder().build()); + final QueryRpc rpc = new QueryRpc(); + final HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/query/exp", json); + NettyMocks.mockChannelFuture(query); + + rpc.execute(tsdb, query); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); +System.out.println(response); + assertTrue(response.contains("\"alias\":\"A plus B\"")); + assertTrue(response.contains("\"dps\":[[1431561600000,47.0]")); + assertTrue(response.contains("[1431561660000,52.0]")); + assertTrue(response.contains("[1431561720000,57.0]")); + assertTrue(response.contains("\"firstTimestamp\":1431561600000")); + assertTrue(response.contains("\"index\":1")); + assertTrue(response.contains("\"metrics\":[\"A\",\"B\"]")); + assertTrue(response.contains("\"index\":1")); + } + + @Test + public void twoExpressionsDefaultOutput() throws Exception { + oneExtraSameE(); + expressions = Arrays.asList( + Expression.Builder().setId("e").setExpression("a + b") + .setJoin(intersection).build(), + Expression.Builder().setId("e2").setExpression("a * b") + .setJoin(intersection).build()); + + final Query q = Query.Builder().setExpressions(expressions) + .setFilters(filters).setMetrics(metrics).setName("q1") + .setTime(time).build(); + final String json = JSON.serializeToString(q); + final QueryRpc rpc = new QueryRpc(); + final HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/query/exp", json); + NettyMocks.mockChannelFuture(query); + + rpc.execute(tsdb, query); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"id\":\"e\"")); + assertTrue(response.contains("\"dps\":[[1431561600000,12.0,18.0]")); + assertTrue(response.contains("[1431561660000,14.0,20.0]")); + assertTrue(response.contains("[1431561720000,16.0,22.0]")); + assertTrue(response.contains("\"firstTimestamp\":1431561600000")); + assertTrue(response.contains("\"index\":1")); + assertTrue(response.contains("\"metrics\":[\"A\",\"B\"]")); + assertTrue(response.contains("\"index\":2")); + assertTrue(response.contains("\"id\":\"e2\"")); + assertTrue(response.contains("\"dps\":[[1431561600000,11.0,56.0]")); + assertTrue(response.contains("[1431561660000,24.0,75.0]")); + assertTrue(response.contains("[1431561720000,39.0,96.0]")); + } + + @Test + public void twoExpressionsOneWithoutResultsDefaultOutput() throws Exception { + oneExtraSameE(); + final Metric metric1 = Metric.Builder().setMetric("A").setId("a") + .setFilter("f1").setAggregator("sum").build(); + final Metric metric2 = Metric.Builder().setMetric("B").setId("b") + .setFilter("f1").setAggregator("sum").build(); + final Metric metric3 = Metric.Builder().setMetric("D").setId("d") + .setFilter("f1").setAggregator("sum").build(); + final Metric metric4 = Metric.Builder().setMetric("F").setId("f") + .setFilter("f1").setAggregator("sum").build(); + metrics = Arrays.asList(metric1, metric2, metric3, metric4); + + expressions = Arrays.asList( + Expression.Builder().setId("e").setExpression("a + b") + .setJoin(intersection).build(), + Expression.Builder().setId("x").setExpression("d + f") + .setJoin(intersection).build()); + + final Query q = Query.Builder().setExpressions(expressions) + .setFilters(filters).setMetrics(metrics).setName("q1") + .setTime(time).build(); + String json = JSON.serializeToString(q); + final QueryRpc rpc = new QueryRpc(); + final HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/query/exp", json); + query.getQueryBaseRoute(); // to the correct serializer + NettyMocks.mockChannelFuture(query); + + rpc.execute(tsdb, query); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"id\":\"e\"")); + assertTrue(response.contains("\"dps\":[[1431561600000,12.0,18.0]")); + assertTrue(response.contains("[1431561660000,14.0,20.0]")); + assertTrue(response.contains("[1431561720000,16.0,22.0]")); + assertTrue(response.contains("\"firstTimestamp\":1431561600000")); + assertTrue(response.contains("\"index\":1")); + assertTrue(response.contains("\"metrics\":[\"A\",\"B\"]")); + assertTrue(response.contains("\"index\":2")); + assertTrue(response.contains("\"id\":\"x\"")); + assertTrue(response.contains("\"dps\":[]")); + assertTrue(response.contains("\"firstTimestamp\":0")); + assertTrue(response.contains("\"series\":0")); + } + + @Test + public void multiExpressionsOneOutput() throws Exception { + oneExtraSameE(); + expressions = Arrays.asList( + Expression.Builder().setId("e").setExpression("a + b").setJoin(intersection).build(), + Expression.Builder().setId("e2").setExpression("e * 2").setJoin(intersection).build(), + Expression.Builder().setId("e3").setExpression("e * 2").setJoin(intersection).build(), + Expression.Builder().setId("e4").setExpression("e2 + e3").setJoin(intersection).build()); + + final String json = JSON.serializeToString(getDefaultQueryBuilder()); + final QueryRpc rpc = new QueryRpc(); + final HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/query/exp", json); + query.getQueryBaseRoute(); // to the correct serializer + NettyMocks.mockChannelFuture(query); + + rpc.execute(tsdb, query); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("[1431561660000,14.0,20.0]")); + assertTrue(response.contains("[1431561720000,16.0,22.0]")); + assertTrue(response.contains("\"firstTimestamp\":1431561600000")); + assertTrue(response.contains("\"index\":1")); + assertTrue(response.contains("\"metrics\":[\"A\",\"B\"]")); + assertTrue(response.contains("\"index\":2")); + } + + @Test + public void nestedExpressionsOneLevelDefaultOutput() throws Exception { + oneExtraSameE(); + expressions = Arrays.asList( + Expression.Builder().setId("e").setExpression("a + b") + .setJoin(intersection).build(), + Expression.Builder().setId("e2").setExpression("e * 2") + .setJoin(intersection).build()); + + final Query q = Query.Builder().setExpressions(expressions) + .setFilters(filters).setMetrics(metrics).setName("q1") + .setTime(time).build(); + final String json = JSON.serializeToString(q); + final QueryRpc rpc = new QueryRpc(); + final HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/query/exp", json); + NettyMocks.mockChannelFuture(query); + + rpc.execute(tsdb, query); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"id\":\"e\"")); + assertTrue(response.contains("\"dps\":[[1431561600000,12.0,18.0]")); + assertTrue(response.contains("[1431561660000,14.0,20.0]")); + assertTrue(response.contains("[1431561720000,16.0,22.0]")); + assertTrue(response.contains("\"firstTimestamp\":1431561600000")); + assertTrue(response.contains("\"index\":1")); + assertTrue(response.contains("\"metrics\":[\"A\",\"B\"]")); + assertTrue(response.contains("\"index\":2")); + assertTrue(response.contains("\"id\":\"e2\"")); + assertTrue(response.contains("\"dps\":[[1431561600000,24.0,36.0]")); + assertTrue(response.contains("[1431561660000,28.0,40.0]")); + assertTrue(response.contains("[1431561720000,32.0,44.0]")); + } + + @Test + public void nestedExpressionsTwoLevelsDefaultOutput() throws Exception { + oneExtraSameE(); + expressions = Arrays.asList( + Expression.Builder().setId("e").setExpression("a + b").setJoin(intersection).build(), + Expression.Builder().setId("e2").setExpression("e * 2").setJoin(intersection).build(), + Expression.Builder().setId("e3").setExpression("e * 2").setJoin(intersection).build(), + Expression.Builder().setId("e4").setExpression("e2 + e3").setJoin(intersection).build()); + + final Query q = Query.Builder().setExpressions(expressions) + .setFilters(filters).setMetrics(metrics).setName("q1") + .setTime(time).build(); + final String json = JSON.serializeToString(q); + final QueryRpc rpc = new QueryRpc(); + final HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/query/exp", json); + NettyMocks.mockChannelFuture(query); + + rpc.execute(tsdb, query); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"id\":\"e\"")); + assertTrue(response.contains("\"dps\":[[1431561600000,12.0,18.0]")); + assertTrue(response.contains("[1431561660000,14.0,20.0]")); + assertTrue(response.contains("[1431561720000,16.0,22.0]")); + assertTrue(response.contains("\"firstTimestamp\":1431561600000")); + assertTrue(response.contains("\"index\":1")); + assertTrue(response.contains("\"metrics\":[\"A\",\"B\"]")); + assertTrue(response.contains("\"index\":2")); + assertTrue(response.contains("\"id\":\"e2\"")); + assertTrue(response.contains("\"dps\":[[1431561600000,24.0,36.0]")); + assertTrue(response.contains("[1431561660000,28.0,40.0]")); + assertTrue(response.contains("[1431561720000,32.0,44.0]")); + assertTrue(response.contains("\"id\":\"e3\"")); + assertTrue(response.contains("\"id\":\"e4\"")); + assertTrue(response.contains("\"dps\":[[1431561600000,48.0,72.0]")); + assertTrue(response.contains("[1431561660000,56.0,80.0]")); + assertTrue(response.contains("[1431561720000,64.0,88.0]")); + } + + @Test + public void nestedExpressionsTwoLevelsDefaultOutputOrdering() throws Exception { + oneExtraSameE(); + expressions = Arrays.asList( + Expression.Builder().setId("e2").setExpression("e * 2").setJoin(intersection).build(), + Expression.Builder().setId("e4").setExpression("e2 + e3").setJoin(intersection).build(), + Expression.Builder().setId("e3").setExpression("e * 2").setJoin(intersection).build(), + Expression.Builder().setId("e").setExpression("a + b").setJoin(intersection).build() + ); + + final Query q = Query.Builder().setExpressions(expressions) + .setFilters(filters).setMetrics(metrics).setName("q1") + .setTime(time).build(); + final String json = JSON.serializeToString(q); + final QueryRpc rpc = new QueryRpc(); + final HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/query/exp", json); + NettyMocks.mockChannelFuture(query); + + rpc.execute(tsdb, query); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"id\":\"e\"")); + assertTrue(response.contains("\"dps\":[[1431561600000,12.0,18.0]")); + assertTrue(response.contains("[1431561660000,14.0,20.0]")); + assertTrue(response.contains("[1431561720000,16.0,22.0]")); + assertTrue(response.contains("\"firstTimestamp\":1431561600000")); + assertTrue(response.contains("\"index\":1")); + assertTrue(response.contains("\"metrics\":[\"A\",\"B\"]")); + assertTrue(response.contains("\"index\":2")); + assertTrue(response.contains("\"id\":\"e2\"")); + assertTrue(response.contains("\"dps\":[[1431561600000,24.0,36.0]")); + assertTrue(response.contains("[1431561660000,28.0,40.0]")); + assertTrue(response.contains("[1431561720000,32.0,44.0]")); + assertTrue(response.contains("\"id\":\"e3\"")); + assertTrue(response.contains("\"id\":\"e4\"")); + assertTrue(response.contains("\"dps\":[[1431561600000,48.0,72.0]")); + assertTrue(response.contains("[1431561660000,56.0,80.0]")); + assertTrue(response.contains("[1431561720000,64.0,88.0]")); + } + + @Test + public void emptyResultSet() throws Exception { + setDataPointStorage(); + String json = JSON.serializeToString(getDefaultQueryBuilder()); + final QueryRpc rpc = new QueryRpc(); + final HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/query/exp", json); + query.getQueryBaseRoute(); // to the correct serializer + NettyMocks.mockChannelFuture(query); + + rpc.execute(tsdb, query); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"dps\":[]")); + assertTrue(response.contains("\"firstTimestamp\":0")); + assertTrue(response.contains("\"series\":0")); + } + + @Test + public void scannerException() throws Exception { + oneExtraSameE(); + storage.throwException(MockBase.stringToBytes( + "00000B5553E58000000D00000F00000E00000E"), + new RuntimeException("Boo!"), true); + final String json = JSON.serializeToString(getDefaultQueryBuilder().build()); + + final QueryRpc rpc = new QueryRpc(); + final HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/query/exp", json); + query.getQueryBaseRoute(); // to the correct serializer + NettyMocks.mockChannelFuture(query); + + rpc.execute(tsdb, query); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"code\":400")); + assertTrue(response.contains("\"message\":\"Boo!\"")); + } + + @Test + public void nsunMetric() throws Exception { + oneExtraSameE(); + final Metric metric1 = Metric.Builder().setMetric("A").setId("a") + .setFilter("f1").setAggregator("sum").build(); + final Metric metric2 = Metric.Builder().setMetric(NSUN_METRIC).setId("b") + .setFilter("f1").setAggregator("sum").build(); + metrics = Arrays.asList(metric1, metric2); + final String json = JSON.serializeToString(getDefaultQueryBuilder().build()); + + final QueryRpc rpc = new QueryRpc(); + final HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/query/exp", json); + query.getQueryBaseRoute(); // to the correct serializer + NettyMocks.mockChannelFuture(query); + + rpc.execute(tsdb, query); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"code\":400")); + assertTrue(response.contains("\"message\":\"No such name for '" + + NSUN_METRIC + "'")); + } + + @Test + public void selfReferencingExpression() throws Exception { + oneExtraSameE(); + expressions = Arrays.asList( + Expression.Builder().setId("e").setExpression("a + b").build(), + Expression.Builder().setId("e2").setExpression("e * 2").build(), + Expression.Builder().setId("e3").setExpression("e * 2").build(), + Expression.Builder().setId("e4").setExpression("e2 + e4").build()); + + final Query q = Query.Builder().setExpressions(expressions) + .setFilters(filters).setMetrics(metrics).setName("q1") + .setTime(time).build(); + final String json = JSON.serializeToString(q); + final QueryRpc rpc = new QueryRpc(); + final HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/query/exp", json); + query.getQueryBaseRoute(); // to the correct serializer + NettyMocks.mockChannelFuture(query); + + rpc.execute(tsdb, query); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"code\":400")); + assertTrue(response.contains("\"message\":\"Self referencing")); + } + + @Test + public void circularReferenceExpression() throws Exception { + oneExtraSameE(); + expressions = Arrays.asList( + Expression.Builder().setId("e").setExpression("a + e4").build(), + Expression.Builder().setId("e2").setExpression("e * 2").build(), + Expression.Builder().setId("e3").setExpression("e * 2").build(), + Expression.Builder().setId("e4").setExpression("e2 + e3").build()); + + final Query q = Query.Builder().setExpressions(expressions) + .setFilters(filters).setMetrics(metrics).setName("q1") + .setTime(time).build(); + final String json = JSON.serializeToString(q); + final QueryRpc rpc = new QueryRpc(); + final HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/query/exp", json); + query.getQueryBaseRoute(); // to the correct serializer + NettyMocks.mockChannelFuture(query); + + rpc.execute(tsdb, query); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"code\":400")); + assertTrue(response.contains("\"message\":\"Circular reference found:")); + } + + @Test + public void noIntersectionsFound() throws Exception { + threeDifE(); + + String json = JSON.serializeToString(getDefaultQueryBuilder()); + final QueryRpc rpc = new QueryRpc(); + final HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/query/exp", json); + query.getQueryBaseRoute(); // to the correct serializer + NettyMocks.mockChannelFuture(query); + + rpc.execute(tsdb, query); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"code\":400")); + assertTrue(response.contains("\"message\":\"No intersections found")); + } + + @Test + public void noIntersectionsFoundNestedExpression() throws Exception { + oneExtraSameE(); + + final Metric metric1 = Metric.Builder().setMetric("A").setId("a") + .setFilter("f1").setAggregator("sum").build(); + final Metric metric2 = Metric.Builder().setMetric("B").setId("b") + .setFilter("f1").setAggregator("sum").build(); + final Metric metric3 = Metric.Builder().setMetric("D").setId("d") + .setFilter("f1").setAggregator("sum").build(); + metrics = Arrays.asList(metric1, metric2, metric3); + + expressions = Arrays.asList( + Expression.Builder().setId("e").setExpression("a + b").setJoin(intersection).build(), + Expression.Builder().setId("x").setExpression("d + e").setJoin(intersection).build()); + + final Query q = Query.Builder().setExpressions(expressions) + .setFilters(filters).setMetrics(metrics).setName("q1") + .setTime(time).build(); + String json = JSON.serializeToString(q); + final QueryRpc rpc = new QueryRpc(); + final HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/query/exp", json); + query.getQueryBaseRoute(); // to the correct serializer + NettyMocks.mockChannelFuture(query); + + rpc.execute(tsdb, query); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"code\":400")); + assertTrue(response.contains("\"message\":\"No intersections found")); + } + + @Test + public void noIntersectionsFoundOneMetricEmpty() throws Exception { + oneExtraSameE(); + final Metric metric1 = Metric.Builder().setMetric("A").setId("a") + .setFilter("f1").setAggregator("sum").build(); + final Metric metric2 = Metric.Builder().setMetric("D").setId("b") + .setFilter("f1").setAggregator("sum").build(); + metrics = Arrays.asList(metric1, metric2); + String json = JSON.serializeToString(getDefaultQueryBuilder()); + final QueryRpc rpc = new QueryRpc(); + final HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/query/exp", json); + query.getQueryBaseRoute(); // to the correct serializer + NettyMocks.mockChannelFuture(query); + + rpc.execute(tsdb, query); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(response.contains("\"code\":400")); + assertTrue(response.contains("\"message\":\"No intersections found")); + } + + @Test (expected = IllegalArgumentException.class) + public void notEnoughMetrics() throws Exception { + oneExtraSameE(); + expressions = Arrays.asList( + Expression.Builder().setId("e").setExpression("a + b + c").build()); + String json = JSON.serializeToString(getDefaultQueryBuilder()); + final QueryRpc rpc = new QueryRpc(); + final HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/query/exp", json); + query.getQueryBaseRoute(); // to the correct serializer + NettyMocks.mockChannelFuture(query); + + rpc.execute(tsdb, query); + } + + protected Query.Builder getDefaultQueryBuilder() { + return Query.Builder().setExpressions(expressions).setFilters(filters) + .setMetrics(metrics).setName("q1").setTime(time).setOutputs(outputs); + } +} diff --git a/test/tsd/TestQueryRpc.java b/test/tsd/TestQueryRpc.java index d07627788a..c38b3bc0d6 100644 --- a/test/tsd/TestQueryRpc.java +++ b/test/tsd/TestQueryRpc.java @@ -15,66 +15,95 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.when; import static org.powermock.api.mockito.PowerMockito.mock; -import org.powermock.api.mockito.PowerMockito; -import org.mockito.Matchers; + import java.lang.reflect.Method; -import java.util.Collection; -import java.util.Collections; +import java.nio.charset.Charset; import java.util.ArrayList; +import java.util.List; + +import net.opentsdb.auth.AuthState; +import net.opentsdb.auth.Authentication; +import net.opentsdb.auth.Authorization; +import net.opentsdb.auth.AuthState.AuthStatus; import net.opentsdb.core.DataPoints; import net.opentsdb.core.Query; import net.opentsdb.core.TSDB; import net.opentsdb.core.TSQuery; import net.opentsdb.core.TSSubQuery; +import net.opentsdb.query.expression.ExpressionTree; +import net.opentsdb.query.filter.TagVLiteralOrFilter; +import net.opentsdb.query.filter.TagVRegexFilter; +import net.opentsdb.query.filter.TagVWildcardFilter; +import net.opentsdb.storage.MockDataPoints; import net.opentsdb.utils.Config; -import org.hbase.async.HBaseClient; +import net.opentsdb.utils.DateTime; + import org.jboss.netty.handler.codec.http.HttpResponseStatus; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; + import net.opentsdb.uid.NoSuchUniqueName; + import com.stumbleupon.async.Deferred; +import com.stumbleupon.async.DeferredGroupException; + /** * Unit tests for the Query RPC class that handles parsing user queries for * timeseries data and returning that data - * <b>Note:</b> Testing query validation and such should be done in the + * <b>Note:</b> Testing query validation and such should be done in the * core.TestTSQuery and TestTSSubQuery classes */ @RunWith(PowerMockRunner.class) -@PrepareForTest({TSDB.class, Config.class, HttpQuery.class, Query.class, - Deferred.class, TSQuery.class}) +@PrepareForTest({ TSDB.class, Config.class, HttpQuery.class, Query.class, + Deferred.class, TSQuery.class, DateTime.class, DeferredGroupException.class }) public final class TestQueryRpc { private TSDB tsdb = null; - final private QueryRpc rpc = new QueryRpc(); - final private Query empty_query = mock(Query.class); - + private QueryRpc rpc; + private Query empty_query = mock(Query.class); + private Query query_result; + private List<ExpressionTree> expressions; + private static final Method parseQuery; static { try { - parseQuery = QueryRpc.class.getDeclaredMethod("parseQuery", - TSDB.class, HttpQuery.class); + parseQuery = QueryRpc.class.getDeclaredMethod("parseQuery", + TSDB.class, HttpQuery.class, List.class); parseQuery.setAccessible(true); } catch (Exception e) { throw new RuntimeException("Failed in static initializer", e); } } - + @Before public void before() throws Exception { tsdb = NettyMocks.getMockedHTTPTSDB(); - when(tsdb.newQuery()).thenReturn(empty_query); + empty_query = mock(Query.class); + query_result = mock(Query.class); + rpc = new QueryRpc(); + expressions = null; + + when(tsdb.newQuery()).thenReturn(query_result); when(empty_query.run()).thenReturn(new DataPoints[0]); + when(query_result.configureFromQuery((TSQuery)any(), anyInt())) + .thenReturn(Deferred.fromResult(null)); + when(query_result.runAsync()) + .thenReturn(Deferred.fromResult(new DataPoints[0])); } - + @Test public void parseQueryMType() throws Exception { - HttpQuery query = NettyMocks.getQuery(tsdb, + HttpQuery query = NettyMocks.getQuery(tsdb, "/api/query?start=1h-ago&m=sum:sys.cpu.0"); - TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); assertNotNull(tsq); assertEquals("1h-ago", tsq.getStart()); assertNotNull(tsq.getQueries()); @@ -83,20 +112,20 @@ public void parseQueryMType() throws Exception { assertEquals("sum", sub.getAggregator()); assertEquals("sys.cpu.0", sub.getMetric()); } - + @Test public void parseQueryMTypeWEnd() throws Exception { - HttpQuery query = NettyMocks.getQuery(tsdb, + HttpQuery query = NettyMocks.getQuery(tsdb, "/api/query?start=1h-ago&end=5m-ago&m=sum:sys.cpu.0"); - TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); assertEquals("5m-ago", tsq.getEnd()); } - + @Test public void parseQuery2MType() throws Exception { - HttpQuery query = NettyMocks.getQuery(tsdb, + HttpQuery query = NettyMocks.getQuery(tsdb, "/api/query?start=1h-ago&m=sum:sys.cpu.0&m=avg:sys.cpu.1"); - TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); assertNotNull(tsq.getQueries()); assertEquals(2, tsq.getQueries().size()); TSSubQuery sub1 = tsq.getQueries().get(0); @@ -108,50 +137,236 @@ public void parseQuery2MType() throws Exception { assertEquals("avg", sub2.getAggregator()); assertEquals("sys.cpu.1", sub2.getMetric()); } - + @Test public void parseQueryMTypeWRate() throws Exception { - HttpQuery query = NettyMocks.getQuery(tsdb, + HttpQuery query = NettyMocks.getQuery(tsdb, "/api/query?start=1h-ago&m=sum:rate:sys.cpu.0"); - TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); TSSubQuery sub = tsq.getQueries().get(0); assertTrue(sub.getRate()); } - + @Test public void parseQueryMTypeWDS() throws Exception { - HttpQuery query = NettyMocks.getQuery(tsdb, + HttpQuery query = NettyMocks.getQuery(tsdb, "/api/query?start=1h-ago&m=sum:1h-avg:sys.cpu.0"); - TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); TSSubQuery sub = tsq.getQueries().get(0); assertEquals("1h-avg", sub.getDownsample()); } - + + @Test + public void parseQueryMTypeWDSAndFill() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:1h-avg-lerp:sys.cpu.0"); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); + TSSubQuery sub = tsq.getQueries().get(0); + assertEquals("1h-avg-lerp", sub.getDownsample()); + } + @Test public void parseQueryMTypeWRateAndDS() throws Exception { - HttpQuery query = NettyMocks.getQuery(tsdb, + HttpQuery query = NettyMocks.getQuery(tsdb, "/api/query?start=1h-ago&m=sum:1h-avg:rate:sys.cpu.0"); - TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); TSSubQuery sub = tsq.getQueries().get(0); assertTrue(sub.getRate()); assertEquals("1h-avg", sub.getDownsample()); } - + @Test public void parseQueryMTypeWTag() throws Exception { - HttpQuery query = NettyMocks.getQuery(tsdb, + HttpQuery query = NettyMocks.getQuery(tsdb, "/api/query?start=1h-ago&m=sum:sys.cpu.0{host=web01}"); - TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); TSSubQuery sub = tsq.getQueries().get(0); assertNotNull(sub.getTags()); - assertEquals("web01", sub.getTags().get("host")); + assertEquals("literal_or(web01)", sub.getTags().get("host")); + } + + @Test + public void parseQueryMTypeWGroupByRegex() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:sys.cpu.0{host=" + + TagVRegexFilter.FILTER_NAME + "(something(foo|bar))}"); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); + TSSubQuery sub = tsq.getQueries().get(0); + sub.validateAndSetQuery(); + assertEquals(1, sub.getFilters().size()); + assertTrue(sub.getFilters().get(0) instanceof TagVRegexFilter); + } + + @Test + public void parseQueryMTypeWGroupByWildcardExplicit() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:sys.cpu.0{host=" + + TagVWildcardFilter.FILTER_NAME + "(*quirm)}"); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); + TSSubQuery sub = tsq.getQueries().get(0); + sub.validateAndSetQuery(); + assertEquals(1, sub.getFilters().size()); + assertTrue(sub.getFilters().get(0) instanceof TagVWildcardFilter); } - + + @Test + public void parseQueryMTypeWGroupByWildcardImplicit() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:sys.cpu.0{host=*quirm}"); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); + TSSubQuery sub = tsq.getQueries().get(0); + sub.validateAndSetQuery(); + assertEquals(1, sub.getFilters().size()); + assertTrue(sub.getFilters().get(0) instanceof TagVWildcardFilter); + } + + @Test + public void parseQueryMTypeWWildcardFilterExplicit() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:sys.cpu.0{}{host=wildcard(*quirm)}"); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); + TSSubQuery sub = tsq.getQueries().get(0); + sub.validateAndSetQuery(); + assertEquals(1, sub.getFilters().size()); + assertTrue(sub.getFilters().get(0) instanceof TagVWildcardFilter); + } + + @Test + public void parseQueryMTypeWWildcardFilterImplicit() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:sys.cpu.0{}{host=*quirm}"); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); + TSSubQuery sub = tsq.getQueries().get(0); + sub.validateAndSetQuery(); + assertEquals(1, sub.getFilters().size()); + assertTrue(sub.getFilters().get(0) instanceof TagVWildcardFilter); + } + + @Test + public void parseQueryMTypeWGroupByAndWildcardFilterExplicit() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:sys.cpu.0{colo=lga}{host=wildcard(*quirm)}"); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); + TSSubQuery sub = tsq.getQueries().get(0); + sub.validateAndSetQuery(); + assertTrue(sub.getFilters().get(0) instanceof TagVWildcardFilter); + assertTrue(sub.getFilters().get(1) instanceof TagVLiteralOrFilter); + } + + @Test + public void parseQueryMTypeWGroupByAndWildcardFilterSameTagK() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:sys.cpu.0{host=quirm|tsort}" + + "{host=wildcard(*quirm)}"); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); + TSSubQuery sub = tsq.getQueries().get(0); + sub.validateAndSetQuery(); + assertTrue(sub.getFilters().get(0) instanceof TagVWildcardFilter); + assertTrue(sub.getFilters().get(1) instanceof TagVLiteralOrFilter); + } + + @Test + public void parseQueryMTypeWGroupByFilterAndWildcardFilterSameTagK() + throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:sys.cpu.0{host=wildcard(*tsort)}" + + "{host=wildcard(*quirm)}"); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); + TSSubQuery sub = tsq.getQueries().get(0); + sub.validateAndSetQuery(); + assertEquals(2, sub.getFilters().size()); + assertTrue(sub.getFilters().get(0) instanceof TagVWildcardFilter); + assertTrue(sub.getFilters().get(1) instanceof TagVWildcardFilter); + } + + @Test (expected = IllegalArgumentException.class) + public void parseQueryMTypeWGroupByFilterMissingClose() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:sys.cpu.0{host=wildcard(*tsort)}" + + "{host=wildcard(*quirm)"); + parseQuery.invoke(rpc, tsdb, query, expressions); + } + + @Test (expected = IllegalArgumentException.class) + public void parseQueryMTypeWGroupByFilterMissingEquals() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:sys.cpu.0{host=wildcard(*tsort)}" + + "{hostwildcard(*quirm)}"); + parseQuery.invoke(rpc, tsdb, query, expressions); + } + + @Test (expected = IllegalArgumentException.class) + public void parseQueryMTypeWGroupByNoSuchFilter() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:sys.cpu.0{host=nosuchfilter(*tsort)}" + + "{host=dummyfilter(*quirm)}"); + parseQuery.invoke(rpc, tsdb, query, expressions); + } + + @Test + public void parseQueryMTypeWEmptyFilterBrackets() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:sys.cpu.0{}{}"); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); + TSSubQuery sub = tsq.getQueries().get(0); + sub.validateAndSetQuery(); + assertEquals(0, sub.getFilters().size()); + } + + @Test + public void parseQueryMTypeWExplicit() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:explicit_tags:sys.cpu.0{host=web01}"); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); + TSSubQuery sub = tsq.getQueries().get(0); + assertNotNull(sub.getTags()); + assertEquals("literal_or(web01)", sub.getTags().get("host")); + assertTrue(sub.getExplicitTags()); + } + + @Test + public void parseQueryMTypeWExplicitAndRate() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:explicit_tags:rate:sys.cpu.0{host=web01}"); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); + TSSubQuery sub = tsq.getQueries().get(0); + assertNotNull(sub.getTags()); + assertEquals("literal_or(web01)", sub.getTags().get("host")); + assertTrue(sub.getRate()); + assertTrue(sub.getExplicitTags()); + } + + @Test + public void parseQueryMTypeWExplicitAndRateAndDS() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:explicit_tags:rate:1m-sum:sys.cpu.0{host=web01}"); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); + TSSubQuery sub = tsq.getQueries().get(0); + assertNotNull(sub.getTags()); + assertEquals("literal_or(web01)", sub.getTags().get("host")); + assertTrue(sub.getRate()); + assertTrue(sub.getExplicitTags()); + assertEquals("1m-sum", sub.getDownsample()); + } + + @Test + public void parseQueryMTypeWExplicitAndDSAndRate() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:explicit_tags:1m-sum:rate:sys.cpu.0{host=web01}"); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); + TSSubQuery sub = tsq.getQueries().get(0); + assertNotNull(sub.getTags()); + assertEquals("literal_or(web01)", sub.getTags().get("host")); + assertTrue(sub.getRate()); + assertTrue(sub.getExplicitTags()); + assertEquals("1m-sum", sub.getDownsample()); + } + @Test public void parseQueryTSUIDType() throws Exception { - HttpQuery query = NettyMocks.getQuery(tsdb, + HttpQuery query = NettyMocks.getQuery(tsdb, "/api/query?start=1h-ago&tsuid=sum:010101"); - TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); assertNotNull(tsq); assertEquals("1h-ago", tsq.getStart()); assertNotNull(tsq.getQueries()); @@ -161,12 +376,12 @@ public void parseQueryTSUIDType() throws Exception { assertEquals(1, sub.getTsuids().size()); assertEquals("010101", sub.getTsuids().get(0)); } - + @Test public void parseQueryTSUIDTypeMulti() throws Exception { - HttpQuery query = NettyMocks.getQuery(tsdb, + HttpQuery query = NettyMocks.getQuery(tsdb, "/api/query?start=1h-ago&tsuid=sum:010101,020202"); - TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); assertNotNull(tsq); assertEquals("1h-ago", tsq.getStart()); assertNotNull(tsq.getQueries()); @@ -177,12 +392,12 @@ public void parseQueryTSUIDTypeMulti() throws Exception { assertEquals("010101", sub.getTsuids().get(0)); assertEquals("020202", sub.getTsuids().get(1)); } - + @Test public void parseQuery2TSUIDType() throws Exception { - HttpQuery query = NettyMocks.getQuery(tsdb, + HttpQuery query = NettyMocks.getQuery(tsdb, "/api/query?start=1h-ago&tsuid=sum:010101&tsuid=avg:020202"); - TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); assertNotNull(tsq); assertEquals("1h-ago", tsq.getStart()); assertNotNull(tsq.getQueries()); @@ -198,12 +413,12 @@ public void parseQuery2TSUIDType() throws Exception { assertEquals(1, sub.getTsuids().size()); assertEquals("020202", sub.getTsuids().get(0)); } - + @Test public void parseQueryTSUIDTypeWRate() throws Exception { - HttpQuery query = NettyMocks.getQuery(tsdb, + HttpQuery query = NettyMocks.getQuery(tsdb, "/api/query?start=1h-ago&tsuid=sum:rate:010101"); - TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); assertNotNull(tsq); assertEquals("1h-ago", tsq.getStart()); assertNotNull(tsq.getQueries()); @@ -214,12 +429,12 @@ public void parseQueryTSUIDTypeWRate() throws Exception { assertEquals("010101", sub.getTsuids().get(0)); assertTrue(sub.getRate()); } - + @Test public void parseQueryTSUIDTypeWDS() throws Exception { - HttpQuery query = NettyMocks.getQuery(tsdb, + HttpQuery query = NettyMocks.getQuery(tsdb, "/api/query?start=1h-ago&tsuid=sum:1m-sum:010101"); - TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); assertNotNull(tsq); assertEquals("1h-ago", tsq.getStart()); assertNotNull(tsq.getQueries()); @@ -230,12 +445,12 @@ public void parseQueryTSUIDTypeWDS() throws Exception { assertEquals("010101", sub.getTsuids().get(0)); assertEquals("1m-sum", sub.getDownsample()); } - + @Test public void parseQueryTSUIDTypeWRateAndDS() throws Exception { - HttpQuery query = NettyMocks.getQuery(tsdb, + HttpQuery query = NettyMocks.getQuery(tsdb, "/api/query?start=1h-ago&tsuid=sum:1m-sum:rate:010101"); - TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); assertNotNull(tsq); assertEquals("1h-ago", tsq.getStart()); assertNotNull(tsq.getQueries()); @@ -247,78 +462,340 @@ public void parseQueryTSUIDTypeWRateAndDS() throws Exception { assertEquals("1m-sum", sub.getDownsample()); assertTrue(sub.getRate()); } - + @Test public void parseQueryWPadding() throws Exception { - HttpQuery query = NettyMocks.getQuery(tsdb, + HttpQuery query = NettyMocks.getQuery(tsdb, "/api/query?start=1h-ago&m=sum:sys.cpu.0&padding"); - TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); assertNotNull(tsq); assertTrue(tsq.getPadding()); } - + @Test (expected = BadRequestException.class) public void parseQueryStartMissing() throws Exception { - HttpQuery query = NettyMocks.getQuery(tsdb, + HttpQuery query = NettyMocks.getQuery(tsdb, "/api/query?end=1h-ago&m=sum:sys.cpu.0"); - parseQuery.invoke(rpc, tsdb, query); + parseQuery.invoke(rpc, tsdb, query, expressions); } - + @Test (expected = BadRequestException.class) public void parseQueryNoSubQuery() throws Exception { - HttpQuery query = NettyMocks.getQuery(tsdb, + HttpQuery query = NettyMocks.getQuery(tsdb, "/api/query?start=1h-ago"); - parseQuery.invoke(rpc, tsdb, query); + parseQuery.invoke(rpc, tsdb, query, expressions); } - + @Test public void postQuerySimplePass() throws Exception { - Deferred<ArrayList<DataPoints[]>> deferredMock = - (Deferred<ArrayList<DataPoints[]>>)mock(Deferred.class); - PowerMockito.mockStatic(Deferred.class); - PowerMockito.when(Deferred.groupInOrder(Matchers.anyCollection())) - .thenReturn(deferredMock); - PowerMockito.when(deferredMock.joinUninterruptibly()) - .thenReturn(null); - PowerMockito.when(deferredMock.addCallback(Matchers.any(com.stumbleupon.async.Callback.class))) - .thenReturn(deferredMock); + final DataPoints[] datapoints = new DataPoints[1]; + datapoints[0] = new MockDataPoints().getMock(); + when(query_result.runAsync()).thenReturn( + Deferred.fromResult(datapoints)); HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query", "{\"start\":1425440315306,\"queries\":" + "[{\"metric\":\"somemetric\",\"aggregator\":\"sum\",\"rate\":true," + "\"rateOptions\":{\"counter\":false}}]}"); + NettyMocks.mockChannelFuture(query); rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); } - @Test (expected = BadRequestException.class) + @Test public void postQueryNoMetricBadRequest() throws Exception { - Deferred<ArrayList<DataPoints[]>> deferredMock = - (Deferred<ArrayList<DataPoints[]>>)mock(Deferred.class); - PowerMockito.mockStatic(Deferred.class); - PowerMockito.when(Deferred.groupInOrder(Matchers.anyCollection())) - .thenReturn(deferredMock); - PowerMockito.when(deferredMock.joinUninterruptibly()) - .thenReturn(null); - PowerMockito.when(deferredMock.addCallback( - Matchers.any(com.stumbleupon.async.Callback.class))) - .thenReturn(deferredMock); - - Query mockQuery = mock(Query.class); - PowerMockito.doThrow(new NoSuchUniqueName("metric", "nonexistent")) - .when(mockQuery).setTimeSeries( - Matchers.anyString(), - Matchers.anyMap(), - Matchers.any(net.opentsdb.core.Aggregator.class), - Matchers.anyBoolean(), - Matchers.any(net.opentsdb.core.RateOptions.class)); - when(tsdb.newQuery()).thenReturn(mockQuery); + final DeferredGroupException dge = mock(DeferredGroupException.class); + when(dge.getCause()).thenReturn(new NoSuchUniqueName("foo", "metrics")); + + when(query_result.configureFromQuery((TSQuery)any(), anyInt())) + .thenReturn(Deferred.fromError(dge)); HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query", "{\"start\":1425440315306,\"queries\":" + "[{\"metric\":\"nonexistent\",\"aggregator\":\"sum\",\"rate\":true," + "\"rateOptions\":{\"counter\":false}}]}"); rpc.execute(tsdb, query); + assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); + final String json = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(json.contains("No such name for 'foo': 'metrics'")); + } + + @Test + public void executeEmpty() throws Exception { + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:sys.cpu.user"); + NettyMocks.mockChannelFuture(query); + rpc.execute(tsdb, query); + final String json = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertEquals("[]", json); + } + + @Test + public void executeURI() throws Exception { + final DataPoints[] datapoints = new DataPoints[1]; + datapoints[0] = new MockDataPoints().getMock(); + when(query_result.runAsync()).thenReturn( + Deferred.fromResult(datapoints)); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:sys.cpu.user"); + NettyMocks.mockChannelFuture(query); + rpc.execute(tsdb, query); + final String json = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"metric\":\"system.cpu.user\"")); + } + + @Test + public void executeURIDuplicates() throws Exception { + final DataPoints[] datapoints = new DataPoints[1]; + datapoints[0] = new MockDataPoints().getMock(); + when(query_result.runAsync()).thenReturn( + Deferred.fromResult(datapoints)); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:sys.cpu.user&m=sum:sys.cpu.user" + + "&m=sum:sys.cpu.user"); + NettyMocks.mockChannelFuture(query); + rpc.execute(tsdb, query); + final String json = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"metric\":\"system.cpu.user\"")); + } + + @Test + public void executeNSU() throws Exception { + final DeferredGroupException dge = mock(DeferredGroupException.class); + when(dge.getCause()).thenReturn(new NoSuchUniqueName("foo", "metrics")); + + when(query_result.configureFromQuery((TSQuery)any(), anyInt())) + .thenReturn(Deferred.fromError(dge)); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:sys.cpu.user"); + rpc.execute(tsdb, query); + assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); + final String json = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(json.contains("No such name for 'foo': 'metrics'")); + } + + @Test + public void executeWithBadDSFill() throws Exception { + final DataPoints[] datapoints = new DataPoints[1]; + datapoints[0] = new MockDataPoints().getMock(); + when(query_result.runAsync()).thenReturn( + Deferred.fromResult(datapoints)); + + try { + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:10m-avg-badbadbad:sys.cpu.user"); + rpc.execute(tsdb, query); + fail("expected BadRequestException"); + } catch (final BadRequestException exn) { + System.out.println(exn.getMessage()); + assertTrue(exn.getMessage().startsWith( + "Unrecognized fill policy: badbadbad")); + } + } + + @Test + public void executePOST() throws Exception { + final DataPoints[] datapoints = new DataPoints[1]; + datapoints[0] = new MockDataPoints().getMock(); + when(query_result.runAsync()).thenReturn( + Deferred.fromResult(datapoints)); + + final HttpQuery query = NettyMocks.postQuery(tsdb,"/api/query", + "{\"start\":\"1h-ago\",\"queries\":" + + "[{\"metric\":\"sys.cpu.user\",\"aggregator\":\"sum\"}]}"); + NettyMocks.mockChannelFuture(query); + rpc.execute(tsdb, query); + final String json = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"metric\":\"system.cpu.user\"")); + } + + @Test + public void executePOSTDuplicates() throws Exception { + final DataPoints[] datapoints = new DataPoints[1]; + datapoints[0] = new MockDataPoints().getMock(); + when(query_result.runAsync()).thenReturn( + Deferred.fromResult(datapoints)); + + final HttpQuery query = NettyMocks.postQuery(tsdb,"/api/query", + "{\"start\":\"1h-ago\",\"queries\":" + + "[{\"metric\":\"sys.cpu.user\",\"aggregator\":\"sum\"}," + + "{\"metric\":\"sys.cpu.user\",\"aggregator\":\"sum\"}]}"); + NettyMocks.mockChannelFuture(query); + rpc.execute(tsdb, query); + final String json = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"metric\":\"system.cpu.user\"")); + } + + @Test (expected = BadRequestException.class) + public void deleteDatapointsBadRequest() throws Exception { + HttpQuery query = NettyMocks.deleteQuery(tsdb, + "/api/query?start=1356998400&m=sum:sys.cpu.user", ""); + rpc.execute(tsdb, query); + assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); + final String json = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(json.contains("Deleting data is not enabled")); + } + + @Test + public void gexp() throws Exception { + final DataPoints[] datapoints = new DataPoints[1]; + datapoints[0] = new MockDataPoints().getMock(); + when(query_result.runAsync()).thenReturn( + Deferred.fromResult(datapoints)); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/gexp?start=1h-ago&exp=scale(sum:sys.cpu.user,1)"); + NettyMocks.mockChannelFuture(query); + rpc.execute(tsdb, query); + assertEquals(query.response().getStatus(), HttpResponseStatus.OK); + final String json = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"metric\":\"system.cpu.user\"")); + } + + @Test + public void gexpBadExpression() throws Exception { + final DataPoints[] datapoints = new DataPoints[1]; + datapoints[0] = new MockDataPoints().getMock(); + when(query_result.runAsync()).thenReturn( + Deferred.fromResult(datapoints)); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/gexp?start=1h-ago&exp=scale(sum:sys.cpu.user,notanumber)"); + rpc.execute(tsdb, query); + assertEquals(query.response().getStatus(), HttpResponseStatus.BAD_REQUEST); + final String json = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(json.contains("factor")); + } + + @Test + public void testParsePercentile() { + final String s = "percentile[0.98,0.95,0.99]"; + final String ss = "percentile [0.98,0.95,0.99]"; + final String sss = "percentile[ 0.98,0.95,0.99]"; + final String ssss = "percentile[0.98,0.95,0.99 ]"; + final String sssss = "percentile[ 0.98, 0.95,0.99]"; + List<String> strs = new ArrayList<String>(); + strs.add(sssss); + strs.add(ssss); + strs.add(sss); + strs.add(ss); + strs.add(s); + + for (String str : strs) { + List<Float> fs = QueryRpc.parsePercentiles(str); + assertEquals(3, fs.size()); + assertEquals(0.98, fs.get(0), 0.0001); + assertEquals(0.95, fs.get(1), 0.0001); + assertEquals(0.99, fs.get(2), 0.0001); + } + } + + @Test + public void parseHistogramQueryMType() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:percentiles[0.98]:msg.end2end.latency"); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); + assertNotNull(tsq); + assertEquals("1h-ago", tsq.getStart()); + assertNotNull(tsq.getQueries()); + TSSubQuery sub = tsq.getQueries().get(0); + + assertNotNull(sub); + assertEquals("sum", sub.getAggregator()); + assertEquals("msg.end2end.latency", sub.getMetric()); + assertEquals(0.98f, sub.getPercentiles().get(0).floatValue(), 0.0001); + } + + @Test + public void parseHistogramQueryTSUIDType() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&tsuid=sum:percentiles[0.98]:010101"); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions); + assertNotNull(tsq); + assertEquals("1h-ago", tsq.getStart()); + assertNotNull(tsq.getQueries()); + TSSubQuery sub = tsq.getQueries().get(0); + assertNotNull(sub); + assertEquals("sum", sub.getAggregator()); + assertEquals(1, sub.getTsuids().size()); + assertEquals("010101", sub.getTsuids().get(0)); + assertEquals(0.98f, sub.getPercentiles().get(0).floatValue(), 0.0001); + } + + @Test + public void v1AuthAllowed() throws Exception { + final TSDB tsdb = NettyMocks.getMockedHTTPTSDBWithAuthEnabled(AuthStatus.SUCCESS); + when(tsdb.newQuery()).thenReturn(query_result); + when(query_result.configureFromQuery((TSQuery)any(), anyInt())) + .thenReturn(Deferred.fromResult(null)); + when(query_result.runAsync()) + .thenReturn(Deferred.fromResult(new DataPoints[0])); + + final DataPoints[] datapoints = new DataPoints[1]; + datapoints[0] = new MockDataPoints().getMock(); + when(query_result.runAsync()).thenReturn(Deferred.fromResult(datapoints)); + + final HttpQuery query = NettyMocks.postQuery(tsdb,"/api/query", + "{\"start\":\"1h-ago\",\"queries\":" + + "[{\"metric\":\"sys.cpu.user\",\"aggregator\":\"sum\"}]}"); + + NettyMocks.mockChannelFuture(query); + rpc.execute(tsdb, query); + final String json = query.response().getContent().toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"metric\":\"system.cpu.user\"")); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + } + + @Test + public void v1AuthUnauthorized() throws Exception { + final TSDB tsdb = NettyMocks.getMockedHTTPTSDBWithAuthEnabled(AuthStatus.UNAUTHORIZED); + final DataPoints[] datapoints = new DataPoints[1]; + datapoints[0] = new MockDataPoints().getMock(); + when(query_result.runAsync()).thenReturn( + Deferred.fromResult(datapoints)); + + final HttpQuery query = NettyMocks.getQuery(tsdb,"/api/query?start=1h-ago&m=sum:sys.cpu.user"); + + try { + TestHttpQuery.mockChannelFuture(query); + rpc.execute(tsdb, query); + fail("Expected BadRequestException"); + } catch (BadRequestException e) { + assertEquals(HttpResponseStatus.UNAUTHORIZED, e.getStatus()); + } + } + + @Test + public void v1AuthForbidden() throws Exception { + final TSDB tsdb = NettyMocks.getMockedHTTPTSDBWithAuthEnabled(AuthStatus.FORBIDDEN); + final DataPoints[] datapoints = new DataPoints[1]; + datapoints[0] = new MockDataPoints().getMock(); + when(query_result.runAsync()).thenReturn( + Deferred.fromResult(datapoints)); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:sys.cpu.user"); + + try { + TestHttpQuery.mockChannelFuture(query); + rpc.execute(tsdb, query); + fail("Expected BadRequestException"); + } catch (BadRequestException e) { + assertEquals(HttpResponseStatus.FORBIDDEN, e.getStatus()); + } } //TODO(cl) add unit tests for the rate options parsing diff --git a/test/tsd/TestQueryRpcLastDataPoint.java b/test/tsd/TestQueryRpcLastDataPoint.java new file mode 100644 index 0000000000..1494d95806 --- /dev/null +++ b/test/tsd/TestQueryRpcLastDataPoint.java @@ -0,0 +1,961 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.tsd; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.nio.charset.Charset; + +import net.opentsdb.core.BaseTsdbTest; +import net.opentsdb.core.Query; +import net.opentsdb.core.TSDB; +import net.opentsdb.meta.TestTSUIDQuery; +import net.opentsdb.storage.MockBase; +import net.opentsdb.uid.UniqueId; +import net.opentsdb.utils.Config; +import net.opentsdb.utils.DateTime; + +import org.hbase.async.HBaseClient; +import org.hbase.async.KeyValue; +import org.hbase.async.Scanner; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +import com.stumbleupon.async.Deferred; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({ TSDB.class, HBaseClient.class, Config.class, HttpQuery.class, + Query.class, Deferred.class, UniqueId.class, DateTime.class, KeyValue.class, + Scanner.class }) +public class TestQueryRpcLastDataPoint extends BaseTsdbTest { + private QueryRpc rpc; + + @Before + public void beforeLocal() throws Exception { + Whitebox.setInternalState(config, "enable_tsuid_incrementing", true); + Whitebox.setInternalState(config, "enable_realtime_ts", true); + rpc = new QueryRpc(); + storage = new MockBase(tsdb, client, true, true, true, true); + TestTSUIDQuery.setupStorage(tsdb, storage); + } + + @Test + public void qsMetricMeta() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user{host=web01}"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsMetricMetaScan() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsMetricMetaScanResolve() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user&resolve"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertTrue(json.contains("\"metric\":\"sys.cpu.user\"")); + assertTrue(json.contains("\"tags\":{\"host\":\"web01\"}")); + assertTrue(json.contains("\"tags\":{\"host\":\"web02\"}")); + } + + @Test + public void qsMetricMetaScanOneMissing() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertFalse(json.contains("\"value\":\"42\"")); + assertFalse(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsMetricMetaScanNoResults() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals("[]", json); + } + + @Test + public void qsMetricMetaScanBackscanZero() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user&back_scan=0"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsMetricBackscan() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1356998400L, 42, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user{host=web01}&back_scan=1"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1356998400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsMetricBackscanResolved() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1356998400L, 42, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user{host=web01}" + + "&back_scan=1&resolve=true"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1356998400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"metric\":\"sys.cpu.user\"")); + assertTrue(json.contains("\"tags\":{\"host\":\"web01\"}")); + } + + @Test + public void qsMetricBackscanNoResult() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user{host=web01}&back_scan=1"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertEquals("[]", json); + } + + @Test + public void qsMetricTwoQueriesBackscan() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1356998400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1356998400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user{host=web01}" + + "×eries=sys.cpu.user{host=web02}&back_scan=1"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1356998400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsMetricTwoQueriesBackscanResolve() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1356998400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1356998400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user{host=web01}" + + "×eries=sys.cpu.user{host=web02}&back_scan=1&resolve"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1356998400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertTrue(json.contains("\"metric\":\"sys.cpu.user\"")); + assertTrue(json.contains("\"tags\":{\"host\":\"web01\"}")); + assertTrue(json.contains("\"tags\":{\"host\":\"web02\"}")); + } + + @Test + public void qsMetricTwoQueriesOneMissingBackscan() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1356998400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user{host=web01}" + + "×eries=sys.cpu.user{host=web02}&back_scan=1"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1356998400000")); + assertFalse(json.contains("\"value\":\"42\"")); + assertFalse(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsMetricBackscanMissingTags() throws Exception { + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user&back_scan=1"); + try { + rpc.execute(tsdb, query); + fail("Expected a BadRequestException"); + } catch (BadRequestException e) { + assertTrue(e.getMessage().contains("Tags")); + } + } + + @Test + public void qsMetricNSUNMetric() throws Exception { + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.nice{host=web01}"); + try { + rpc.execute(tsdb, query); + fail("Expected a BadRequestException"); + } catch (BadRequestException e) { + assertTrue(e.getMessage().contains("No such name")); + assertTrue(e.getMessage().contains("metric")); + } + } + + @Test + public void qsMetricNSUNTagk() throws Exception { + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user{dc=web01}"); + try { + rpc.execute(tsdb, query); + fail("Expected a BadRequestException"); + } catch (BadRequestException e) { + assertTrue(e.getMessage().contains("No such name")); + assertTrue(e.getMessage().contains("tagk")); + } + } + + @Test + public void qsMetricNSUNTagv() throws Exception { + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user{host=web03}"); + try { + rpc.execute(tsdb, query); + fail("Expected a BadRequestException"); + } catch (BadRequestException e) { + assertTrue(e.getMessage().contains("No such name")); + assertTrue(e.getMessage().contains("tagv")); + } + } + + @Test + public void qsTSUIDMeta() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?tsuids=000001000001000001"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertFalse(json.contains("\"value\":\"24\"")); + assertFalse(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsTSUIDMetaCommaSeparated() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?tsuids=000001000001000001,000001000001000002"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsTSUIDMetaTwoQueries() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?tsuids=000001000001000001&tsuids=000001000001000002"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsTSUIDMetaNoResults() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?tsuids=000001000001000001"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals("[]", json); + } + + @Test + public void qsTSUIDBackscan() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1356998400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1356998400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?tsuids=000001000001000001&back_scan=1"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1356998400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsTSUIDBackscanNoResult() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1356998400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?tsuids=000001000001000001&back_scan=1"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertEquals("[]", json); + } + + @Test + public void qsTSUIDCommaSeparatedBackscan() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1356998400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1356998400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?tsuids=000001000001000001,000001000001000002&back_scan=1"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1356998400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsTSUIDCommaSeparatedOneMissingBackscan() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1356998400L, 42, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?tsuids=000001000001000001,000001000001000002&back_scan=1"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1356998400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertFalse(json.contains("\"value\":\"24\"")); + assertFalse(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsTSUIDTwoQueriesBackscan() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1356998400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1356998400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?tsuids=000001000001000001" + + "&tsuids=000001000001000002&back_scan=1"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1356998400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsTSUIDTwoQueriesOneMissingBackscan() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1356998400L, 42, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?tsuids=000001000001000001" + + "&tsuids=000001000001000002&back_scan=1"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1356998400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertFalse(json.contains("\"value\":\"24\"")); + assertFalse(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsTSUIDNSUIMetric() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + storage.addColumn(MockBase.stringToBytes("00000350E22700000001000001"), + new byte[] { 0, 0 }, new byte[] { 0x2A }); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?tsuids=000003000001000001&back_scan=1&resolve"); + try { + rpc.execute(tsdb, query); + fail("Expected a BadRequestException"); + } catch (BadRequestException e) { + assertTrue(e.getMessage().contains("No such unique ID")); + assertTrue(e.getMessage().contains("metric")); + } + } + + @Test + public void qsTSUIDNSUITagk() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + storage.addColumn(MockBase.stringToBytes("00000150E22700000004000001"), + new byte[] { 0, 0 }, new byte[] { 0x2A }); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?tsuids=000001000004000001&back_scan=1&resolve"); + try { + rpc.execute(tsdb, query); + fail("Expected a BadRequestException"); + } catch (BadRequestException e) { + assertTrue(e.getMessage().contains("No such unique ID")); + assertTrue(e.getMessage().contains("tagk")); + } + } + + @Test + public void qsTSUIDNSUITagv() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + storage.addColumn(MockBase.stringToBytes("00000150E22700000001000003"), + new byte[] { 0, 0 }, new byte[] { 0x2A }); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?tsuids=000001000001000003&back_scan=1&resolve"); + try { + rpc.execute(tsdb, query); + fail("Expected a BadRequestException"); + } catch (BadRequestException e) { + assertTrue(e.getMessage().contains("No such unique ID")); + assertTrue(e.getMessage().contains("tagv")); + } + } + + @Test + public void qsDualMeta() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user{host=web01}" + + "&tsuids=000001000001000002"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsDualBackscan() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1356998400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1356998400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user{host=web01}" + + "&tsuids=000001000001000002&back_scan=1"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1356998400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsEmpty() throws Exception { + final HttpQuery query = NettyMocks.getQuery(tsdb, "/api/query/last"); + try { + rpc.execute(tsdb, query); + fail("Expected a BadRequestException"); + } catch (BadRequestException e) { } + } + + @Test + public void postMetricMetaWithTags() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{\"queries\":[{\"metric\":\"sys.cpu.user\",\"tags\":" + + "{\"host\":\"web01\"}}]}"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void postMetricMetaWithoutTags() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{\"queries\":[{\"metric\":\"sys.cpu.user\"}]}"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void postMetricMetaWithoutTagsResolve() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{\"queries\":[{\"metric\":\"sys.cpu.user\"}],\"resolveNames\":true}"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertTrue(json.contains("\"metric\":\"sys.cpu.user\"")); + assertTrue(json.contains("\"tags\":{\"host\":\"web01\"}")); + assertTrue(json.contains("\"tags\":{\"host\":\"web02\"}")); + } + + @Test + public void postMetricMetaTwoQueries() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{\"queries\":[{\"metric\":\"sys.cpu.user\",\"tags\":" + + "{\"host\":\"web01\"}}," + + "{\"metric\":\"sys.cpu.user\",\"tags\":" + + "{\"host\":\"web02\"}}]}"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void postMetricBackscanWithTags() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1356998400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1356998400L, 24, tags); + + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{\"queries\":[{\"metric\":\"sys.cpu.user\",\"tags\":" + + "{\"host\":\"web01\"}}],\"backScan\":1}"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1356998400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void postTSUIDMeta() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{\"queries\":[{\"tsuids\":[\"000001000001000001\"]}]}"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertFalse(json.contains("\"value\":\"24\"")); + assertFalse(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void postTSUIDMetaList() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{\"queries\":[{\"tsuids\":[\"000001000001000001\"," + + "\"000001000001000002\"]}]}"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void postTSUIDMetaResolve() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{\"queries\":[{\"tsuids\":[\"000001000001000001\"," + + "\"000001000001000002\"]}],\"resolveNames\":true}"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertTrue(json.contains("\"metric\":\"sys.cpu.user\"")); + assertTrue(json.contains("\"tags\":{\"host\":\"web01\"}")); + assertTrue(json.contains("\"tags\":{\"host\":\"web02\"}")); + } + + @Test + public void postTSUIDMetaTwoQueries() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{\"queries\":[{\"tsuids\":[\"000001000001000001\"]}," + + "{\"tsuids\":[\"000001000001000002\"]}]}"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void postTSUIDBackscan() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1356998400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1356998400L, 24, tags); + + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{\"queries\":[{\"tsuids\":[\"000001000001000001\"]}],\"backScan\":1}"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1356998400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertFalse(json.contains("\"value\":\"24\"")); + assertFalse(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void postDualMeta() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{\"queries\":[{\"metric\":\"sys.cpu.user\",\"tags\":" + + "{\"host\":\"web01\"}}," + + "{\"tsuids\":[\"000001000001000002\"]}]}"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void postDualMetaResolve() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{\"queries\":[{\"metric\":\"sys.cpu.user\",\"tags\":" + + "{\"host\":\"web01\"}}," + + "{\"tsuids\":[\"000001000001000002\"]}]," + + "\"resolveNames\":true}"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertTrue(json.contains("\"metric\":\"sys.cpu.user\"")); + assertTrue(json.contains("\"tags\":{\"host\":\"web01\"}")); + assertTrue(json.contains("\"tags\":{\"host\":\"web02\"}")); + } + + @Test + public void postEmpty() throws Exception { + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{\"queries\":[]}"); + try { + rpc.execute(tsdb, query); + fail("Expected a BadRequestException"); + } catch (BadRequestException e) { } + } + + @Test + public void postEmptyList() throws Exception { + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{}"); + try { + rpc.execute(tsdb, query); + fail("Expected a BadRequestException"); + } catch (BadRequestException e) { } + } + + /** + * Returns the content of the response buffer + * @param query The query to parse + * @return Some string if we were lucky + */ + private String getContent(final HttpQuery query) { + return query.response().getContent().toString(Charset.forName("UTF-8")); + } +} \ No newline at end of file diff --git a/test/tsd/TestQueryRpcLastDataPointWhenEnableAppends.java b/test/tsd/TestQueryRpcLastDataPointWhenEnableAppends.java new file mode 100644 index 0000000000..b91d2d1d47 --- /dev/null +++ b/test/tsd/TestQueryRpcLastDataPointWhenEnableAppends.java @@ -0,0 +1,957 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.tsd; + +import com.stumbleupon.async.Deferred; +import net.opentsdb.core.BaseTsdbTest; +import net.opentsdb.core.Query; +import net.opentsdb.core.TSDB; +import net.opentsdb.meta.TestTSUIDQuery; +import net.opentsdb.storage.MockBase; +import net.opentsdb.uid.UniqueId; +import net.opentsdb.utils.Config; +import net.opentsdb.utils.DateTime; +import org.hbase.async.HBaseClient; +import org.hbase.async.KeyValue; +import org.hbase.async.Scanner; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +import java.nio.charset.Charset; + +import static org.junit.Assert.*; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({ TSDB.class, HBaseClient.class, Config.class, HttpQuery.class, + Query.class, Deferred.class, UniqueId.class, DateTime.class, KeyValue.class, + Scanner.class }) +public class TestQueryRpcLastDataPointWhenEnableAppends extends BaseTsdbTest { + private QueryRpc rpc; + + @Before + public void beforeLocal() throws Exception { + Whitebox.setInternalState(config, "enable_tsuid_incrementing", true); + Whitebox.setInternalState(config, "enable_realtime_ts", true); + Whitebox.setInternalState(config, "enable_appends", true); + rpc = new QueryRpc(); + storage = new MockBase(tsdb, client, true, true, true, true); + TestTSUIDQuery.setupStorage(tsdb, storage); + } + + @Test + public void qsMetricMeta() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user{host=web01}"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsMetricMetaScan() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsMetricMetaScanResolve() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user&resolve"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertTrue(json.contains("\"metric\":\"sys.cpu.user\"")); + assertTrue(json.contains("\"tags\":{\"host\":\"web01\"}")); + assertTrue(json.contains("\"tags\":{\"host\":\"web02\"}")); + } + + @Test + public void qsMetricMetaScanOneMissing() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertFalse(json.contains("\"value\":\"42\"")); + assertFalse(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsMetricMetaScanNoResults() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals("[]", json); + } + + @Test + public void qsMetricMetaScanBackscanZero() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user&back_scan=0"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsMetricBackscan() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1356998400L, 42, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user{host=web01}&back_scan=1"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1356998400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsMetricBackscanResolved() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1356998400L, 42, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user{host=web01}" + + "&back_scan=1&resolve=true"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1356998400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"metric\":\"sys.cpu.user\"")); + assertTrue(json.contains("\"tags\":{\"host\":\"web01\"}")); + } + + @Test + public void qsMetricBackscanNoResult() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user{host=web01}&back_scan=1"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertEquals("[]", json); + } + + @Test + public void qsMetricTwoQueriesBackscan() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1356998400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1356998400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user{host=web01}" + + "×eries=sys.cpu.user{host=web02}&back_scan=1"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1356998400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsMetricTwoQueriesBackscanResolve() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1356998400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1356998400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user{host=web01}" + + "×eries=sys.cpu.user{host=web02}&back_scan=1&resolve"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1356998400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertTrue(json.contains("\"metric\":\"sys.cpu.user\"")); + assertTrue(json.contains("\"tags\":{\"host\":\"web01\"}")); + assertTrue(json.contains("\"tags\":{\"host\":\"web02\"}")); + } + + @Test + public void qsMetricTwoQueriesOneMissingBackscan() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1356998400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user{host=web01}" + + "×eries=sys.cpu.user{host=web02}&back_scan=1"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1356998400000")); + assertFalse(json.contains("\"value\":\"42\"")); + assertFalse(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsMetricBackscanMissingTags() throws Exception { + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user&back_scan=1"); + try { + rpc.execute(tsdb, query); + fail("Expected a BadRequestException"); + } catch (BadRequestException e) { + assertTrue(e.getMessage().contains("Tags")); + } + } + + @Test + public void qsMetricNSUNMetric() throws Exception { + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.nice{host=web01}"); + try { + rpc.execute(tsdb, query); + fail("Expected a BadRequestException"); + } catch (BadRequestException e) { + assertTrue(e.getMessage().contains("No such name")); + assertTrue(e.getMessage().contains("metric")); + } + } + + @Test + public void qsMetricNSUNTagk() throws Exception { + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user{dc=web01}"); + try { + rpc.execute(tsdb, query); + fail("Expected a BadRequestException"); + } catch (BadRequestException e) { + assertTrue(e.getMessage().contains("No such name")); + assertTrue(e.getMessage().contains("tagk")); + } + } + + @Test + public void qsMetricNSUNTagv() throws Exception { + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user{host=web03}"); + try { + rpc.execute(tsdb, query); + fail("Expected a BadRequestException"); + } catch (BadRequestException e) { + assertTrue(e.getMessage().contains("No such name")); + assertTrue(e.getMessage().contains("tagv")); + } + } + + @Test + public void qsTSUIDMeta() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?tsuids=000001000001000001"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertFalse(json.contains("\"value\":\"24\"")); + assertFalse(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsTSUIDMetaCommaSeparated() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?tsuids=000001000001000001,000001000001000002"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsTSUIDMetaTwoQueries() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?tsuids=000001000001000001&tsuids=000001000001000002"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsTSUIDMetaNoResults() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?tsuids=000001000001000001"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals("[]", json); + } + + @Test + public void qsTSUIDBackscan() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1356998400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1356998400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?tsuids=000001000001000001&back_scan=1"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1356998400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsTSUIDBackscanNoResult() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1356998400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?tsuids=000001000001000001&back_scan=1"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertEquals("[]", json); + } + + @Test + public void qsTSUIDCommaSeparatedBackscan() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1356998400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1356998400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?tsuids=000001000001000001,000001000001000002&back_scan=1"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1356998400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsTSUIDCommaSeparatedOneMissingBackscan() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1356998400L, 42, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?tsuids=000001000001000001,000001000001000002&back_scan=1"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1356998400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertFalse(json.contains("\"value\":\"24\"")); + assertFalse(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsTSUIDTwoQueriesBackscan() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1356998400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1356998400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?tsuids=000001000001000001" + + "&tsuids=000001000001000002&back_scan=1"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1356998400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsTSUIDTwoQueriesOneMissingBackscan() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1356998400L, 42, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?tsuids=000001000001000001" + + "&tsuids=000001000001000002&back_scan=1"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1356998400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertFalse(json.contains("\"value\":\"24\"")); + assertFalse(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsTSUIDNSUIMetric() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + storage.addColumn(MockBase.stringToBytes("00000350E22700000001000001"), + new byte[] { 0, 0 }, new byte[] { 0x2A }); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?tsuids=000003000001000001&back_scan=1&resolve"); + try { + rpc.execute(tsdb, query); + fail("Expected a BadRequestException"); + } catch (BadRequestException e) { + assertTrue(e.getMessage().contains("No such unique ID")); + assertTrue(e.getMessage().contains("metric")); + } + } + + @Test + public void qsTSUIDNSUITagk() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + storage.addColumn(MockBase.stringToBytes("00000150E22700000004000001"), + new byte[] { 0, 0 }, new byte[] { 0x2A }); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?tsuids=000001000004000001&back_scan=1&resolve"); + try { + rpc.execute(tsdb, query); + fail("Expected a BadRequestException"); + } catch (BadRequestException e) { + assertTrue(e.getMessage().contains("No such unique ID")); + assertTrue(e.getMessage().contains("tagk")); + } + } + + @Test + public void qsTSUIDNSUITagv() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + storage.addColumn(MockBase.stringToBytes("00000150E22700000001000003"), + new byte[] { 0, 0 }, new byte[] { 0x2A }); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?tsuids=000001000001000003&back_scan=1&resolve"); + try { + rpc.execute(tsdb, query); + fail("Expected a BadRequestException"); + } catch (BadRequestException e) { + assertTrue(e.getMessage().contains("No such unique ID")); + assertTrue(e.getMessage().contains("tagv")); + } + } + + @Test + public void qsDualMeta() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user{host=web01}" + + "&tsuids=000001000001000002"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsDualBackscan() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1356998400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1356998400L, 24, tags); + + final HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query/last?timeseries=sys.cpu.user{host=web01}" + + "&tsuids=000001000001000002&back_scan=1"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1356998400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void qsEmpty() throws Exception { + final HttpQuery query = NettyMocks.getQuery(tsdb, "/api/query/last"); + try { + rpc.execute(tsdb, query); + fail("Expected a BadRequestException"); + } catch (BadRequestException e) { } + } + + @Test + public void postMetricMetaWithTags() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{\"queries\":[{\"metric\":\"sys.cpu.user\",\"tags\":" + + "{\"host\":\"web01\"}}]}"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void postMetricMetaWithoutTags() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{\"queries\":[{\"metric\":\"sys.cpu.user\"}]}"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void postMetricMetaWithoutTagsResolve() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{\"queries\":[{\"metric\":\"sys.cpu.user\"}],\"resolveNames\":true}"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertTrue(json.contains("\"metric\":\"sys.cpu.user\"")); + assertTrue(json.contains("\"tags\":{\"host\":\"web01\"}")); + assertTrue(json.contains("\"tags\":{\"host\":\"web02\"}")); + } + + @Test + public void postMetricMetaTwoQueries() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{\"queries\":[{\"metric\":\"sys.cpu.user\",\"tags\":" + + "{\"host\":\"web01\"}}," + + "{\"metric\":\"sys.cpu.user\",\"tags\":" + + "{\"host\":\"web02\"}}]}"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void postMetricBackscanWithTags() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1356998400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1356998400L, 24, tags); + + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{\"queries\":[{\"metric\":\"sys.cpu.user\",\"tags\":" + + "{\"host\":\"web01\"}}],\"backScan\":1}"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1356998400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void postTSUIDMeta() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{\"queries\":[{\"tsuids\":[\"000001000001000001\"]}]}"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertFalse(json.contains("\"value\":\"24\"")); + assertFalse(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void postTSUIDMetaList() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{\"queries\":[{\"tsuids\":[\"000001000001000001\"," + + "\"000001000001000002\"]}]}"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void postTSUIDMetaResolve() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{\"queries\":[{\"tsuids\":[\"000001000001000001\"," + + "\"000001000001000002\"]}],\"resolveNames\":true}"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertTrue(json.contains("\"metric\":\"sys.cpu.user\"")); + assertTrue(json.contains("\"tags\":{\"host\":\"web01\"}")); + assertTrue(json.contains("\"tags\":{\"host\":\"web02\"}")); + } + + @Test + public void postTSUIDMetaTwoQueries() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{\"queries\":[{\"tsuids\":[\"000001000001000001\"]}," + + "{\"tsuids\":[\"000001000001000002\"]}]}"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void postTSUIDBackscan() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1356998400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1356998400L, 24, tags); + + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{\"queries\":[{\"tsuids\":[\"000001000001000001\"]}],\"backScan\":1}"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1356998400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertFalse(json.contains("\"value\":\"24\"")); + assertFalse(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void postDualMeta() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{\"queries\":[{\"metric\":\"sys.cpu.user\",\"tags\":" + + "{\"host\":\"web01\"}}," + + "{\"tsuids\":[\"000001000001000002\"]}]}"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertFalse(json.contains("\"metric\"")); + assertFalse(json.contains("\"tags\"")); + } + + @Test + public void postDualMetaResolve() throws Exception { + PowerMockito.mockStatic(DateTime.class); + PowerMockito.when(DateTime.currentTimeMillis()).thenReturn(1356998400000L); + tsdb.addPoint("sys.cpu.user", 1388534400L, 42, tags); + tags.put("host", "web02"); + tsdb.addPoint("sys.cpu.user", 1388534400L, 24, tags); + + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{\"queries\":[{\"metric\":\"sys.cpu.user\",\"tags\":" + + "{\"host\":\"web01\"}}," + + "{\"tsuids\":[\"000001000001000002\"]}]," + + "\"resolveNames\":true}"); + rpc.execute(tsdb, query); + final String json = getContent(query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertTrue(json.contains("\"timestamp\":1388534400000")); + assertTrue(json.contains("\"value\":\"42\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(json.contains("\"value\":\"24\"")); + assertTrue(json.contains("\"tsuid\":\"000001000001000002\"")); + assertTrue(json.contains("\"metric\":\"sys.cpu.user\"")); + assertTrue(json.contains("\"tags\":{\"host\":\"web01\"}")); + assertTrue(json.contains("\"tags\":{\"host\":\"web02\"}")); + } + + @Test + public void postEmpty() throws Exception { + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{\"queries\":[]}"); + try { + rpc.execute(tsdb, query); + fail("Expected a BadRequestException"); + } catch (BadRequestException e) { } + } + + @Test + public void postEmptyList() throws Exception { + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/last", + "{}"); + try { + rpc.execute(tsdb, query); + fail("Expected a BadRequestException"); + } catch (BadRequestException e) { } + } + + /** + * Returns the content of the response buffer + * @param query The query to parse + * @return Some string if we were lucky + */ + private String getContent(final HttpQuery query) { + return query.response().getContent().toString(Charset.forName("UTF-8")); + } +} \ No newline at end of file diff --git a/test/tsd/TestRollupRpc.java b/test/tsd/TestRollupRpc.java new file mode 100644 index 0000000000..ea7158f6dc --- /dev/null +++ b/test/tsd/TestRollupRpc.java @@ -0,0 +1,854 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.tsd; + +import net.opentsdb.rollup.RollupConfig; +import net.opentsdb.rollup.RollupInterval; +import net.opentsdb.storage.MockBase; + +import org.hamcrest.CoreMatchers; +import org.hbase.async.HBaseException; +import org.hbase.async.PleaseThrottleException; +import org.jboss.netty.channel.Channel; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; + +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.anyMap; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; + +@RunWith(PowerMockRunner.class) +//"Classloader hell"... It's real. Tell PowerMock to ignore these classes +//because they fiddle with the class loader. We don't test them anyway. +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +public class TestRollupRpc extends BaseTestPutRpc { + private final static byte[] FAMILY = "t".getBytes(MockBase.ASCII()); + protected final static byte[] AGG_TABLE = "tsdb-agg".getBytes(MockBase.ASCII()); + private RollupConfig rollup_config; + protected String agg_tag_key; + protected byte[] row; + + @Before + public void beforeLocal() throws Exception { + agg_tag_key = config.getString("tsd.rollups.agg_tag_key"); + + final List<byte[]> families = new ArrayList<byte[]>(); + families.add(FAMILY); + + rollup_config = RollupConfig.builder() + .addAggregationId("sum", 0) + .addAggregationId("count", 1) + .addAggregationId("max", 2) + .addAggregationId("min", 3) + .addInterval(RollupInterval.builder() + .setTable("tsdb") + .setPreAggregationTable("tsdb-agg") + .setInterval("1m") + .setRowSpan("1h") + .setDefaultInterval(true)) + .addInterval(RollupInterval.builder() + .setTable("tsdb-rollup-1h") + .setPreAggregationTable("tsdb-rollup-agg-1h") + .setInterval("1h") + .setRowSpan("1n")) + .addInterval(RollupInterval.builder() + .setTable("tsdb-rollup-1d") + .setPreAggregationTable("tsdb-rollup-agg-1d") + .setInterval("1d") + .setRowSpan("1n")) + .build(); + Whitebox.setInternalState(tsdb, "rollup_config", rollup_config); + Whitebox.setInternalState(tsdb, "default_interval", + rollup_config.getRollupInterval("1m")); + + storage = new MockBase(tsdb, client, true, true, true, true); + storage.addTable("tsdb-rollup-1h".getBytes(), families); + storage.addTable("tsdb-rollup-agg-1h".getBytes(), families); + storage.addTable("tsdb-agg".getBytes(), families); + Whitebox.setInternalState(tsdb, "rollups_block_derived", true); + Whitebox.setInternalState(tsdb, "rollups_split_queries", false); + Whitebox.setInternalState(tsdb, "agg_tag_key", + config.getString("tsd.rollups.agg_tag_key")); + Whitebox.setInternalState(tsdb, "raw_agg_tag_value", + config.getString("tsd.rollups.raw_agg_tag_value")); + setupGroupByTagValues(); + + row = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_STRING); + } + + @Test + public void constructor() { + assertNotNull(new RollupDataPointRpc(tsdb.getConfig())); + } + + // Socket RPC Tests ------------------------------------ + + // TODO - something odd going on with this timestamp falling in the wrong + // row.... hmm.. +// @Test +// public void execute() throws Exception { +// final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); +// final Channel chan = NettyMocks.fakeChannel(); +// assertNotNull(rollup.execute(tsdb, chan, new String[] { "rollup", +// "1h-sum", METRIC_STRING, "1365465600", "42", +// TAGK_STRING + "=" + TAGV_STRING }) +// .joinUninterruptibly()); +// validateCounters(1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0); +// verify(chan, never()).write(any()); +// verify(chan, never()).isConnected(); +// validateSEH(false); +// storage.dumpToSystemOut(); +// System.out.println(MockBase.bytesToString(row)); +// final byte[] qualifier = new byte[] {0, 0, 0}; +// final byte[] value = storage.getColumn( +// rollup_config.getRollupInterval("1h").getTemporalTable(), +// row, FAMILY, qualifier); +// final byte[] expected = {0x2A}; +// assertArrayEquals(expected, value); +// } + + @Test + public void execute() throws Exception { + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + assertNotNull(rollup.execute(tsdb, chan, new String[] { "rollup", + "1h-sum", METRIC_STRING, "1356998400", "42", + TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly()); + validateCounters(1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0); + verify(chan, never()).write(any()); + verify(chan, never()).isConnected(); + validateSEH(false); + + final byte[] qualifier = new byte[] {0, 0, 0}; + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("1h").getTemporalTable(), + row, FAMILY, qualifier); + final byte[] expected = {0x2A}; + assertArrayEquals(expected, value); + } + + @Test + public void executeRollupsDisabled() throws Exception { + Whitebox.setInternalState(tsdb, "rollup_config", (RollupConfig) null); + Whitebox.setInternalState(tsdb, "default_interval", (RollupInterval) null); + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + rollup.execute(tsdb, chan, new String[] { "rollup", "1h-sum", METRIC_STRING, + "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(false); + } + + @Test + public void executeAggOnly() throws Exception { + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + rollup.execute(tsdb, chan, new String[] { "rollup", "sum", + METRIC_STRING, "1356998400", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0); + verify(chan, never()).write(any()); + verify(chan, never()).isConnected(); + validateSEH(false); + + row = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_STRING, + agg_tag_key, "SUM"); + final byte[] qualifier = new byte[] {0, 0}; + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("1m").getGroupbyTable(), + row, FAMILY, qualifier); + final byte[] expected = {0x2A}; + assertArrayEquals(expected, value); + } + + @Test + public void executeRollupWithAgg() throws Exception { + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + rollup.execute(tsdb, chan, new String[] { "rollup", "1h-sum:sum", + METRIC_STRING, "1356998400", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0); + verify(chan, never()).write(any()); + verify(chan, never()).isConnected(); + validateSEH(false); + + row = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_STRING, + agg_tag_key, "SUM"); + final byte[] qualifier = new byte[] {0, 0, 0}; + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("1h").getGroupbyTable(), + row, FAMILY, qualifier); + final byte[] expected = {0x2A}; + assertArrayEquals(expected, value); + } + + @Test + public void executeBadValue() throws Exception { + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + rollup.execute(tsdb, chan, new String[] { "rollup", "10-msum", + METRIC_STRING, "1365465600", "notanum", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(false); + } + + @Test + public void executeBadValueNotWriteable() throws Exception { + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + when(chan.isWritable()).thenReturn(false); + rollup.execute(tsdb, chan, new String[] { "rollup", "1h-sum", + METRIC_STRING, "1365465600", "notanum", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0); + verify(chan, never()).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(false); + } + + @Test + public void executeMissingMetric() throws Exception { + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + rollup.execute(tsdb, chan, new String[] { "rollup", "1h-sum", "", + "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(false); + } + + @Test + public void executeUnknownMetric() throws Exception { + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + rollup.execute(tsdb, chan, new String[] { "rollup", "1h-sum", NSUN_METRIC, + "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(false); + } + + @SuppressWarnings("unchecked") + @Test (expected = RuntimeException.class) + public void executeRuntimeException() throws Exception { + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + PowerMockito.when(tsdb.addAggregatePoint(anyString(), anyLong(), anyLong(), + anyMap(), anyBoolean(), anyString(), anyString(), anyString())) + .thenThrow(new RuntimeException("Fail!")); + rollup.execute(tsdb, chan, new String[] { "rollup", "rollup", "1h-sum", + METRIC_STRING, "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + } + + @Test + public void executeHBaseError() throws Exception { + storage.throwException(MockBase.stringToBytes("0000015158CE00000001000001"), + mock(HBaseException.class)); + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + rollup.execute(tsdb, chan, new String[] { "rollup", "1h-sum", + METRIC_STRING, "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(true); + } + + @Test + public void executeHBaseErrorNotWriteable() throws Exception { + storage.throwException(MockBase.stringToBytes("0000015158CE00000001000001"), + mock(HBaseException.class)); + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + when(chan.isWritable()).thenReturn(false); + rollup.execute(tsdb, chan, new String[] { "rollup", "1h-sum", + METRIC_STRING, "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0); + verify(chan, never()).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(true); + } + + @Test + public void executeHBaseErrorHandler() throws Exception { + setStorageExceptionHandler(); + storage.throwException(MockBase.stringToBytes("0000015158CE00000001000001"), + mock(HBaseException.class)); + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + rollup.execute(tsdb, chan, new String[] { "rollup", "1h-sum", + METRIC_STRING, "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(true); + } + + @Test + public void executePleaseThrottle() throws Exception { + storage.throwException(MockBase.stringToBytes("0000015158CE00000001000001"), + mock(PleaseThrottleException.class)); + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + rollup.execute(tsdb, chan, new String[] { "rollup", "1h-sum", + METRIC_STRING, "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(true); + } + + @Test + public void executePleaseThrottleNotWriteable() throws Exception { + storage.throwException(MockBase.stringToBytes("0000015158CE00000001000001"), + mock(PleaseThrottleException.class)); + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + when(chan.isWritable()).thenReturn(false); + rollup.execute(tsdb, chan, new String[] { "rollup", "1h-sum", + METRIC_STRING, "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0); + verify(chan, never()).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(true); + } + + @Test + public void executePleaseThrottleHandler() throws Exception { + setStorageExceptionHandler(); + storage.throwException(MockBase.stringToBytes("0000015158CE00000001000001"), + mock(PleaseThrottleException.class)); + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + rollup.execute(tsdb, chan, new String[] { "rollup", "1h-sum", + METRIC_STRING, "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(true); + } + + @Test (expected = NullPointerException.class) + public void executeNullTSDB() throws Exception { + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + rollup.execute(null, chan, new String[] { "rollup", "1h-sum", + METRIC_STRING, "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + } + + @Test + public void executeNullChannelOK() throws Exception { + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + // we can pass in a null channel but since we only write when an error occurs + // then we won't fail. + rollup.execute(tsdb, null, new String[] { "rollup", "1h-sum", + METRIC_STRING, "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); + } + + @Test (expected = NullPointerException.class) + public void executeNullChannelError() throws Exception { + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + rollup.execute(tsdb, null, new String[] { "rollup", "1h-sum", + METRIC_STRING, "1365465600", "notanumber", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + } + + @Test (expected = NullPointerException.class) + public void executeNullArray() throws Exception { + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + rollup.execute(tsdb, chan, null).joinUninterruptibly(); + } + + @Test (expected = ArrayIndexOutOfBoundsException.class) + public void executeEmptyArray() throws Exception { + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + rollup.execute(tsdb, chan, new String[0]).joinUninterruptibly(); + } + + @Test + public void executeShortArray() throws Exception { + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + rollup.execute(tsdb, chan, new String[] { "rollup", "1h-sum", + METRIC_STRING, "1365465600", "42" }).joinUninterruptibly(); + validateCounters(1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(false); + } + + @Test + public void executeMissingInterval() throws Exception { + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + rollup.execute(tsdb, chan, new String[] { "rollup","", METRIC_STRING, + "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(false); + } + + @Test + public void executeUnknownInterval() throws Exception { + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + rollup.execute(tsdb, chan, new String[] { "rollup", "13m-sum", + METRIC_STRING, "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(false); + } + + // TODO - revisit +// @Test +// public void executePreAggOnly() throws Exception { +// final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); +// final Channel chan = NettyMocks.fakeChannel(); +// rollup.execute(tsdb, chan, new String[] { "rollup", "sum", METRIC_STRING, +// "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) +// .joinUninterruptibly(); +// validateCounters(1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); +// verify(chan, never()).write(any()); +// verify(chan, never()).isConnected(); +// validateSEH(false); +// } + + @Test + public void executeMissingAggregator() throws Exception { + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + rollup.execute(tsdb, chan, new String[] { "rollup", METRIC_STRING, + "1365465600", "42", TAGK_STRING + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(false); + } + + // TODO - test unknown aggs if we decide to implement that check + + @Test + public void executeMissingTags() throws Exception { + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + rollup.execute(tsdb, chan, new String[] { "rollup","1h-sum", METRIC_STRING, + "1365465600", "42", "" }) + .joinUninterruptibly(); + validateCounters(1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(false); + } + + @Test + public void executeNSUNTagk() throws Exception { + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + rollup.execute(tsdb, chan, new String[] { "rollup","1h-sum", METRIC_STRING, + "1365465600", "42", NSUN_TAGK + "=" + TAGV_STRING }) + .joinUninterruptibly(); + validateCounters(1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(false); + } + + @Test + public void executeNSUNTagV() throws Exception { + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + final Channel chan = NettyMocks.fakeChannel(); + rollup.execute(tsdb, chan, new String[] { "rollup", "1h-sum", METRIC_STRING, + "1365465600", "42", TAGK_STRING + "=" + NSUN_TAGV }) + .joinUninterruptibly(); + validateCounters(1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0); + verify(chan, times(1)).write(any()); + verify(chan, times(1)).isConnected(); + validateSEH(false); + } + +// HTTP RPC Tests -------------------------------------- + + // TODO - Something odd with this timestamp +// @Test +// public void httpAddSingleRollupPoint() throws Exception { +// HttpQuery query = NettyMocks.postQuery(tsdb, "/api/rollup", +// "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\":42, " +// + "\"interval\":\"1h\", \"aggregator\":\"sum\"," +// + "\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); +// final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); +// rollup.execute(tsdb, query); +// assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); +// validateCounters(0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0); +// validateSEH(false); +// +// final byte[] qualifier = new byte[] {0, 0, 0}; +// final byte[] value = storage.getColumn( +// rollup_config.getRollupInterval("1h").getTemporalTable(), +// row, FAMILY, qualifier); +// final byte[] expected = {0x2A}; +// assertArrayEquals(expected, value); +// } + + @Test + public void httpAddSingleRollupPoint() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/rollup", + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1356998400,\"value\":42, " + + "\"interval\":\"1h\", \"aggregator\":\"sum\"," + + "\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + rollup.execute(tsdb, query); + assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateCounters(0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); + + final byte[] qualifier = new byte[] {0, 0, 0}; + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("1h").getTemporalTable(), + row, FAMILY, qualifier); + final byte[] expected = {0x2A}; + assertArrayEquals(expected, value); + } + + @Test + public void httpAddSingleGroupByPoint() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/rollup", + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1356998400,\"value\":42, " + + "\"groupByAggregator\":\"sum\"," + + "\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + rollup.execute(tsdb, query); + + assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateCounters(0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); + row = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_STRING, + agg_tag_key, "SUM"); + + final byte[] qualifier = new byte[] { 0, 0 }; + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("1m").getGroupbyTable(), + row, FAMILY, qualifier); + final byte[] expected = {0x2A}; + assertArrayEquals(expected, value); + } + + @Test + public void httpAddSingleRollupAndGroupByPoint() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/rollup", + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1356998400,\"value\":42, " + + "\"interval\":\"1h\", \"aggregator\":\"sum\", " + + "\"groupByAggregator\":\"sum\"," + + "\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + rollup.execute(tsdb, query); + + assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateCounters(0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); + row = getRowKey(METRIC_STRING, 1356998400, TAGK_STRING, TAGV_STRING, + agg_tag_key, "SUM"); + + final byte[] qualifier = new byte[] { 0, 0, 0 }; + final byte[] value = storage.getColumn( + rollup_config.getRollupInterval("1h").getGroupbyTable(), + row, FAMILY, qualifier); + final byte[] expected = {0x2A}; + assertArrayEquals(expected, value); + } + + @Test + public void httpAddTwoRollupPoints() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/rollup", + "[{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1356998400,\"value\":42, " + + "\"interval\":\"1h\", \"aggregator\":\"sum\"," + + "\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}, " + + "{\"metric\":\"" + METRIC_B_STRING + "\",\"timestamp\":1356998400,\"value\":24, " + + "\"interval\":\"1h\", \"aggregator\":\"sum\"," + + "\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}]"); + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + rollup.execute(tsdb, query); + assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateCounters(0, 1, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); + + final byte[] qualifier = new byte[] {0, 0, 0}; + byte[] value = storage.getColumn( + rollup_config.getRollupInterval("1h").getTemporalTable(), + row, FAMILY, qualifier); + byte[] expected = {0x2A}; + assertArrayEquals(expected, value); + + row = getRowKey(METRIC_B_STRING, 1356998400, TAGK_STRING, TAGV_STRING); + value = storage.getColumn( + rollup_config.getRollupInterval("1h").getTemporalTable(), + row, FAMILY, qualifier); + expected = new byte[] {0x18}; + assertArrayEquals(expected, value); + } + + @Test + public void httpAddTwoRollupPointsOneGoodOneBad() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/rollup", + "[{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1356998400,\"value\":42, " + + "\"interval\":\"1h\", \"aggregator\":\"sum\"," + + "\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}, " + + "{\"metric\":\"" + NSUN_METRIC + "\",\"timestamp\":1356998400,\"value\":24, " + + "\"interval\":\"1h\", \"aggregator\":\"sum\"," + + "\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}]"); + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + rollup.execute(tsdb, query); + assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); + validateCounters(0, 1, 0, 2, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0); + validateSEH(false); + + final byte[] qualifier = new byte[] {0, 0, 0}; + byte[] value = storage.getColumn( + rollup_config.getRollupInterval("1h").getTemporalTable(), + row, FAMILY, qualifier); + byte[] expected = {0x2A}; + assertArrayEquals(expected, value); + + row = getRowKey(METRIC_B_STRING, 1356998400, TAGK_STRING, TAGV_STRING); + value = storage.getColumn( + rollup_config.getRollupInterval("1h").getTemporalTable(), + row, FAMILY, qualifier); + assertNull(value); + } + + @Test + public void httpMissingInterval() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/rollup?details", + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\":42, " + + "\"aggregator\":\"sum\",\"tags\":{\"" + TAGK_STRING + "\":\"" + + TAGV_STRING + "\"}}"); + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + rollup.execute(tsdb, query); + assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertThat(response, CoreMatchers.containsString("\"error\":\"Missing interval\"")); + assertThat(response, CoreMatchers.containsString("\"failed\":1")); + assertThat(response, CoreMatchers.containsString("\"success\":0")); + assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); + validateCounters(0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + validateSEH(false); + } + + @Test + public void httpEmptyInterval() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/rollup?details", + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\":42, " + + "\"interval\":\"\", \"aggregator\":\"sum\"," + + "\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + rollup.execute(tsdb, query); + assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertThat(response, CoreMatchers.containsString("\"error\":\"Missing interval\"")); + assertThat(response, CoreMatchers.containsString("\"failed\":1")); + assertThat(response, CoreMatchers.containsString("\"success\":0")); + validateCounters(0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + validateSEH(false); + } + + @Test + public void httpMissingAggregator() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/rollup?details", + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\":42, " + + "\"interval\":\"1h\",\"tags\":{\"" + TAGK_STRING + "\":\"" + + TAGV_STRING + "\"}}"); + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + rollup.execute(tsdb, query); + assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertThat(response, CoreMatchers.containsString("\"error\":\"Missing aggregator\"")); + assertThat(response, CoreMatchers.containsString("\"failed\":1")); + assertThat(response, CoreMatchers.containsString("\"success\":0")); + validateCounters(0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + validateSEH(false); + } + + @Test + public void httpInvalidAggregator() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/rollup?details", + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1356998400,\"value\":42, " + + "\"interval\":\"1h\", \"aggregator\":\"nosuchagg\"," + + "\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + rollup.execute(tsdb, query); + assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); + validateCounters(0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + validateSEH(false); + + final byte[] qualifier = new byte[] {0, 0, 0}; + assertNull(storage.getColumn( + rollup_config.getRollupInterval("1h").getTemporalTable(), + row, FAMILY, qualifier)); + } + + @Test + public void httpEmptyAggregator() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/rollup?details", + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\":42, " + + "\"interval\":\"1h\", \"aggregator\":\"\"," + + "\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + rollup.execute(tsdb, query); + assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); + final String response = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertThat(response, CoreMatchers.containsString("\"error\":\"Missing aggregator\"")); + assertThat(response, CoreMatchers.containsString("\"failed\":1")); + assertThat(response, CoreMatchers.containsString("\"success\":0")); + validateCounters(0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0); + validateSEH(false); + } + + @Test + public void httpNSUNMetric() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/rollup", + "{\"metric\":\"" + NSUN_METRIC + "\",\"timestamp\":1365465600,\"value\":42, " + + "\"interval\":\"1h\", \"aggregator\":\"sum\"," + + "\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + rollup.execute(tsdb, query); + assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); + validateCounters(0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0); + validateSEH(false); + } + + @Test + public void httpNSUNTagk() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/rollup", + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\":42, " + + "\"interval\":\"1h\", \"aggregator\":\"sum\"," + + "\"tags\":{\"" + NSUN_TAGK + "\":\"" + TAGV_STRING + "\"}}"); + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + rollup.execute(tsdb, query); + assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); + validateCounters(0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0); + validateSEH(false); + } + + @Test + public void httpNSUNTagv() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/rollup", + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\":42, " + + "\"interval\":\"1h\", \"aggregator\":\"sum\"," + + "\"tags\":{\"" + TAGK_STRING + "\":\"" + NSUN_TAGV + "\"}}"); + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + rollup.execute(tsdb, query); + assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); + validateCounters(0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0); + validateSEH(false); + } + + @Test + public void httpHBaseError() throws Exception { + storage.throwException(MockBase.stringToBytes("0000015158CE00000001000001"), + mock(HBaseException.class)); + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/rollup", + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\":42, " + + "\"interval\":\"1h\", \"aggregator\":\"sum\"," + + "\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + rollup.execute(tsdb, query); + assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateCounters(0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0); + validateSEH(true); + } + + @Test + public void httpPleaseThrottleError() throws Exception { + storage.throwException(MockBase.stringToBytes("0000015158CE00000001000001"), + mock(PleaseThrottleException.class)); + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/rollup", + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\":42, " + + "\"interval\":\"1h\", \"aggregator\":\"sum\"," + + "\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + rollup.execute(tsdb, query); + assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); + validateCounters(0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0); + validateSEH(true); + } + + @Test + public void httpUnknownInterval() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/rollup", + "{\"metric\":\"" + METRIC_STRING + "\",\"timestamp\":1365465600,\"value\":42, " + + "\"interval\":\"13m\", \"aggregator\":\"sum\"," + + "\"tags\":{\"" + TAGK_STRING + "\":\"" + TAGV_STRING + "\"}}"); + final RollupDataPointRpc rollup = new RollupDataPointRpc(tsdb.getConfig()); + rollup.execute(tsdb, query); + assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); + validateCounters(0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + validateSEH(false); + } +} diff --git a/test/tsd/TestRpcHandler.java b/test/tsd/TestRpcHandler.java index 1528cfc9ea..9407937555 100644 --- a/test/tsd/TestRpcHandler.java +++ b/test/tsd/TestRpcHandler.java @@ -15,46 +15,51 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -import static org.powermock.api.mockito.PowerMockito.mock; +import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.when; +import static org.powermock.api.mockito.PowerMockito.mock; -import net.opentsdb.core.TSDB; -import net.opentsdb.utils.Config; +import java.lang.reflect.Method; + +import com.google.common.net.HttpHeaders; import org.hbase.async.HBaseClient; import org.jboss.netty.channel.Channel; import org.jboss.netty.channel.ChannelFuture; import org.jboss.netty.channel.ChannelHandlerContext; import org.jboss.netty.channel.MessageEvent; +import org.jboss.netty.channel.SucceededChannelFuture; import org.jboss.netty.handler.codec.http.DefaultHttpRequest; import org.jboss.netty.handler.codec.http.DefaultHttpResponse; import org.jboss.netty.handler.codec.http.HttpMethod; import org.jboss.netty.handler.codec.http.HttpRequest; import org.jboss.netty.handler.codec.http.HttpResponseStatus; import org.jboss.netty.handler.codec.http.HttpVersion; +import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; -import com.google.common.net.HttpHeaders; +import net.opentsdb.core.TSDB; +import net.opentsdb.utils.Config; @PowerMockIgnore({"javax.management.*", "javax.xml.*", "ch.qos.*", "org.slf4j.*", "com.sum.*", "org.xml.*"}) @RunWith(PowerMockRunner.class) -@PrepareForTest({ TSDB.class, Config.class, HBaseClient.class, RpcHandler.class, +@PrepareForTest({ TSDB.class, Config.class, HBaseClient.class, RpcHandler.class, HttpQuery.class, MessageEvent.class, DefaultHttpResponse.class, ChannelHandlerContext.class }) public final class TestRpcHandler { private TSDB tsdb = null; + private RpcManager rpc_manager; private ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); private HBaseClient client = mock(HBaseClient.class); private MessageEvent message = mock(MessageEvent.class); @@ -62,21 +67,25 @@ public final class TestRpcHandler { @Before public void before() throws Exception { final Config config = new Config(false); - PowerMockito.whenNew(HBaseClient.class) - .withArguments(anyString(), anyString()).thenReturn(client); - tsdb = new TSDB(config); + tsdb = new TSDB(client, config); + rpc_manager = RpcManager.instance(tsdb); + } + + @After + public void after() { + rpc_manager.shutdown(); } @Test public void ctorDefaults() { - final RpcHandler rpc = new RpcHandler(tsdb); + final RpcHandler rpc = new RpcHandler(tsdb, rpc_manager); assertNotNull(rpc); } @Test public void ctorCORSPublic() { tsdb.getConfig().overrideConfig("tsd.http.request.cors_domains", "*"); - final RpcHandler rpc = new RpcHandler(tsdb); + final RpcHandler rpc = new RpcHandler(tsdb, rpc_manager); assertNotNull(rpc); } @@ -84,7 +93,7 @@ public void ctorCORSPublic() { public void ctorCORSSeparated() { tsdb.getConfig().overrideConfig("tsd.http.request.cors_domains", "aurther.com,dent.net,beeblebrox.org"); - final RpcHandler rpc = new RpcHandler(tsdb); + final RpcHandler rpc = new RpcHandler(tsdb, rpc_manager); assertNotNull(rpc); } @@ -92,7 +101,7 @@ public void ctorCORSSeparated() { public void ctorCORSPublicAndDomains() { tsdb.getConfig().overrideConfig("tsd.http.request.cors_domains", "*,aurther.com,dent.net,beeblebrox.org"); - new RpcHandler(tsdb); + new RpcHandler(tsdb, rpc_manager); } @Test @@ -114,7 +123,7 @@ public ChannelFuture answer(final InvocationOnMock args) } ); - final RpcHandler rpc = new RpcHandler(tsdb); + final RpcHandler rpc = new RpcHandler(tsdb, rpc_manager); rpc.messageReceived(ctx, message); } @@ -139,7 +148,7 @@ public ChannelFuture answer(final InvocationOnMock args) ); tsdb.getConfig().overrideConfig("tsd.http.request.cors_domains", "*"); - final RpcHandler rpc = new RpcHandler(tsdb); + final RpcHandler rpc = new RpcHandler(tsdb, rpc_manager); rpc.messageReceived(ctx, message); } @@ -165,7 +174,7 @@ public ChannelFuture answer(final InvocationOnMock args) tsdb.getConfig().overrideConfig("tsd.http.request.cors_domains", "aurther.com,dent.net,42.com,beeblebrox.org"); - final RpcHandler rpc = new RpcHandler(tsdb); + final RpcHandler rpc = new RpcHandler(tsdb, rpc_manager); rpc.messageReceived(ctx, message); } @@ -190,7 +199,7 @@ public ChannelFuture answer(final InvocationOnMock args) tsdb.getConfig().overrideConfig("tsd.http.request.cors_domains", "aurther.com,dent.net,beeblebrox.org"); - final RpcHandler rpc = new RpcHandler(tsdb); + final RpcHandler rpc = new RpcHandler(tsdb, rpc_manager); rpc.messageReceived(ctx, message); } @@ -212,7 +221,7 @@ public ChannelFuture answer(final InvocationOnMock args) } ); - final RpcHandler rpc = new RpcHandler(tsdb); + final RpcHandler rpc = new RpcHandler(tsdb, rpc_manager); rpc.messageReceived(ctx, message); } @@ -235,7 +244,7 @@ public ChannelFuture answer(final InvocationOnMock args) } ); - final RpcHandler rpc = new RpcHandler(tsdb); + final RpcHandler rpc = new RpcHandler(tsdb, rpc_manager); rpc.messageReceived(ctx, message); } @@ -260,7 +269,7 @@ public ChannelFuture answer(final InvocationOnMock args) ); tsdb.getConfig().overrideConfig("tsd.http.request.cors_domains", "*"); - final RpcHandler rpc = new RpcHandler(tsdb); + final RpcHandler rpc = new RpcHandler(tsdb, rpc_manager); rpc.messageReceived(ctx, message); } @@ -286,7 +295,7 @@ public ChannelFuture answer(final InvocationOnMock args) tsdb.getConfig().overrideConfig("tsd.http.request.cors_domains", "aurther.com,dent.net,42.com,beeblebrox.org"); - final RpcHandler rpc = new RpcHandler(tsdb); + final RpcHandler rpc = new RpcHandler(tsdb, rpc_manager); rpc.messageReceived(ctx, message); } @@ -311,14 +320,79 @@ public ChannelFuture answer(final InvocationOnMock args) tsdb.getConfig().overrideConfig("tsd.http.request.cors_domains", "aurther.com,dent.net,beeblebrox.org"); - final RpcHandler rpc = new RpcHandler(tsdb); + final RpcHandler rpc = new RpcHandler(tsdb, rpc_manager); rpc.messageReceived(ctx, message); } - private void handleHttpRpc(final HttpRequest req, final Answer<?> answer) { + @Test + public void createQueryInstanceForBuiltin() throws Exception { + final RpcHandler rpc = new RpcHandler(tsdb, rpc_manager); + final Channel mockChan = NettyMocks.fakeChannel(); + final Method meth = Whitebox.getMethod(RpcHandler.class, "createQueryInstance", + TSDB.class, HttpRequest.class, Channel.class); + AbstractHttpQuery query = (AbstractHttpQuery) meth.invoke( + rpc, tsdb, + new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.OPTIONS, "/api/v1/version"), + mockChan); + assertTrue(query instanceof HttpQuery); + + query = (AbstractHttpQuery) meth.invoke( + rpc, tsdb, + new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.OPTIONS, "/api/version"), + mockChan); + assertTrue(query instanceof HttpQuery); + + query = (AbstractHttpQuery) meth.invoke( + rpc, tsdb, + new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.OPTIONS, "/q"), + mockChan); + assertTrue(query instanceof HttpQuery); + + query = (AbstractHttpQuery) meth.invoke( + rpc, tsdb, + new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.OPTIONS, "/"), + mockChan); + assertTrue(query instanceof HttpQuery); + } + + @Test(expected=BadRequestException.class) + public void createQueryInstanceEmptyRequestInvalid() throws Exception { + final RpcHandler rpc = new RpcHandler(tsdb, rpc_manager); + final Channel mockChan = NettyMocks.fakeChannel(); + final Method meth = Whitebox.getMethod(RpcHandler.class, "createQueryInstance", + TSDB.class, HttpRequest.class, Channel.class); + meth.invoke( + rpc, tsdb, + new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.OPTIONS, ""), + mockChan); + } + + @Test + public void emptyPathIsBadRequest() throws Exception { + final HttpRequest req = new DefaultHttpRequest(HttpVersion.HTTP_1_1, + HttpMethod.GET, ""); + + final Channel mockChan = handleHttpRpc(req, + new Answer<ChannelFuture>() { + public ChannelFuture answer(final InvocationOnMock args) + throws Throwable { + DefaultHttpResponse response = + (DefaultHttpResponse)args.getArguments()[0]; + assertEquals(HttpResponseStatus.BAD_REQUEST, response.getStatus()); + return new SucceededChannelFuture((Channel) args.getMock()); + } + } + ); + + final RpcHandler rpc = new RpcHandler(tsdb, rpc_manager); + Whitebox.invokeMethod(rpc, "handleHttpQuery", tsdb, mockChan, req); + } + + private Channel handleHttpRpc(final HttpRequest req, final Answer<?> answer) { final Channel channel = NettyMocks.fakeChannel(); when(message.getMessage()).thenReturn(req); when(message.getChannel()).thenReturn(channel); when(channel.write((DefaultHttpResponse)any())).thenAnswer(answer); + return channel; } } diff --git a/test/tsd/TestRpcManager.java b/test/tsd/TestRpcManager.java new file mode 100644 index 0000000000..e97cadfc06 --- /dev/null +++ b/test/tsd/TestRpcManager.java @@ -0,0 +1,224 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2014 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.tsd; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; +import static org.powermock.api.mockito.PowerMockito.mock; + +import org.hbase.async.HBaseClient; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import net.opentsdb.core.TSDB; +import net.opentsdb.core.TSDB.OperationMode; +import net.opentsdb.utils.Config; +import net.opentsdb.utils.PluginLoader; + +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) +@RunWith(PowerMockRunner.class) +@PrepareForTest({ TSDB.class, Config.class, HBaseClient.class, RpcManager.class }) +public class TestRpcManager { + private TSDB mock_tsdb_no_plugins; + + // Set in individual test methods; shutdown by after() if set. + private RpcManager mgr_under_test; + + @Before + public void before() { + Config config = mock(Config.class); + when(config.getString("tsd.core.enable_api")) + .thenReturn("true"); + when(config.getString("tsd.core.enable_ui")) + .thenReturn("true"); + when(config.getString("tsd.no_diediedie")) + .thenReturn("false"); + TSDB tsdb = mock(TSDB.class); + when(tsdb.getMode()).thenReturn(OperationMode.READWRITE); + when(tsdb.getConfig()).thenReturn(config); + mock_tsdb_no_plugins = tsdb; + } + + @After + public void after() throws Exception { + if (mgr_under_test != null) { + mgr_under_test.shutdown().join(); + } + } + + @Test + public void loadHttpRpcPlugins() throws Exception { + Config config = mock(Config.class); + when(config.hasProperty("tsd.http.rpc.plugins")) + .thenReturn(true); + when(config.getString("tsd.http.rpc.plugins")) + .thenReturn("net.opentsdb.tsd.DummyHttpRpcPlugin"); + when(config.getString("tsd.core.enable_api")) + .thenReturn("true"); + when(config.getString("tsd.core.enable_ui")) + .thenReturn("true"); + when(config.getString("tsd.no_diediedie")) + .thenReturn("false"); + + TSDB tsdb = mock(TSDB.class); + when(tsdb.getMode()).thenReturn(OperationMode.READWRITE); + when(tsdb.getConfig()).thenReturn(config); + + PluginLoader.loadJAR("plugin_test.jar"); + mgr_under_test = RpcManager.instance(tsdb); + + HttpRpcPlugin plugin = mgr_under_test.lookupHttpRpcPlugin("dummy/test"); + assertNotNull(plugin); + } + + @Test + public void loadRpcPlugin() throws Exception { + Config config = mock(Config.class); + when(config.hasProperty("tsd.rpc.plugins")) + .thenReturn(true); + when(config.getString("tsd.rpc.plugins")) + .thenReturn("net.opentsdb.tsd.DummyRpcPlugin"); + when(config.hasProperty("tsd.rpcplugin.DummyRPCPlugin.hosts")) + .thenReturn(true); + when(config.getString("tsd.rpcplugin.DummyRPCPlugin.hosts")) + .thenReturn("blah"); + when(config.getInt("tsd.rpcplugin.DummyRPCPlugin.port")) + .thenReturn(1000); + when(config.getString("tsd.core.enable_api")) + .thenReturn("true"); + when(config.getString("tsd.core.enable_ui")) + .thenReturn("true"); + when(config.getString("tsd.no_diediedie")) + .thenReturn("false"); + + TSDB tsdb = mock(TSDB.class); + when(tsdb.getMode()).thenReturn(OperationMode.READWRITE); + when(tsdb.getConfig()).thenReturn(config); + + PluginLoader.loadJAR("plugin_test.jar"); + mgr_under_test= RpcManager.instance(tsdb); + + assertFalse(mgr_under_test.getRpcPlugins().isEmpty()); + } + + @Test + public void isHttpRpcPluginPathValid() { + mgr_under_test = RpcManager.instance(mock_tsdb_no_plugins); + assertTrue(mgr_under_test.isHttpRpcPluginPath("/plugin/my/http/plugin")); + assertTrue(mgr_under_test.isHttpRpcPluginPath("plugin/my/http/plugin")); + assertTrue(mgr_under_test.isHttpRpcPluginPath("/plugin/my?hey=hi&howdy=ho")); + assertTrue(mgr_under_test.isHttpRpcPluginPath("plugin/my?hey=hi&howdy=ho")); + assertTrue(mgr_under_test.isHttpRpcPluginPath("plugin/my/?hey=hi&howdy=ho")); + } + + @Test + public void isHttpRpcPluginPathInvalid() { + mgr_under_test = RpcManager.instance(mock_tsdb_no_plugins); + assertFalse(mgr_under_test.isHttpRpcPluginPath("/plugin/")); + assertFalse(mgr_under_test.isHttpRpcPluginPath("plugin/")); + assertFalse(mgr_under_test.isHttpRpcPluginPath("plugin")); + assertFalse(mgr_under_test.isHttpRpcPluginPath("/plugin")); + assertFalse(mgr_under_test.isHttpRpcPluginPath("/plugin?howdy=ho")); + assertFalse(mgr_under_test.isHttpRpcPluginPath("/plugin/?howdy=ho")); + assertFalse(mgr_under_test.isHttpRpcPluginPath("api/query")); + } + + @Test + public void validateHttpRpcPluginPathValid() { + mgr_under_test = RpcManager.instance(mock_tsdb_no_plugins); + mgr_under_test.validateHttpRpcPluginPath("/my/test/path"); + mgr_under_test.validateHttpRpcPluginPath("my/test/path"); + mgr_under_test.validateHttpRpcPluginPath("my/test/path"); + mgr_under_test.validateHttpRpcPluginPath("api/query"); + } + + @Test + public void validateHttpRpcPluginPathInvalid() { + mgr_under_test = RpcManager.instance(mock_tsdb_no_plugins); + try { + mgr_under_test.validateHttpRpcPluginPath("/plugin/my/test"); + assertTrue(false); + } catch (IllegalArgumentException e) { } + try { + mgr_under_test.validateHttpRpcPluginPath("plugin/my/test"); + assertTrue(false); + } catch (IllegalArgumentException e) { } + try { + mgr_under_test.validateHttpRpcPluginPath("plugin/"); + assertTrue(false); + } catch (IllegalArgumentException e) { } + try { + mgr_under_test.validateHttpRpcPluginPath("/plugin/"); + assertTrue(false); + } catch (IllegalArgumentException e) { } + try { + mgr_under_test.validateHttpRpcPluginPath("/plugin"); + assertTrue(false); + } catch (IllegalArgumentException e) { } + try { + mgr_under_test.validateHttpRpcPluginPath("plugin"); + assertTrue(false); + } catch (IllegalArgumentException e) { } + } + + @Test + public void canonicalizePluginPathsValid() throws Exception { + mgr_under_test = RpcManager.instance(mock_tsdb_no_plugins); + assertEquals("my/test/path", + mgr_under_test.canonicalizePluginPath("/my/test/path")); + assertEquals("my/test/path", + mgr_under_test.canonicalizePluginPath("/my/test/path/")); + assertEquals("my/test/path", + mgr_under_test.canonicalizePluginPath("my/test/path/")); + assertEquals("my/test/path", + mgr_under_test.canonicalizePluginPath("my/test/path")); + + assertEquals("my", + mgr_under_test.canonicalizePluginPath("/my/")); + assertEquals("my", + mgr_under_test.canonicalizePluginPath("my/")); + assertEquals("my", + mgr_under_test.canonicalizePluginPath("my")); + } + + @Test(expected=IllegalArgumentException.class) + public void canonicalizePluginPathIsRoot() { + mgr_under_test = RpcManager.instance(mock_tsdb_no_plugins); + assertEquals(RpcManager.PLUGIN_BASE_WEBPATH + "/", + mgr_under_test.canonicalizePluginPath("")); + } + + @Test + public void validHttpPathEndToEnd() { + mgr_under_test = RpcManager.instance(mock_tsdb_no_plugins); + mgr_under_test.validateHttpRpcPluginPath("myplugin"); + assertEquals("myplugin", mgr_under_test.canonicalizePluginPath("myplugin")); + mgr_under_test.validateHttpRpcPluginPath("/myplugin"); + assertEquals("myplugin", mgr_under_test.canonicalizePluginPath("/myplugin")); + + mgr_under_test.validateHttpRpcPluginPath("myplugin/subcommand"); + assertEquals("myplugin/subcommand", mgr_under_test.canonicalizePluginPath("myplugin/subcommand")); + mgr_under_test.validateHttpRpcPluginPath("/myplugin/subcommand"); + assertEquals("myplugin/subcommand", mgr_under_test.canonicalizePluginPath("/myplugin/subcommand")); + } +} diff --git a/test/tsd/TestSearchRpc.java b/test/tsd/TestSearchRpc.java index 5232fdf512..3ba1ea7d74 100644 --- a/test/tsd/TestSearchRpc.java +++ b/test/tsd/TestSearchRpc.java @@ -13,9 +13,6 @@ package net.opentsdb.tsd; import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyChar; -import static org.mockito.Matchers.anyList; -import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.when; import static org.powermock.api.mockito.PowerMockito.mock; import static org.junit.Assert.assertEquals; @@ -24,25 +21,26 @@ import java.lang.reflect.Field; import java.nio.charset.Charset; -import java.util.AbstractMap; import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import java.util.Map; +import net.opentsdb.core.BaseTsdbTest; import net.opentsdb.core.RowKey; import net.opentsdb.core.TSDB; import net.opentsdb.core.Tags; import net.opentsdb.meta.Annotation; import net.opentsdb.meta.TSMeta; import net.opentsdb.meta.UIDMeta; +import net.opentsdb.search.SearchPlugin; import net.opentsdb.search.SearchQuery; +import net.opentsdb.search.TestTimeSeriesLookup; import net.opentsdb.search.TimeSeriesLookup; +import net.opentsdb.storage.MockBase; +import net.opentsdb.uid.NoSuchUniqueId; import net.opentsdb.uid.UniqueId; import net.opentsdb.uid.UniqueId.UniqueIdType; import net.opentsdb.utils.Config; -import net.opentsdb.utils.JSON; -import net.opentsdb.utils.Pair; import org.jboss.netty.handler.codec.http.DefaultHttpRequest; import org.jboss.netty.handler.codec.http.HttpMethod; @@ -54,31 +52,26 @@ import org.junit.runner.RunWith; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; import com.stumbleupon.async.Deferred; +import com.sun.java_cup.internal.runtime.Scanner; @RunWith(PowerMockRunner.class) -@PrepareForTest({TSDB.class, Config.class, HttpQuery.class, UniqueId.class, - RowKey.class, Tags.class, TimeSeriesLookup.class, SearchRpc.class}) -public final class TestSearchRpc { - private TSDB tsdb = null; +@PrepareForTest({ TSDB.class, Config.class, HttpQuery.class, UniqueId.class, + RowKey.class, Tags.class, TimeSeriesLookup.class, SearchRpc.class, + SearchPlugin.class, Scanner.class }) +public final class TestSearchRpc extends BaseTsdbTest { + private SearchPlugin mock_plugin; private SearchRpc rpc = new SearchRpc(); private SearchQuery search_query = null; - private TimeSeriesLookup mock_lookup = null; private static final Charset UTF = Charset.forName("UTF-8"); - private static List<byte[]> test_tsuids = new ArrayList<byte[]>(3); - static { - test_tsuids.add(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }); - test_tsuids.add(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 2 }); - test_tsuids.add(new byte[] { 0, 0, 2, 0, 0, 1, 0, 0, 1 }); - } @Before - public void before() throws Exception { - tsdb = NettyMocks.getMockedHTTPTSDB(); + public void beforeLocal() throws Exception { + HttpQuery.initializeSerializerMaps(tsdb); } @Test @@ -106,7 +99,8 @@ public void searchTSMeta_Summary() throws Exception { rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); final String result = query.response().getContent().toString(UTF); - assertTrue(result.contains("\"results\":[{\"tags\"")); + assertTrue(result.contains("\"host\":\"web01\"")); + assertTrue(result.contains("\"metric\":\"sys.cpu.0\"")); assertEquals(1, search_query.getResults().size()); } @@ -221,9 +215,7 @@ public void searchMissingQuery() throws Exception { @Test (expected = BadRequestException.class) public void searchPluginNotEnabled() throws Exception { - when(tsdb.executeSearch((SearchQuery)any())) - .thenThrow(new IllegalStateException( - "Searching has not been enabled on this TSD")); + Whitebox.setInternalState(tsdb, "search", (SearchPlugin)null); final HttpQuery query = NettyMocks.getQuery(tsdb, "/api/search/tsmeta?query=*"); rpc.execute(tsdb, query); @@ -244,48 +236,122 @@ public void searchInvalidStartIndex() throws Exception { } @Test - public void searchLookup() throws Exception { - setupAnswerLookupQuery(); + public void searchLookupTagkOnlyMeta() throws Exception { + setupLookup(true); final HttpQuery query = NettyMocks.getQuery(tsdb, "/api/search/lookup?m={host=}"); rpc.execute(tsdb, query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); final String result = query.response().getContent().toString(UTF); assertTrue(result.contains("\"host\":\"web01\"")); - assertTrue(result.contains("\"totalResults\":3")); + assertTrue(result.contains("\"totalResults\":5")); + assertTrue(result.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(result.contains("\"tsuid\":\"000001000001000002\"")); + assertTrue(result.contains("\"tsuid\":\"000002000001000001\"")); + assertTrue(result.contains("\"tsuid\":\"000004000001000001000003000005\"")); + assertTrue(result.contains("\"tsuid\":\"000004000001000002000003000005\"")); } @Test - public void searchLookupPOST() throws Exception { - setupAnswerLookupQuery(); - SearchQuery q = new SearchQuery(); - q.setTags(new ArrayList<Pair<String, String>>(2)); - q.getTags().add(new Pair<String, String>("host", "web01")); - q.getTags().add(new Pair<String, String>("dc", "phx")); - + public void searchLookupPOSTTagkOnlyMeta() throws Exception { + setupLookup(true); final HttpQuery query = NettyMocks.postQuery(tsdb, - "/api/search/lookup", "{\"tags\":[{\"key\":\"host\",\"value\":\"web01\"}]}"); + "/api/search/lookup", "{\"tags\":[{\"key\":\"host\",\"value\":null}]}"); + query.setSerializer(); + rpc.execute(tsdb, query); + + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + final String result = query.response().getContent().toString(UTF); + assertTrue(result.contains("\"host\":\"web01\"")); + assertTrue(result.contains("\"totalResults\":5")); + assertTrue(result.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(result.contains("\"tsuid\":\"000001000001000002\"")); + assertTrue(result.contains("\"tsuid\":\"000002000001000001\"")); + assertTrue(result.contains("\"tsuid\":\"000004000001000001000003000005\"")); + assertTrue(result.contains("\"tsuid\":\"000004000001000002000003000005\"")); + } + + @Test + public void searchLookupPOSTTagkOnlyData() throws Exception { + setupLookup(false); + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/search/lookup", + "{\"tags\":[{\"key\":\"host\",\"value\":null}],\"useMeta\":false}"); rpc.execute(tsdb, query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); final String result = query.response().getContent().toString(UTF); assertTrue(result.contains("\"host\":\"web01\"")); - assertTrue(result.contains("\"totalResults\":3")); + assertTrue(result.contains("\"totalResults\":5")); + assertTrue(result.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(result.contains("\"tsuid\":\"000001000001000002\"")); + assertTrue(result.contains("\"tsuid\":\"000002000001000001\"")); + assertTrue(result.contains("\"tsuid\":\"000004000001000001000003000005\"")); + assertTrue(result.contains("\"tsuid\":\"000004000001000002000003000005\"")); + } + + @Test + public void searchLookupNoMetaTable() throws Exception { + setupLookup(false); + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/search/lookup", + "{\"tags\":[{\"key\":\"host\",\"value\":null}]}"); + rpc.execute(tsdb, query); + + assertEquals(HttpResponseStatus.INTERNAL_SERVER_ERROR, + query.response().getStatus()); + final String result = query.response().getContent().toString(UTF); + assertTrue(result.contains("\"code\":500")); + assertTrue(result.contains("\"message\":\"Unexpected exception\"")); } @Test (expected = BadRequestException.class) public void searchLookupMissingQuery() throws Exception { - setupAnswerLookupQuery(); - final HttpQuery query = NettyMocks.getQuery(tsdb, - "/api/search/lookup"); + setupLookup(true); + final HttpQuery query = NettyMocks.getQuery(tsdb, "/api/search/lookup"); rpc.execute(tsdb, query); } @Test (expected = BadRequestException.class) public void searchLookupBadQuery() throws Exception { - setupAnswerLookupQuery(); + setupLookup(true); + final HttpQuery query = NettyMocks.getQuery(tsdb, "/api/search/lookup?m={"); + rpc.execute(tsdb, query); + } + + @Test + public void searchLookupNSUN() throws Exception { + setupLookup(true); final HttpQuery query = NettyMocks.getQuery(tsdb, - "/api/search/lookup?m={"); + "/api/search/lookup?m=" + NSUN_METRIC); rpc.execute(tsdb, query); + + assertEquals(HttpResponseStatus.NOT_FOUND, query.response().getStatus()); + final String result = query.response().getContent().toString(UTF); + assertTrue(result.contains("\"code\":404")); + assertTrue(result.contains("\"details\":\"No such name")); + } + + @Test + public void searchLookupNSUI() throws Exception { + setupLookup(true); + when(metrics.getNameAsync(new byte[] { 0, 0, 4 })) + .thenAnswer(new Answer<Deferred<String>>() { + @Override + public Deferred<String> answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromError( + new NoSuchUniqueId("metrics", new byte[] { 0, 0, 4 })); + } + }); + final HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/search/lookup", "{\"tags\":[{\"key\":\"host\",\"value\":null}]}"); + query.setSerializer(); + rpc.execute(tsdb, query); + + assertEquals(HttpResponseStatus.NOT_FOUND, query.response().getStatus()); + final String result = query.response().getContent().toString(UTF); + assertTrue(result.contains("\"code\":404")); + assertTrue(result.contains("\"details\":\"No such unique ID")); } /** @@ -294,7 +360,10 @@ public void searchLookupBadQuery() throws Exception { * responses for parsing tests. */ private void setupAnswerSearchQuery() { - when(tsdb.executeSearch((SearchQuery)any())).thenAnswer( + mock_plugin = mock(SearchPlugin.class); + Whitebox.setInternalState(tsdb, "search", mock_plugin); + + when(mock_plugin.executeQuery((SearchQuery)any())).thenAnswer( new Answer<Deferred<SearchQuery>>() { @Override @@ -392,56 +461,70 @@ public Deferred<SearchQuery> answer(InvocationOnMock invocation) }); } - - @SuppressWarnings("unchecked") - private void setupAnswerLookupQuery() throws Exception { - PowerMockito.mockStatic(RowKey.class); - when(RowKey.metricNameAsync(tsdb, test_tsuids.get(0))) - .thenReturn(Deferred.fromResult("sys.cpu.user")); - when(RowKey.metricNameAsync(tsdb, test_tsuids.get(1))) - .thenReturn(Deferred.fromResult("sys.cpu.user")); - when(RowKey.metricNameAsync(tsdb, test_tsuids.get(2))) - .thenReturn(Deferred.fromResult("sys.cpu.nice")); - PowerMockito.mockStatic(UniqueId.class); - final List<byte[]> pair_a = new ArrayList<byte[]>(2); - pair_a.add(new byte[] { 0, 0, 1 }); - pair_a.add(new byte[] { 0, 0, 1 }); + private void setupLookup(final boolean use_meta) { + storage = new MockBase(tsdb, client, true, true, true, true); + if (use_meta) { + TestTimeSeriesLookup.generateMeta(tsdb, storage); + } else { + TestTimeSeriesLookup.generateData(tsdb, storage); + } - final List<byte[]> pair_b = new ArrayList<byte[]>(2); - pair_b.add(new byte[] { 0, 0, 1 }); - pair_b.add(new byte[] { 0, 0, 2 }); - - when(UniqueId.getTagPairsFromTSUID(test_tsuids.get(0))) - .thenReturn(pair_a); - when(UniqueId.getTagPairsFromTSUID(test_tsuids.get(1))) - .thenReturn(pair_b); - when(UniqueId.getTagPairsFromTSUID(test_tsuids.get(2))) - .thenReturn(pair_a); - when(UniqueId.uidToString((byte[])any())).thenCallRealMethod(); - - PowerMockito.mockStatic(Tags.class); - final HashMap<String, String> tags_a = new HashMap<String, String>(1); - tags_a.put("host", "web01"); - - final HashMap<String, String> tags_b = new HashMap<String, String>(1); - tags_b.put("host", "web02"); - - when(Tags.resolveIdsAsync(tsdb, pair_a)) - .thenReturn(Deferred.fromResult(tags_a)); - when(Tags.resolveIdsAsync(tsdb, pair_b)) - .thenReturn(Deferred.fromResult(tags_b)); - - when(Tags.parseWithMetric(anyString(), anyList())).thenCallRealMethod(); - when(Tags.splitString(anyString(), anyChar())).thenCallRealMethod(); - PowerMockito.doCallRealMethod().when(Tags.class, "parse", - anyList(), anyString()); - - mock_lookup = mock(TimeSeriesLookup.class); - PowerMockito.whenNew(TimeSeriesLookup.class) - .withArguments((TSDB)any(), (SearchQuery)any()) - .thenReturn(mock_lookup); - - when(mock_lookup.lookup()).thenReturn(test_tsuids); + when(metrics.getNameAsync(new byte[] { 0, 0, 4 })) + .thenAnswer(new Answer<Deferred<String>>() { + @Override + public Deferred<String> answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult("filtered"); + } + }); + when(tag_names.getNameAsync(new byte[] { 0, 0, 6 })) + .thenAnswer(new Answer<Deferred<String>>() { + @Override + public Deferred<String> answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult("6"); + } + }); + when(tag_names.getNameAsync(new byte[] { 0, 0, 8 })) + .thenAnswer(new Answer<Deferred<String>>() { + @Override + public Deferred<String> answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult("8"); + } + }); + when(tag_names.getNameAsync(new byte[] { 0, 0, 9 })) + .thenAnswer(new Answer<Deferred<String>>() { + @Override + public Deferred<String> answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult("9"); + } + }); + when(tag_values.getNameAsync(new byte[] { 0, 0, 7 })) + .thenAnswer(new Answer<Deferred<String>>() { + @Override + public Deferred<String> answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult("7"); + } + }); + when(tag_values.getNameAsync(new byte[] { 0, 0, 5 })) + .thenAnswer(new Answer<Deferred<String>>() { + @Override + public Deferred<String> answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult("5"); + } + }); + when(tag_values.getNameAsync(new byte[] { 0, 0, 10 })) + .thenAnswer(new Answer<Deferred<String>>() { + @Override + public Deferred<String> answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult("10"); + } + }); } } diff --git a/test/tsd/TestStatsRpc.java b/test/tsd/TestStatsRpc.java new file mode 100644 index 0000000000..cad88aee7a --- /dev/null +++ b/test/tsd/TestStatsRpc.java @@ -0,0 +1,110 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.tsd; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.nio.charset.Charset; + +import net.opentsdb.core.TSDB; +import net.opentsdb.stats.StatsCollector; +import net.opentsdb.utils.Config; + +import org.hbase.async.HBaseClient; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({HttpJsonSerializer.class, TSDB.class, Config.class, + HttpQuery.class, Thread.class, HBaseClient.class }) +public class TestStatsRpc { + private TSDB tsdb; + private HBaseClient client; + + @Before + public void before() throws Exception { + tsdb = NettyMocks.getMockedHTTPTSDB(); + client = mock(HBaseClient.class); + when(tsdb.getClient()).thenReturn(client); + } + +// TODO - revisit these as the one without port is failing intermittently. +// @Test +// public void statsWithOutPort() throws Exception { +// when(tsdb.getConfig().getBoolean("tsd.core.stats_with_port")) +// .thenReturn(false); +// final StatsRpc rpc = new StatsRpc(); +// HttpQuery query = NettyMocks.getQuery(tsdb, "/api/stats"); +// rpc.execute(tsdb, query); +// assertEquals(HttpResponseStatus.OK, query.response().getStatus()); +// final String json = +// query.response().getContent().toString(Charset.forName("UTF-8")); +// assertFalse(json.contains("port=4242")); +// } +// +// @Test +// public void statsWithPort() throws Exception { +// when(tsdb.getConfig().getBoolean("tsd.core.stats_with_port")) +// .thenReturn(true); +// when(tsdb.getConfig().getString("tsd.network.port")) +// .thenReturn("4242"); +// StatsCollector.setGlobalTags(tsdb.getConfig()); +// final StatsRpc rpc = new StatsRpc(); +// HttpQuery query = NettyMocks.getQuery(tsdb, "/api/stats"); +// rpc.execute(tsdb, query); +// assertEquals(HttpResponseStatus.OK, query.response().getStatus()); +// final String json = +// query.response().getContent().toString(Charset.forName("UTF-8")); +// assertTrue(json.contains("port=4242")); +// } + + @Test + public void printThreadStats() throws Exception { + final StatsRpc rpc = new StatsRpc(); + HttpQuery query = NettyMocks.getQuery(tsdb, "/api/stats/threads"); + rpc.execute(tsdb, query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + final String json = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertNotNull(json); + // check for some standard JVM threads since we can't mock Thread easily + assertTrue(json.contains("\"name\":\"Finalizer\"")); + assertTrue(json.contains("java.lang.ref.Finalizer$FinalizerThread.run")); + } + + @Test + public void printJVMStats() throws Exception { + final StatsRpc rpc = new StatsRpc(); + HttpQuery query = NettyMocks.getQuery(tsdb, "/api/stats/jvm"); + rpc.execute(tsdb, query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + final String json = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertNotNull(json); + assertTrue(json.contains("\"os\":{")); + assertTrue(json.contains("\"gc\":{")); + assertTrue(json.contains("\"runtime\":{")); + assertTrue(json.contains("\"pools\":{")); + assertTrue(json.contains("\"memory\":{")); + } +} + diff --git a/test/tsd/TestStatusRpc.java b/test/tsd/TestStatusRpc.java new file mode 100644 index 0000000000..0c4b03120e --- /dev/null +++ b/test/tsd/TestStatusRpc.java @@ -0,0 +1,97 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.tsd; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import com.stumbleupon.async.Deferred; + +import java.nio.charset.Charset; + +import net.opentsdb.core.TSDB; +import net.opentsdb.stats.StatsCollector; +import net.opentsdb.utils.Config; + +import org.hbase.async.HBaseClient; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({HttpJsonSerializer.class, TSDB.class, Config.class, + HttpQuery.class, Thread.class, HBaseClient.class }) +public class TestStatusRpc { + private TSDB tsdb; + private HBaseClient client; + private RpcManager.Status rpc; + + @Before + public void before() throws Exception { + rpc = new RpcManager.Status(); + tsdb = NettyMocks.getMockedHTTPTSDB(); + client = mock(HBaseClient.class); + when(tsdb.getClient()).thenReturn(client); + } + + private String getStatus() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, "/api/status"); + rpc.execute(tsdb, query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + final String json = + query.response().getContent().toString(Charset.forName("UTF-8")); + assertNotNull(json); + return json; + } + + @Test + public void printStatus() throws Exception { + // Initial status is "startup" + when(tsdb.checkNecessaryTablesAvailability()). + thenReturn(Deferred.fromResult(TSDB.TableAvailability.NONE)); + assertEquals(getStatus(), "{\"status\":\"startup\"}"); + + // Partial availability: + when(tsdb.checkNecessaryTablesAvailability()). + thenReturn(Deferred.fromResult(TSDB.TableAvailability.PARTIAL)); + assertEquals(getStatus(), "{\"status\":\"partial\"}"); + + // Full availibility: + when(tsdb.checkNecessaryTablesAvailability()). + thenReturn(Deferred.fromResult(TSDB.TableAvailability.FULL)); + assertEquals(getStatus(), "{\"status\":\"ok\"}"); + + // No availability (after having seen some in the past): + when(tsdb.checkNecessaryTablesAvailability()). + thenReturn(Deferred.fromResult(TSDB.TableAvailability.NONE)); + assertEquals(getStatus(), "{\"status\":\"error\"}"); + + // After shutdown status is "shutting-down", regardless of availability: + rpc.shutdown(); + when(tsdb.checkNecessaryTablesAvailability()). + thenReturn(Deferred.fromResult(TSDB.TableAvailability.NONE)); + assertEquals(getStatus(), "{\"status\":\"shutting-down\"}"); + when(tsdb.checkNecessaryTablesAvailability()). + thenReturn(Deferred.fromResult(TSDB.TableAvailability.PARTIAL)); + assertEquals(getStatus(), "{\"status\":\"shutting-down\"}"); + when(tsdb.checkNecessaryTablesAvailability()). + thenReturn(Deferred.fromResult(TSDB.TableAvailability.FULL)); + assertEquals(getStatus(), "{\"status\":\"shutting-down\"}"); + } +} diff --git a/test/tsd/TestSuggestRpc.java b/test/tsd/TestSuggestRpc.java index dbfdc426a0..c25e428ca3 100644 --- a/test/tsd/TestSuggestRpc.java +++ b/test/tsd/TestSuggestRpc.java @@ -39,7 +39,7 @@ public final class TestSuggestRpc { private SuggestRpc s = null; @Before - public void before() { + public void before() throws Exception { s = new SuggestRpc(); tsdb = NettyMocks.getMockedHTTPTSDB(); final List<String> metrics = new ArrayList<String>(); diff --git a/test/tsd/TestTreeRpc.java b/test/tsd/TestTreeRpc.java index f706fa67e0..4ec9fb7c28 100644 --- a/test/tsd/TestTreeRpc.java +++ b/test/tsd/TestTreeRpc.java @@ -15,10 +15,11 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.anyString; import static org.powermock.api.mockito.PowerMockito.mock; import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; import java.util.TreeMap; import net.opentsdb.core.TSDB; @@ -50,7 +51,6 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; @@ -62,7 +62,8 @@ @PrepareForTest({ TSDB.class, HBaseClient.class, GetRequest.class, Tree.class, PutRequest.class, KeyValue.class, Scanner.class, DeleteRequest.class }) public final class TestTreeRpc { - private static byte[] NAME_FAMILY = "name".getBytes(MockBase.ASCII()); + private final static byte[] NAME_FAMILY = "name".getBytes(MockBase.ASCII()); + private final static byte[] TREE_TABLE = "tsdb-tree".getBytes(); private TSDB tsdb; private HBaseClient client = mock(HBaseClient.class); private MockBase storage; @@ -121,10 +122,11 @@ public final class TestTreeRpc { @Before public void before() throws Exception { final Config config = new Config(false); - PowerMockito.whenNew(HBaseClient.class) - .withArguments(anyString(), anyString()).thenReturn(client); - tsdb = new TSDB(config); + tsdb = new TSDB(client, config); storage = new MockBase(tsdb, client, true, true, true, true); + final List<byte[]> families = new ArrayList<byte[]>(1); + families.add(Tree.TREE_FAMILY()); + storage.addTable(TREE_TABLE, families); } @Test @@ -192,7 +194,7 @@ public void handleTreeQSCreate() throws Exception { "/api/tree?name=NewTree&method_override=post"); rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); - assertEquals(1, storage.numColumns(new byte[] { 0, 3 })); + assertEquals(1, storage.numColumns(TREE_TABLE, new byte[] { 0, 3 })); } @Test (expected = BadRequestException.class) @@ -220,7 +222,7 @@ public void handleTreePOSTCreate() throws Exception { "/api/tree", "{\"name\":\"New Tree\"}"); rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); - assertEquals(1, storage.numColumns(new byte[] { 0, 3 })); + assertEquals(1, storage.numColumns(TREE_TABLE, new byte[] { 0, 3 })); } @Test @@ -315,14 +317,14 @@ public void handleTreeQSDeleteDefault() throws Exception { HttpQuery query = NettyMocks.getQuery(tsdb, "/api/tree?treeid=1&method_override=delete"); // make sure the root is there BEFORE we delete - assertEquals(4, storage.numColumns(new byte[] { 0, 1 })); + assertEquals(4, storage.numColumns(TREE_TABLE, new byte[] { 0, 1 })); rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); // make sure the definition is still there but the root is gone - assertEquals(3, storage.numColumns(new byte[] { 0, 1 })); - assertEquals(-1, storage.numColumns( + assertEquals(3, storage.numColumns(TREE_TABLE, new byte[] { 0, 1 })); + assertEquals(-1, storage.numColumns(TREE_TABLE, Branch.stringToId("00010001BECD000181A8"))); - assertEquals(-1, storage.numColumns( + assertEquals(-1, storage.numColumns(TREE_TABLE, Branch.stringToId("00010001BECD000181A8BF992A99"))); } @@ -332,14 +334,14 @@ public void handleTreeQSDeleteDefinition() throws Exception { HttpQuery query = NettyMocks.getQuery(tsdb, "/api/tree?treeid=1&method_override=delete&definition=true"); // make sure the root is there BEFORE we delete - assertEquals(4, storage.numColumns(new byte[] { 0, 1 })); + assertEquals(4, storage.numColumns(TREE_TABLE, new byte[] { 0, 1 })); rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); // make sure the definition has been deleted too - assertEquals(-1, storage.numColumns(new byte[] { 0, 1 })); - assertEquals(-1, storage.numColumns( + assertEquals(-1, storage.numColumns(TREE_TABLE, new byte[] { 0, 1 })); + assertEquals(-1, storage.numColumns(TREE_TABLE, Branch.stringToId("00010001BECD000181A8"))); - assertEquals(-1, storage.numColumns( + assertEquals(-1, storage.numColumns(TREE_TABLE, Branch.stringToId("00010001BECD000181A8BF992A99"))); } @@ -349,14 +351,14 @@ public void handleTreePOSTDeleteDefault() throws Exception { HttpQuery query = NettyMocks.deleteQuery(tsdb, "/api/tree", "{\"treeId\":1}"); // make sure the root is there BEFORE we delete - assertEquals(4, storage.numColumns(new byte[] { 0, 1 })); + assertEquals(4, storage.numColumns(TREE_TABLE, new byte[] { 0, 1 })); rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); // make sure the definition is still there but the root is gone - assertEquals(3, storage.numColumns(new byte[] { 0, 1 })); - assertEquals(-1, storage.numColumns( + assertEquals(3, storage.numColumns(TREE_TABLE, new byte[] { 0, 1 })); + assertEquals(-1, storage.numColumns(TREE_TABLE, Branch.stringToId("00010001BECD000181A8"))); - assertEquals(-1, storage.numColumns( + assertEquals(-1, storage.numColumns(TREE_TABLE, Branch.stringToId("00010001BECD000181A8BF992A99"))); } @@ -366,14 +368,14 @@ public void handleTreePOSTDeleteDefinition() throws Exception { HttpQuery query = NettyMocks.deleteQuery(tsdb, "/api/tree", "{\"treeId\":1,\"definition\":true}"); // make sure the root is there BEFORE we delete - assertEquals(4, storage.numColumns(new byte[] { 0, 1 })); + assertEquals(4, storage.numColumns(TREE_TABLE, new byte[] { 0, 1 })); rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); // make sure the definition has been deleted too - assertEquals(-1, storage.numColumns(new byte[] { 0, 1 })); - assertEquals(-1, storage.numColumns( + assertEquals(-1, storage.numColumns(TREE_TABLE, new byte[] { 0, 1 })); + assertEquals(-1, storage.numColumns(TREE_TABLE, Branch.stringToId("00010001BECD000181A8"))); - assertEquals(-1, storage.numColumns( + assertEquals(-1, storage.numColumns(TREE_TABLE, Branch.stringToId("00010001BECD000181A8BF992A99"))); } @@ -628,7 +630,7 @@ public void handleRuleQSDelete() throws Exception { "/api/tree/rule?treeid=1&level=1&order=0&method_override=delete"); rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); - assertEquals(3, storage.numColumns(new byte[] { 0, 1 })); + assertEquals(3, storage.numColumns(TREE_TABLE, new byte[] { 0, 1 })); } @Test (expected = BadRequestException.class) @@ -646,7 +648,7 @@ public void handleRuleDELETE() throws Exception { "/api/tree/rule", "{\"treeId\":1,\"level\":1,\"order\":0}"); rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); - assertEquals(3, storage.numColumns(new byte[] { 0, 1 })); + assertEquals(3, storage.numColumns(TREE_TABLE, new byte[] { 0, 1 })); } @Test (expected = BadRequestException.class) @@ -675,8 +677,9 @@ public void handleRulesPOST() throws Exception { "\"tagk\",\"field\":\"host\"}]"); rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); - assertEquals(5, storage.numColumns(new byte[] { 0, 1 })); - final String rule = new String(storage.getColumn(new byte[] { 0, 1 }, + assertEquals(5, storage.numColumns(TREE_TABLE, new byte[] { 0, 1 })); + final String rule = new String(storage.getColumn(TREE_TABLE, + new byte[] { 0, 1 }, Tree.TREE_FAMILY(), "tree_rule:0:0".getBytes(MockBase.ASCII())), MockBase.ASCII()); assertTrue(rule.contains("\"type\":\"METRIC\"")); assertTrue(rule.contains("description\":\"Host Name\"")); @@ -700,8 +703,9 @@ public void handleRulesPUT() throws Exception { "\"tagk\",\"field\":\"host\"}]"); rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); - assertEquals(5, storage.numColumns(new byte[] { 0, 1 })); - final String rule = new String(storage.getColumn(new byte[] { 0, 1 }, + assertEquals(5, storage.numColumns(TREE_TABLE, new byte[] { 0, 1 })); + final String rule = new String(storage.getColumn(TREE_TABLE, + new byte[] { 0, 1 }, Tree.TREE_FAMILY(), "tree_rule:0:0".getBytes(MockBase.ASCII())), MockBase.ASCII()); assertTrue(rule.contains("\"type\":\"METRIC\"")); assertFalse(rule.contains("\"description\":\"Host Name\"")); @@ -725,7 +729,7 @@ public void handleRulesDeleteQS() throws Exception { "/api/tree/rules?treeid=1&method_override=delete"); rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); - assertEquals(2, storage.numColumns(new byte[] { 0, 1 })); + assertEquals(2, storage.numColumns(TREE_TABLE, new byte[] { 0, 1 })); } @Test @@ -735,7 +739,7 @@ public void handleRulesDelete() throws Exception { "/api/tree/rules?treeid=1", ""); rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.NO_CONTENT, query.response().getStatus()); - assertEquals(2, storage.numColumns(new byte[] { 0, 1 })); + assertEquals(2, storage.numColumns(TREE_TABLE, new byte[] { 0, 1 })); } @Test (expected = BadRequestException.class) @@ -855,7 +859,7 @@ public void handleTestNSU() throws Exception { setupStorage(); setupBranch(); setupTSMeta(); - storage.flushRow(new byte[] { 0, 0, 2 }); + storage.flushRow("tsdb-uid".getBytes(), new byte[] { 0, 0, 2 }); HttpQuery query = NettyMocks.getQuery(tsdb, "/api/tree/test?treeid=1&tsuids=000001000001000001000002000002"); rpc.execute(tsdb, query); @@ -1143,7 +1147,7 @@ public void handleNotMatchedBadMethod() throws Exception { * child branch, leaves and some collisions and no matches. These are used for * most of the tests so they're all here. */ - private void setupStorage() throws Exception { + private void setupStorage() throws Exception { Tree tree = TestTree.buildTestTree(); // store root @@ -1152,13 +1156,14 @@ private void setupStorage() throws Exception { root.setDisplayName("ROOT"); root_path.put(0, "ROOT"); root.prependParentPath(root_path); - storage.addColumn(root.compileBranchId(), Tree.TREE_FAMILY(), + storage.addColumn(TREE_TABLE, root.compileBranchId(), Tree.TREE_FAMILY(), "branch".getBytes(MockBase.ASCII()), (byte[])branchToStorageJson.invoke(root)); // store the first tree byte[] key = new byte[] { 0, 1 }; - storage.addColumn(key, Tree.TREE_FAMILY(), "tree".getBytes(MockBase.ASCII()), + storage.addColumn(TREE_TABLE, key, Tree.TREE_FAMILY(), + "tree".getBytes(MockBase.ASCII()), (byte[])TreetoStorageJson.invoke(TestTree.buildTestTree())); TreeRule rule = new TreeRule(1); @@ -1166,7 +1171,7 @@ private void setupStorage() throws Exception { rule.setDescription("Hostname rule"); rule.setType(TreeRuleType.TAGK); rule.setDescription("Host Name"); - storage.addColumn(key, Tree.TREE_FAMILY(), + storage.addColumn(TREE_TABLE, key, Tree.TREE_FAMILY(), "tree_rule:0:0".getBytes(MockBase.ASCII()), JSON.serializeToBytes(rule)); @@ -1175,7 +1180,7 @@ private void setupStorage() throws Exception { rule.setLevel(1); rule.setNotes("Metric rule"); rule.setType(TreeRuleType.METRIC); - storage.addColumn(key, Tree.TREE_FAMILY(), + storage.addColumn(TREE_TABLE, key, Tree.TREE_FAMILY(), "tree_rule:1:0".getBytes(MockBase.ASCII()), JSON.serializeToBytes(rule)); @@ -1184,7 +1189,7 @@ private void setupStorage() throws Exception { root_path = new TreeMap<Integer, String>(); root_path.put(0, "ROOT"); root.prependParentPath(root_path); - storage.addColumn(key, Tree.TREE_FAMILY(), + storage.addColumn(TREE_TABLE, key, Tree.TREE_FAMILY(), "branch".getBytes(MockBase.ASCII()), (byte[])branchToStorageJson.invoke(root)); @@ -1195,13 +1200,14 @@ private void setupStorage() throws Exception { tree2.setTreeId(2); tree2.setName("2nd Tree"); tree2.setDescription("Other Tree"); - storage.addColumn(key, Tree.TREE_FAMILY(), "tree".getBytes(MockBase.ASCII()), + storage.addColumn(TREE_TABLE, key, Tree.TREE_FAMILY(), + "tree".getBytes(MockBase.ASCII()), (byte[])TreetoStorageJson.invoke(tree2)); rule = new TreeRule(2); rule.setField("host"); rule.setType(TreeRuleType.TAGK); - storage.addColumn(key, Tree.TREE_FAMILY(), + storage.addColumn(TREE_TABLE, key, Tree.TREE_FAMILY(), "tree_rule:0:0".getBytes(MockBase.ASCII()), JSON.serializeToBytes(rule)); @@ -1209,7 +1215,7 @@ private void setupStorage() throws Exception { rule.setField(""); rule.setLevel(1); rule.setType(TreeRuleType.METRIC); - storage.addColumn(key, Tree.TREE_FAMILY(), + storage.addColumn(TREE_TABLE, key, Tree.TREE_FAMILY(), "tree_rule:1:0".getBytes(MockBase.ASCII()), JSON.serializeToBytes(rule)); @@ -1218,7 +1224,7 @@ private void setupStorage() throws Exception { root_path = new TreeMap<Integer, String>(); root_path.put(0, "ROOT"); root.prependParentPath(root_path); - storage.addColumn(key, Tree.TREE_FAMILY(), + storage.addColumn(TREE_TABLE, key, Tree.TREE_FAMILY(), "branch".getBytes(MockBase.ASCII()), (byte[])branchToStorageJson.invoke(root)); @@ -1233,7 +1239,7 @@ private void setupStorage() throws Exception { byte[] tsuid_bytes = UniqueId.stringToUid(tsuid); System.arraycopy(tsuid_bytes, 0, qualifier, Tree.COLLISION_PREFIX().length, tsuid_bytes.length); - storage.addColumn(key, Tree.TREE_FAMILY(), qualifier, + storage.addColumn(TREE_TABLE, key, Tree.TREE_FAMILY(), qualifier, "AAAAAA".getBytes(MockBase.ASCII())); tsuid = "020202"; @@ -1244,7 +1250,7 @@ private void setupStorage() throws Exception { tsuid_bytes = UniqueId.stringToUid(tsuid); System.arraycopy(tsuid_bytes, 0, qualifier, Tree.COLLISION_PREFIX().length, tsuid_bytes.length); - storage.addColumn(key, Tree.TREE_FAMILY(), qualifier, + storage.addColumn(TREE_TABLE, key, Tree.TREE_FAMILY(), qualifier, "BBBBBB".getBytes(MockBase.ASCII())); // not matched @@ -1257,7 +1263,7 @@ private void setupStorage() throws Exception { tsuid_bytes = UniqueId.stringToUid(tsuid); System.arraycopy(tsuid_bytes, 0, qualifier, Tree.NOT_MATCHED_PREFIX().length, tsuid_bytes.length); - storage.addColumn(key, Tree.TREE_FAMILY(), qualifier, + storage.addColumn(TREE_TABLE, key, Tree.TREE_FAMILY(), qualifier, "Failed rule 0:0".getBytes(MockBase.ASCII())); tsuid = "020202"; @@ -1268,7 +1274,7 @@ private void setupStorage() throws Exception { tsuid_bytes = UniqueId.stringToUid(tsuid); System.arraycopy(tsuid_bytes, 0, qualifier, Tree.NOT_MATCHED_PREFIX().length, tsuid_bytes.length); - storage.addColumn(key, Tree.TREE_FAMILY(), qualifier, + storage.addColumn(TREE_TABLE, key, Tree.TREE_FAMILY(), qualifier, "Failed rule 1:1".getBytes(MockBase.ASCII())); // drop some branches in for tree 1 @@ -1279,18 +1285,18 @@ private void setupStorage() throws Exception { path.put(2, "cpu"); branch.prependParentPath(path); branch.setDisplayName("cpu"); - storage.addColumn(branch.compileBranchId(), Tree.TREE_FAMILY(), + storage.addColumn(TREE_TABLE, branch.compileBranchId(), Tree.TREE_FAMILY(), "branch".getBytes(MockBase.ASCII()), (byte[])branchToStorageJson.invoke(branch)); Leaf leaf = new Leaf("user", "000001000001000001"); qualifier = leaf.columnQualifier(); - storage.addColumn(branch.compileBranchId(), Tree.TREE_FAMILY(), + storage.addColumn(TREE_TABLE, branch.compileBranchId(), Tree.TREE_FAMILY(), qualifier, (byte[])LeaftoStorageJson.invoke(leaf)); leaf = new Leaf("nice", "000002000002000002"); qualifier = leaf.columnQualifier(); - storage.addColumn(branch.compileBranchId(), Tree.TREE_FAMILY(), + storage.addColumn(TREE_TABLE, branch.compileBranchId(), Tree.TREE_FAMILY(), qualifier, (byte[])LeaftoStorageJson.invoke(leaf)); // child branch @@ -1298,13 +1304,13 @@ private void setupStorage() throws Exception { path.put(3, "mboard"); branch.prependParentPath(path); branch.setDisplayName("mboard"); - storage.addColumn(branch.compileBranchId(), Tree.TREE_FAMILY(), + storage.addColumn(TREE_TABLE, branch.compileBranchId(), Tree.TREE_FAMILY(), "branch".getBytes(MockBase.ASCII()), (byte[])branchToStorageJson.invoke(branch)); leaf = new Leaf("Asus", "000003000003000003"); qualifier = leaf.columnQualifier(); - storage.addColumn(branch.compileBranchId(), Tree.TREE_FAMILY(), + storage.addColumn(TREE_TABLE, branch.compileBranchId(), Tree.TREE_FAMILY(), qualifier, (byte[])LeaftoStorageJson.invoke(leaf)); } @@ -1314,13 +1320,13 @@ private void setupStorage() throws Exception { * find their name maps. */ private void setupBranch() { - storage.addColumn(new byte[] { 0, 0, 1 }, NAME_FAMILY, + storage.addColumn("tsdb-uid".getBytes(), new byte[] { 0, 0, 1 }, NAME_FAMILY, "metrics".getBytes(MockBase.ASCII()), "sys.cpu.0".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1 }, NAME_FAMILY, + storage.addColumn("tsdb-uid".getBytes(), new byte[] { 0, 0, 1 }, NAME_FAMILY, "tagk".getBytes(MockBase.ASCII()), "host".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1 }, NAME_FAMILY, + storage.addColumn("tsdb-uid".getBytes(), new byte[] { 0, 0, 1 }, NAME_FAMILY, "tagv".getBytes(MockBase.ASCII()), "web01".getBytes(MockBase.ASCII())); } @@ -1331,41 +1337,47 @@ private void setupBranch() { * parsed through the tree. */ private void setupTSMeta() throws Exception { + final byte[] meta_table = "tsdb-meta".getBytes(); + final byte[] uid_table = "tsdb-uid".getBytes(); + final List<byte[]> families = new ArrayList<byte[]>(1); + families.add(TSMeta.FAMILY); + storage.addTable(meta_table, families); final TSMeta meta = new TSMeta("000001000001000001000002000002"); - storage.addColumn(UniqueId.stringToUid("000001000001000001000002000002"), + storage.addColumn(meta_table, + UniqueId.stringToUid("000001000001000001000002000002"), NAME_FAMILY, "ts_meta".getBytes(MockBase.ASCII()), (byte[])TSMetagetStorageJSON.invoke(meta)); final UIDMeta metric = new UIDMeta(UniqueIdType.METRIC, new byte[] { 0, 0, 1 }, "sys.cpu.0"); - storage.addColumn(new byte[] { 0, 0, 1 }, NAME_FAMILY, + storage.addColumn(uid_table, new byte[] { 0, 0, 1 }, NAME_FAMILY, "metric_meta".getBytes(MockBase.ASCII()), (byte[])UIDMetagetStorageJSON.invoke(metric)); final UIDMeta tagk1 = new UIDMeta(UniqueIdType.TAGK, new byte[] { 0, 0, 1 }, "host"); - storage.addColumn(new byte[] { 0, 0, 1 }, NAME_FAMILY, + storage.addColumn(uid_table, new byte[] { 0, 0, 1 }, NAME_FAMILY, "tagk_meta".getBytes(MockBase.ASCII()), (byte[])UIDMetagetStorageJSON.invoke(tagk1)); final UIDMeta tagv1 = new UIDMeta(UniqueIdType.TAGV, new byte[] { 0, 0, 1 }, "web-01.lga.mysite.com"); - storage.addColumn(new byte[] { 0, 0, 1 }, NAME_FAMILY, + storage.addColumn(uid_table, new byte[] { 0, 0, 1 }, NAME_FAMILY, "tagv_meta".getBytes(MockBase.ASCII()), (byte[])UIDMetagetStorageJSON.invoke(tagv1)); final UIDMeta tagk2 = new UIDMeta(UniqueIdType.TAGK, new byte[] { 0, 0, 2 }, "type"); - storage.addColumn(new byte[] { 0, 0, 2 }, NAME_FAMILY, + storage.addColumn(uid_table, new byte[] { 0, 0, 2 }, NAME_FAMILY, "tagk_meta".getBytes(MockBase.ASCII()), (byte[])UIDMetagetStorageJSON.invoke(tagk2)); final UIDMeta tagv2 = new UIDMeta(UniqueIdType.TAGV, new byte[] { 0, 0, 2 }, "user"); - storage.addColumn(new byte[] { 0, 0, 2 }, NAME_FAMILY, + storage.addColumn(uid_table, new byte[] { 0, 0, 2 }, NAME_FAMILY, "tagv_meta".getBytes(MockBase.ASCII()), (byte[])UIDMetagetStorageJSON.invoke(tagv2)); - storage.addColumn(new byte[] { 0, 0, 2 }, NAME_FAMILY, + storage.addColumn(uid_table, new byte[] { 0, 0, 2 }, NAME_FAMILY, "tagk".getBytes(MockBase.ASCII()), "type".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 2 }, NAME_FAMILY, + storage.addColumn(uid_table, new byte[] { 0, 0, 2 }, NAME_FAMILY, "tagv".getBytes(MockBase.ASCII()), "user".getBytes(MockBase.ASCII())); } diff --git a/test/tsd/TestUniqueIdRpc.java b/test/tsd/TestUniqueIdRpc.java index 16ef3ec874..b90e1e6ceb 100644 --- a/test/tsd/TestUniqueIdRpc.java +++ b/test/tsd/TestUniqueIdRpc.java @@ -14,12 +14,14 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.when; import static org.powermock.api.mockito.PowerMockito.mock; import java.lang.reflect.Field; import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; import net.opentsdb.core.TSDB; import net.opentsdb.meta.TSMeta; @@ -39,7 +41,6 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; @@ -54,14 +55,16 @@ HBaseClient.class, RowLock.class, UniqueIdRpc.class, KeyValue.class, GetRequest.class, Scanner.class, UniqueId.class}) public final class TestUniqueIdRpc { - private static byte[] NAME_FAMILY = "name".getBytes(MockBase.ASCII()); + private final static byte[] NAME_FAMILY = "name".getBytes(MockBase.ASCII()); + private final static byte[] UID_TABLE = "tsdb-uid".getBytes(MockBase.ASCII()); + private final static byte[] META_TABLE = "tsdb-meta".getBytes(); private TSDB tsdb = null; private HBaseClient client = mock(HBaseClient.class); private UniqueId metrics = mock(UniqueId.class); private UniqueId tag_names = mock(UniqueId.class); private UniqueId tag_values = mock(UniqueId.class); private MockBase storage; - private UniqueIdRpc rpc = new UniqueIdRpc(); + private UniqueIdRpc rpc = new UniqueIdRpc(TSDB.OperationMode.READWRITE); @Before public void before() throws Exception { @@ -86,6 +89,15 @@ public void notImplemented() throws Exception { } // Test /api/uid/assign ---------------------- + + @Test (expected = BadRequestException.class) + public void assignReadOnlyMode() throws Exception { + setupAssign(); + rpc = new UniqueIdRpc(TSDB.OperationMode.READONLY); + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/uid/assign?metric=sys.cpu.0"); + this.rpc.execute(tsdb, query); + } @Test public void assignQsMetricSingle() throws Exception { @@ -105,9 +117,10 @@ public void assignQsMetricDouble() throws Exception { "/api/uid/assign?metric=sys.cpu.0,sys.cpu.2"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); - assertEquals( - "{\"metric\":{\"sys.cpu.0\":\"000001\",\"sys.cpu.2\":\"000003\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"sys.cpu.0\":\"000001\"")); + assertTrue(json.contains("\"sys.cpu.2\":\"000003\"")); } @Test @@ -117,9 +130,10 @@ public void assignQsMetricSingleBad() throws Exception { "/api/uid/assign?metric=sys.cpu.1"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); - assertEquals("{\"metric_errors\":{\"sys.cpu.1\":\"Name already exists with " - + "UID: 000002\"},\"metric\":{}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("{\"sys.cpu.1\":\"Name already exists with " + + "UID: 000002\"}")); } @Test @@ -129,10 +143,12 @@ public void assignQsMetric2Good1Bad() throws Exception { "/api/uid/assign?metric=sys.cpu.0,sys.cpu.1,sys.cpu.2"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); - assertEquals("{\"metric_errors\":{\"sys.cpu.1\":\"Name already exists with " - + "UID: 000002\"},\"metric\":{\"sys.cpu.0\":\"000001\",\"sys.cpu.2\":" - + "\"000003\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("{\"sys.cpu.1\":\"Name already exists with " + + "UID: 000002\"}")); + assertTrue(json.contains("{\"sys.cpu.0\":\"000001\",\"sys.cpu.2\":" + + "\"000003\"}")); } @Test @@ -153,9 +169,10 @@ public void assignQsTagkDouble() throws Exception { "/api/uid/assign?tagk=host,fqdn"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); - assertEquals( - "{\"tagk\":{\"fqdn\":\"000003\",\"host\":\"000001\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"fqdn\":\"000003\"")); + assertTrue(json.contains("\"host\":\"000001\"")); } @Test @@ -165,9 +182,10 @@ public void assignQsTagkSingleBad() throws Exception { "/api/uid/assign?tagk=datacenter"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); - assertEquals("{\"tagk_errors\":{\"datacenter\":\"Name already exists with " - + "UID: 000002\"},\"tagk\":{}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"tagk_errors\":{\"datacenter\":" + + "\"Name already exists with UID: 000002\"}")); } @Test @@ -177,9 +195,12 @@ public void assignQsTagk2Good1Bad() throws Exception { "/api/uid/assign?tagk=host,datacenter,fqdn"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); - assertEquals("{\"tagk_errors\":{\"datacenter\":\"Name already exists with " - + "UID: 000002\"},\"tagk\":{\"fqdn\":\"000003\",\"host\":\"000001\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("{\"datacenter\":\"Name already exists with " + + "UID: 000002\"}")); + assertTrue(json.contains("\"fqdn\":\"000003\"")); + assertTrue(json.contains("\"host\":\"000001\"")); } @Test @@ -200,9 +221,10 @@ public void assignQsTagvDouble() throws Exception { "/api/uid/assign?tagv=localhost,foo"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); - assertEquals( - "{\"tagv\":{\"foo\":\"000003\",\"localhost\":\"000001\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"foo\":\"000003\"")); + assertTrue(json.contains("\"localhost\":\"000001\"")); } @Test @@ -212,9 +234,10 @@ public void assignQsTagvSingleBad() throws Exception { "/api/uid/assign?tagv=myserver"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); - assertEquals("{\"tagv\":{},\"tagv_errors\":{\"myserver\":\"Name already " - + "exists with UID: 000002\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"tagv_errors\":{\"myserver\":\"Name already " + + "exists with UID: 000002\"}")); } @Test @@ -224,10 +247,12 @@ public void assignQsTagv2Good1Bad() throws Exception { "/api/uid/assign?tagv=localhost,myserver,foo"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); - assertEquals("{\"tagv\":{\"foo\":\"000003\",\"localhost\":\"000001\"}," - + "\"tagv_errors\":{\"myserver\":\"Name already exists with " - + "UID: 000002\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"foo\":\"000003\"")); + assertTrue(json.contains("\"localhost\":\"000001\"")); + assertTrue(json.contains("{\"myserver\":\"Name already exists with " + + "UID: 000002\"}")); } @Test @@ -295,9 +320,10 @@ public void assignPostMetricDouble() throws Exception { "{\"metric\":[\"sys.cpu.0\",\"sys.cpu.2\"]}"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); - assertEquals( - "{\"metric\":{\"sys.cpu.0\":\"000001\",\"sys.cpu.2\":\"000003\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"sys.cpu.0\":\"000001\"")); + assertTrue(json.contains("\"sys.cpu.2\":\"000003\"")); } public void assignPostMetricSingleBad() throws Exception { @@ -306,9 +332,10 @@ public void assignPostMetricSingleBad() throws Exception { "{\"metric\":[\"sys.cpu.2\"]}"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); - assertEquals("{\"metric_errors\":{\"sys.cpu.1\":\"Name already exists with " - + "UID: 000002\"},\"metric\":{}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("{\"sys.cpu.1\":\"Name already exists with " + + "UID: 000002\"}")); } public void assignPostMetric2Good1Bad() throws Exception { @@ -317,10 +344,12 @@ public void assignPostMetric2Good1Bad() throws Exception { "{\"metric\":[\"sys.cpu.0\",\"sys.cpu.1\",\"sys.cpu.2\"]}"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); - assertEquals("{\"metric_errors\":{\"sys.cpu.1\":\"Name already exists with " - + "UID: 000002\"},\"metric\":{\"sys.cpu.0\":\"000001\",\"sys.cpu.2\":" - + "\"000003\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("{\"sys.cpu.1\":\"Name already exists with " + + "UID: 000002\"}")); + assertTrue(json.contains("\"sys.cpu.0\":\"000001\"")); + assertTrue(json.contains("\"sys.cpu.2\":\"000003\"")); } @Test @@ -340,9 +369,10 @@ public void assignPostTagkDouble() throws Exception { "{\"tagk\":[\"host\",\"fqdn\"]}"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); - assertEquals( - "{\"tagk\":{\"fqdn\":\"000003\",\"host\":\"000001\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"fqdn\":\"000003\"")); + assertTrue(json.contains("\"host\":\"000001\"")); } public void assignPostTagkSingleBad() throws Exception { @@ -351,9 +381,10 @@ public void assignPostTagkSingleBad() throws Exception { "{\"tagk\":[\"datacenter\"]}"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); - assertEquals("{\"tagk_errors\":{\"datacenter\":\"Name already exists with " - + "UID: 000002\"},\"tagk\":{}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"datacenter\":\"Name already exists with " + + "UID: 000002\"")); } public void assignPostTagk2Good1Bad() throws Exception { @@ -362,9 +393,12 @@ public void assignPostTagk2Good1Bad() throws Exception { "{\"tagk\":[\"host\",\"datacenter\",\"fqdn\"]}"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); - assertEquals("{\"tagk_errors\":{\"datacenter\":\"Name already exists with " - + "UID: 000002\"},\"tagk\":{\"fqdn\":\"000003\",\"host\":\"000001\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("{\"datacenter\":\"Name already exists with " + + "UID: 000002\"}")); + assertTrue(json.contains("\"fqdn\":\"000003\"")); + assertTrue(json.contains("\"host\":\"000001\"")); } @Test @@ -384,9 +418,10 @@ public void assignPostTagvDouble() throws Exception { "{\"tagv\":[\"localhost\",\"foo\"]}"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); - assertEquals( - "{\"tagv\":{\"foo\":\"000003\",\"localhost\":\"000001\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"foo\":\"000003\"")); + assertTrue(json.contains("\"localhost\":\"000001\"")); } public void assignPostTagvSingleBad() throws Exception { @@ -395,9 +430,10 @@ public void assignPostTagvSingleBad() throws Exception { "{\"tagv\":[\"myserver\"]}"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); - assertEquals("{\"tagv\":{},\"tagv_errors\":{\"myserver\":\"Name already " - + "exists with UID: 000002\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"tagv_errors\":{\"myserver\":\"Name already " + + "exists with UID: 000002\"}")); } public void assignPostTagv2Good1Bad() throws Exception { @@ -406,10 +442,12 @@ public void assignPostTagv2Good1Bad() throws Exception { "{\"tagv\":[\"localhost\",\"myserver\",\"foo\"]}"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); - assertEquals("{\"tagv\":{\"foo\":\"000003\",\"localhost\":\"000001\"}," - + "\"tagv_errors\":{\"myserver\":\"Name already exists with " - + "UID: 000002\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"foo\":\"000003\"")); + assertTrue(json.contains("\"localhost\":\"000001\"")); + assertTrue(json.contains("\"tagv_errors\":{\"myserver\":\"Name already exists with " + + "UID: 000002\"}")); } @Test @@ -503,7 +541,177 @@ public void stringToUniqueIdTypeEmpty() throws Exception { UniqueId.stringToUniqueIdType("Not a type"); } + // Test /api/uid/rename ---------------------- + + @Test (expected = BadRequestException.class) + public void renameBadMethod() throws Exception { + HttpQuery query = NettyMocks.putQuery(tsdb, "/api/uid/rename", ""); + rpc.execute(tsdb, query); + } + + @Test (expected = BadRequestException.class) + public void renameReadOnlyMode() throws Exception { + rpc = new UniqueIdRpc(TSDB.OperationMode.READONLY); + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/uid/rename", + "{\"metric\":\"sys.cpu.1\",\"name\":\"sys.cpu.2\"}"); + this.rpc.execute(tsdb, query); + } + + @Test + public void renamePostMetric() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/uid/rename", + "{\"metric\":\"sys.cpu.1\",\"name\":\"sys.cpu.2\"}"); + rpc.execute(tsdb, query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertEquals("{\"result\":\"true\"}", + query.response().getContent().toString(Charset.forName("UTF-8"))); + } + + @Test + public void renamePostTagk() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/uid/rename", + "{\"tagk\":\"datacenter\",\"name\":\"datacluster\"}"); + rpc.execute(tsdb, query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertEquals("{\"result\":\"true\"}", + query.response().getContent().toString(Charset.forName("UTF-8"))); + } + + @Test + public void renamePostTagv() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/uid/rename", + "{\"tagv\":\"localhost\",\"name\":\"127.0.0.1\"}"); + rpc.execute(tsdb, query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertEquals("{\"result\":\"true\"}", + query.response().getContent().toString(Charset.forName("UTF-8"))); + } + + @Test (expected = BadRequestException.class) + public void renamePostNoName() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/uid/rename", + "{\"tagk\":\"localhost\",\"not_name\":\"127.0.0.1\"}"); + rpc.execute(tsdb, query); + } + + @Test (expected = BadRequestException.class) + public void renamePostNoType() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/uid/rename", + "{\"name\":\"127.0.0.1\"}"); + rpc.execute(tsdb, query); + } + + @Test (expected = BadRequestException.class) + public void renamePostNotJSON() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/uid/rename", "Not JSON"); + rpc.execute(tsdb, query); + } + + @Test (expected = BadRequestException.class) + public void renamePostZeroLengthContent() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/uid/rename", ""); + rpc.execute(tsdb, query); + } + + @Test (expected = BadRequestException.class) + public void renamePostEmptyJSON() throws Exception { + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/uid/rename", "{}"); + rpc.execute(tsdb, query); + } + + @Test + public void renameQsMetric() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/uid/rename?metric=sys.cpu.1&name=sys.cpu.2"); + rpc.execute(tsdb, query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertEquals("{\"result\":\"true\"}", + query.response().getContent().toString(Charset.forName("UTF-8"))); + } + + @Test + public void renameQsTagk() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/uid/rename?tagk=datacenter&name=datacluster"); + rpc.execute(tsdb, query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertEquals("{\"result\":\"true\"}", + query.response().getContent().toString(Charset.forName("UTF-8"))); + } + + @Test + public void renameQsTagv() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/uid/rename?tagv=localhost&name=127.0.0.1"); + rpc.execute(tsdb, query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertEquals("{\"result\":\"true\"}", + query.response().getContent().toString(Charset.forName("UTF-8"))); + } + + @Test + public void renameQsSkipUnsupportedParam() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/uid/rename?tagv=localhost&name=127.0.0.1&drop=db"); + rpc.execute(tsdb, query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + assertEquals("{\"result\":\"true\"}", + query.response().getContent().toString(Charset.forName("UTF-8"))); + } + + @Test (expected = BadRequestException.class) + public void renameQsMissingType() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/uid/rename?name=127.0.0.1"); + rpc.execute(tsdb, query); + } + + @Test (expected = BadRequestException.class) + public void renameQsMissingName() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/uid/rename?metric=sys.cpu.1"); + rpc.execute(tsdb, query); + } + + @Test (expected = BadRequestException.class) + public void renameQsNoParamValue() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/uid/rename?metric=&name=sys.cpu.2"); + rpc.execute(tsdb, query); + } + + @Test (expected = BadRequestException.class) + public void renameQsNoParam() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/uid/rename?"); + rpc.execute(tsdb, query); + } + + @Test + public void renameRenameException() throws Exception { + final String message = "New name already exists"; + doThrow(new IllegalArgumentException(message)).when(tsdb).renameUid("tagv", + "localhost", "localhost"); + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/uid/rename?tagv=localhost&name=localhost"); + rpc.execute(tsdb, query); + assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"error\":\"" + message + "\"")); + assertTrue(json.contains("\"result\":\"false\"")); + } + // Teset /api/uid/uidmeta -------------------- + + @Test (expected = BadRequestException.class) + public void uidGetWriteOnlyMode() throws Exception { + rpc = new UniqueIdRpc(TSDB.OperationMode.WRITEONLY); + setupUID(); + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/uid/uidmeta?type=metric&uid=000001"); + rpc.execute(tsdb, query); + } @Test public void uidGet() throws Exception { @@ -537,6 +745,15 @@ public void uidGetNSU() throws Exception { "/api/uid/uidmeta?type=metric&uid=000002"); rpc.execute(tsdb, query); } + + @Test (expected = BadRequestException.class) + public void uidPostReadOnlyMode() throws Exception { + rpc = new UniqueIdRpc(TSDB.OperationMode.READONLY); + setupUID(); + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/uid/uidmeta", + "{\"uid\":\"000001\",\"type\":\"metric\",\"displayName\":\"Hello!\"}"); + rpc.execute(tsdb, query); + } @Test public void uidPost() throws Exception { @@ -588,6 +805,15 @@ public void uidPostQS() throws Exception { rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); } + + @Test (expected = BadRequestException.class) + public void uidPutReadOnlyMode() throws Exception { + rpc = new UniqueIdRpc(TSDB.OperationMode.READONLY); + setupUID(); + HttpQuery query = NettyMocks.putQuery(tsdb, "/api/uid/uidmeta", + "{\"uid\":\"000001\",\"type\":\"metric\",\"displayName\":\"Hello!\"}"); + rpc.execute(tsdb, query); + } @Test public void uidPut() throws Exception { @@ -639,6 +865,15 @@ public void uidPutQS() throws Exception { rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); } + + @Test (expected = BadRequestException.class) + public void uidDeleteReadOnlyMode() throws Exception { + rpc = new UniqueIdRpc(TSDB.OperationMode.READONLY); + setupUID(); + HttpQuery query = NettyMocks.deleteQuery(tsdb, "/api/uid/uidmeta", + "{\"uid\":\"000001\",\"type\":\"metric\",\"displayName\":\"Hello!\"}"); + rpc.execute(tsdb, query); + } @Test public void uidDelete() throws Exception { @@ -675,6 +910,15 @@ public void uidDeleteQS() throws Exception { } // Test /api/uid/tsmeta ---------------------- + + @Test (expected = BadRequestException.class) + public void tsuidGetWriteOnlyMode() throws Exception { + rpc = new UniqueIdRpc(TSDB.OperationMode.WRITEONLY); + setupTSUID(); + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/uid/tsmeta?tsuid=000001000001000001"); + rpc.execute(tsdb, query); + } @Test public void tsuidGet() throws Exception { @@ -761,6 +1005,15 @@ public void tsuidGetMissingTSUID() throws Exception { rpc.execute(tsdb, query); } + @Test (expected = BadRequestException.class) + public void tsuidPostReadOnlyMode() throws Exception { + rpc = new UniqueIdRpc(TSDB.OperationMode.READONLY); + setupTSUID(); + HttpQuery query = NettyMocks.postQuery(tsdb, "/api/uid/tsmeta", + "{\"tsuid\":\"000001000001000001\", \"displayName\":\"Hello World\"}"); + rpc.execute(tsdb, query); + } + @Test public void tsuidPost() throws Exception { setupTSUID(); @@ -807,6 +1060,15 @@ public void tsuidPostQSNoTSUID() throws Exception { "/api/uid/tsmeta?display_name=42&method_override=post"); rpc.execute(tsdb, query); } + + @Test (expected = BadRequestException.class) + public void tsuidPutReadOnlyMode() throws Exception { + rpc = new UniqueIdRpc(TSDB.OperationMode.READONLY); + setupTSUID(); + HttpQuery query = NettyMocks.putQuery(tsdb, "/api/uid/tsmeta", + "{\"tsuid\":\"000001000001000001\", \"displayName\":\"Hello World\"}"); + rpc.execute(tsdb, query); + } @Test public void tsuidPut() throws Exception { @@ -854,6 +1116,15 @@ public void tsuidPutQSNoTSUID() throws Exception { "/api/uid/tsmeta?display_name=42&method_override=put"); rpc.execute(tsdb, query); } + + @Test (expected = BadRequestException.class) + public void tsuidDeleteReadOnlyMode() throws Exception { + rpc = new UniqueIdRpc(TSDB.OperationMode.READONLY); + setupTSUID(); + HttpQuery query = NettyMocks.deleteQuery(tsdb, "/api/uid/tsmeta", + "{\"tsuid\":\"000001000001000001\", \"displayName\":\"Hello World\"}"); + rpc.execute(tsdb, query); + } @Test public void tsuidDelete() throws Exception { @@ -911,23 +1182,21 @@ private void setupAssign() throws Exception { */ private void setupUID() throws Exception { final Config config = new Config(false); - PowerMockito.whenNew(HBaseClient.class) - .withArguments(anyString(), anyString()).thenReturn(client); - tsdb = new TSDB(config); + tsdb = new TSDB(client, config); storage = new MockBase(tsdb, client, true, true, true, true); - storage.addColumn(new byte[] { 0, 0, 1 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "metrics".getBytes(MockBase.ASCII()), "sys.cpu.0".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 3 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 3 }, NAME_FAMILY, "metrics".getBytes(MockBase.ASCII()), "sys.cpu.2".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "metric_meta".getBytes(MockBase.ASCII()), ("{\"uid\":\"000001\",\"type\":\"METRIC\",\"name\":\"sys.cpu.0\"," + @@ -942,9 +1211,7 @@ private void setupUID() throws Exception { */ private void setupTSUID() throws Exception { final Config config = new Config(false); - PowerMockito.whenNew(HBaseClient.class) - .withArguments(anyString(), anyString()).thenReturn(client); - tsdb = new TSDB(config); + tsdb = new TSDB(client, config); Field met = tsdb.getClass().getDeclaredField("metrics"); met.setAccessible(true); @@ -959,139 +1226,152 @@ private void setupTSUID() throws Exception { tagv.set(tsdb, tag_values); storage = new MockBase(tsdb, client, true, true, true, true); - storage.setFamily(NAME_FAMILY); + final List<byte[]> families = new ArrayList<byte[]>(1); + families.add(TSMeta.FAMILY); + storage.addTable(META_TABLE, families); - storage.addColumn(new byte[] { 0, 0, 1 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "metrics".getBytes(MockBase.ASCII()), "sys.cpu.0".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "metric_meta".getBytes(MockBase.ASCII()), ("{\"uid\":\"000001\",\"type\":\"METRIC\",\"name\":\"sys.cpu.0\"," + "\"description\":\"Description\",\"notes\":\"MyNotes\",\"created\":" + "1328140801,\"displayName\":\"System CPU\"}").getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 2 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 2 }, NAME_FAMILY, "metrics".getBytes(MockBase.ASCII()), "sys.cpu.2".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 2 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 2 }, NAME_FAMILY, "metric_meta".getBytes(MockBase.ASCII()), ("{\"uid\":\"000002\",\"type\":\"METRIC\",\"name\":\"sys.cpu.2\"," + "\"description\":\"Description\",\"notes\":\"MyNotes\",\"created\":" + "1328140801,\"displayName\":\"System CPU\"}").getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "tagk".getBytes(MockBase.ASCII()), "host".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "tagk_meta".getBytes(MockBase.ASCII()), ("{\"uid\":\"000001\",\"type\":\"TAGK\",\"name\":\"host\"," + "\"description\":\"Description\",\"notes\":\"MyNotes\",\"created\":" + "1328140801,\"displayName\":\"Host server name\"}").getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 2 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 2 }, NAME_FAMILY, "tagk".getBytes(MockBase.ASCII()), "datacenter".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 2 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 2 }, NAME_FAMILY, "tagk_meta".getBytes(MockBase.ASCII()), ("{\"uid\":\"000002\",\"type\":\"TAGK\",\"name\":\"datacenter\"," + "\"description\":\"Description\",\"notes\":\"MyNotes\",\"created\":" + "1328140801,\"displayName\":\"Host server name\"}").getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "tagv".getBytes(MockBase.ASCII()), "web01".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 1 }, NAME_FAMILY, "tagv_meta".getBytes(MockBase.ASCII()), ("{\"uid\":\"000001\",\"type\":\"TAGV\",\"name\":\"web01\"," + "\"description\":\"Description\",\"notes\":\"MyNotes\",\"created\":" + "1328140801,\"displayName\":\"Web server 1\"}").getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 3 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 3 }, NAME_FAMILY, "tagv".getBytes(MockBase.ASCII()), "web02".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 3 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 3 }, NAME_FAMILY, "tagv_meta".getBytes(MockBase.ASCII()), ("{\"uid\":\"000003\",\"type\":\"TAGV\",\"name\":\"web02\"," + "\"description\":\"Description\",\"notes\":\"MyNotes\",\"created\":" + "1328140801,\"displayName\":\"Web server 1\"}").getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 2 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 2 }, NAME_FAMILY, "tagv".getBytes(MockBase.ASCII()), "dc01".getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 2 }, + storage.addColumn(UID_TABLE, new byte[] { 0, 0, 2 }, NAME_FAMILY, "tagv_meta".getBytes(MockBase.ASCII()), ("{\"uid\":\"000002\",\"type\":\"TAGV\",\"name\":\"dc01\"," + "\"description\":\"Description\",\"notes\":\"MyNotes\",\"created\":" + "1328140801,\"displayName\":\"Web server 1\"}").getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }, - NAME_FAMILY, + storage.addColumn(META_TABLE, new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }, + TSMeta.FAMILY, "ts_meta".getBytes(MockBase.ASCII()), ("{\"tsuid\":\"000001000001000001\",\"displayName\":\"Display\"," + "\"description\":\"Description\",\"notes\":\"Notes\",\"created" + "\":1366671600,\"custom\":null,\"units\":\"\",\"dataType\":" + "\"Data\",\"retention\":42,\"max\":1.0,\"min\":\"NaN\"}") .getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }, - NAME_FAMILY, + storage.addColumn(META_TABLE, new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }, + TSMeta.FAMILY, "ts_ctr".getBytes(MockBase.ASCII()), Bytes.fromLong(1L)); - storage.addColumn(new byte[] { 0, 0, 2, 0, 0, 1, 0, 0, 1, 0, 0, 2, 0, 0, 2 }, - NAME_FAMILY, + storage.addColumn(META_TABLE, + new byte[] { 0, 0, 2, 0, 0, 1, 0, 0, 1, 0, 0, 2, 0, 0, 2 }, + TSMeta.FAMILY, "ts_meta".getBytes(MockBase.ASCII()), ("{\"tsuid\":\"000002000001000001000002000002\",\"displayName\":\"Display\"," + "\"description\":\"Description\",\"notes\":\"Notes\",\"created" + "\":1366671600,\"custom\":null,\"units\":\"\",\"dataType\":" + "\"Data\",\"retention\":42,\"max\":1.0,\"min\":\"NaN\"}") .getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 2, 0, 0, 1, 0, 0, 1, 0, 0, 2, 0, 0, 2 }, - NAME_FAMILY, + storage.addColumn(META_TABLE, + new byte[] { 0, 0, 2, 0, 0, 1, 0, 0, 1, 0, 0, 2, 0, 0, 2 }, + TSMeta.FAMILY, "ts_ctr".getBytes(MockBase.ASCII()), Bytes.fromLong(1L)); - storage.addColumn(new byte[] { 0, 0, 2, 0, 0, 1, 0, 0, 3, 0, 0, 2, 0, 0, 2 }, - NAME_FAMILY, + storage.addColumn(META_TABLE, + new byte[] { 0, 0, 2, 0, 0, 1, 0, 0, 3, 0, 0, 2, 0, 0, 2 }, + TSMeta.FAMILY, "ts_meta".getBytes(MockBase.ASCII()), ("{\"tsuid\":\"000002000001000003000002000002\",\"displayName\":\"Display\"," + "\"description\":\"Description\",\"notes\":\"Notes\",\"created" + "\":1366671600,\"custom\":null,\"units\":\"\",\"dataType\":" + "\"Data\",\"retention\":42,\"max\":1.0,\"min\":\"NaN\"}") .getBytes(MockBase.ASCII())); - storage.addColumn(new byte[] { 0, 0, 2, 0, 0, 1, 0, 0, 3, 0, 0, 2, 0, 0, 2 }, - NAME_FAMILY, + storage.addColumn(META_TABLE, + new byte[] { 0, 0, 2, 0, 0, 1, 0, 0, 3, 0, 0, 2, 0, 0, 2 }, + TSMeta.FAMILY, "ts_ctr".getBytes(MockBase.ASCII()), Bytes.fromLong(1L)); when(metrics.getId("sys.cpu.0")).thenReturn(new byte[] { 0, 0, 1 }); - when(metrics.getIdAsync("sys.cpu.0")).thenReturn(Deferred.fromResult(new byte[] { 0, 0, 1 })); + when(metrics.getIdAsync("sys.cpu.0")).thenReturn( + Deferred.fromResult(new byte[] { 0, 0, 1 })); when(metrics.getNameAsync(new byte[] { 0, 0, 1 })) .thenReturn(Deferred.fromResult("sys.cpu.0")); when(metrics.getId("sys.cpu.2")).thenReturn(new byte[] { 0, 0, 2 }); - when(metrics.getIdAsync("sys.cpu.2")).thenReturn(Deferred.fromResult(new byte[] { 0, 0, 2 })); + when(metrics.getIdAsync("sys.cpu.2")).thenReturn( + Deferred.fromResult(new byte[] { 0, 0, 2 })); when(metrics.getNameAsync(new byte[] { 0, 0, 2 })) .thenReturn(Deferred.fromResult("sys.cpu.2")); when(tag_names.getId("host")).thenReturn(new byte[] { 0, 0, 1 }); - when(tag_names.getIdAsync("host")).thenReturn(Deferred.fromResult(new byte[] { 0, 0, 1 })); + when(tag_names.getIdAsync("host")).thenReturn( + Deferred.fromResult(new byte[] { 0, 0, 1 })); when(tag_names.getNameAsync(new byte[] { 0, 0, 1 })) .thenReturn(Deferred.fromResult("host")); when(tag_values.getId("web01")).thenReturn(new byte[] { 0, 0, 1 }); - when(tag_values.getIdAsync("web01")).thenReturn(Deferred.fromResult(new byte[] { 0, 0, 1 })); + when(tag_values.getIdAsync("web01")).thenReturn( + Deferred.fromResult(new byte[] { 0, 0, 1 })); when(tag_values.getNameAsync(new byte[] { 0, 0, 1 })) .thenReturn(Deferred.fromResult("web01")); when(tag_values.getId("web02")).thenReturn(new byte[] { 0, 0, 3 }); - when(tag_values.getIdAsync("web02")).thenReturn(Deferred.fromResult(new byte[] { 0, 0, 3 })); + when(tag_values.getIdAsync("web02")).thenReturn( + Deferred.fromResult(new byte[] { 0, 0, 3 })); when(tag_values.getNameAsync(new byte[] { 0, 0, 3 })) .thenReturn(Deferred.fromResult("web02")); when(tag_names.getId("datacenter")).thenReturn(new byte[] { 0, 0, 2 }); - when(tag_names.getIdAsync("datacenter")).thenReturn(Deferred.fromResult(new byte[] { 0, 0, 2 })); + when(tag_names.getIdAsync("datacenter")).thenReturn( + Deferred.fromResult(new byte[] { 0, 0, 2 })); when(tag_names.getNameAsync(new byte[] { 0, 0, 2 })) .thenReturn(Deferred.fromResult("datacenter")); when(tag_values.getId("dc01")).thenReturn(new byte[] { 0, 0, 2 }); - when(tag_values.getIdAsync("dc01")).thenReturn(Deferred.fromResult(new byte[] { 0, 0, 2 })); + when(tag_values.getIdAsync("dc01")).thenReturn( + Deferred.fromResult(new byte[] { 0, 0, 2 })); when(tag_values.getNameAsync(new byte[] { 0, 0, 2 })) .thenReturn(Deferred.fromResult("dc01")); diff --git a/test/uid/TestRandomUniqueId.java b/test/uid/TestRandomUniqueId.java new file mode 100644 index 0000000000..a1fb6c2fbb --- /dev/null +++ b/test/uid/TestRandomUniqueId.java @@ -0,0 +1,110 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.uid; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import net.opentsdb.core.TSDB; + +import org.hbase.async.Bytes; +import org.junit.Test; + +public final class TestRandomUniqueId { + + @Test + public void getRandomUIDMetricWidth() throws Exception { + generateAndTestUID(TSDB.metrics_width(), 100); + } + + @Test + public void getRandomUID1Byte() throws Exception { + generateAndTestUID(1, 100); + } + + @Test + public void getRandomUID2Byte() throws Exception { + generateAndTestUID(2, 100); + } + + @Test + public void getRandomUID3Byte() throws Exception { + generateAndTestUID(3, 100); + } + + @Test + public void getRandomUID4Byte() throws Exception { + generateAndTestUID(4, 100); + } + + @Test + public void getRandomUID5Byte() throws Exception { + generateAndTestUID(5, 100); + } + + @Test + public void getRandomUID6Byte() throws Exception { + generateAndTestUID(6, 100); + } + + @Test + public void getRandomUID7Byte() throws Exception { + generateAndTestUID(7, 100); + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidWidth() { + RandomUniqueId.getRandomUID(8); + } + + @Test(expected = NegativeArraySizeException.class) + public void testNegativeWidth() { + RandomUniqueId.getRandomUID(-1); + } + + // if you pass in a width of 0 it will always return 1 + @Test + public void testZeroWidth() { + assertEquals(1L, RandomUniqueId.getRandomUID(0)); + } + + /** + * Runs the test n times and makes sure it's greater than 0 and less than or + * equal to the max value on {@link width} bytes. + * @param width The number of bytes to generate a UID for + * @param n How many times to run the tests + */ + private void generateAndTestUID(final int width, final int n) { + final long max_value = getMax(width); + for (int i = 0; i < n; i++) { + long uid = RandomUniqueId.getRandomUID(width); + assertTrue(uid > 0 && uid <= max_value); + } + } + + /** + * Simple helper to calculate the max value for any width of long + * @param width The width of the byte array we're comparing + * @return The maximum integer value on {@link width} bytes. + */ + private long getMax(final int width) { + if (width > 7) { + throw new IllegalArgumentException("Can't use a width of [" + width + + "] in this unit test"); + } + final byte[] value = new byte[8]; + for (int i = 0; i < width; i++) { + value[8 - (i + 1)] = (byte) 0xFF; + } + return Bytes.getLong(value); + } +} diff --git a/test/uid/TestUniqueId.java b/test/uid/TestUniqueId.java index 3db8dbe9e8..0ce2a1224f 100644 --- a/test/uid/TestUniqueId.java +++ b/test/uid/TestUniqueId.java @@ -20,31 +20,36 @@ import com.stumbleupon.async.Callback; import com.stumbleupon.async.Deferred; +import net.opentsdb.core.Const; import net.opentsdb.core.TSDB; +import net.opentsdb.core.TSDB.OperationMode; +import net.opentsdb.core.BaseTsdbTest.UnitTestException; +import net.opentsdb.storage.MockBase; +import net.opentsdb.uid.UniqueId.UniqueIdType; import net.opentsdb.utils.Config; import org.hbase.async.AtomicIncrementRequest; import org.hbase.async.Bytes; +import org.hbase.async.DeleteRequest; import org.hbase.async.GetRequest; import org.hbase.async.HBaseClient; import org.hbase.async.HBaseException; import org.hbase.async.KeyValue; import org.hbase.async.PutRequest; import org.hbase.async.Scanner; - +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.fail; import org.mockito.ArgumentMatcher; import org.mockito.InOrder; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import static org.mockito.Mockito.any; + +import static org.junit.Assert.*; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyMapOf; +import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.anyInt; import static org.mockito.Mockito.argThat; import static org.mockito.Mockito.eq; @@ -58,6 +63,7 @@ import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; + import static org.powermock.api.mockito.PowerMockito.mock; @RunWith(PowerMockRunner.class) @@ -66,24 +72,42 @@ @PowerMockIgnore({"javax.management.*", "javax.xml.*", "ch.qos.*", "org.slf4j.*", "com.sum.*", "org.xml.*"}) -@PrepareForTest({ HBaseClient.class, TSDB.class, Config.class }) +@PrepareForTest({ HBaseClient.class, TSDB.class, Config.class, Scanner.class, + RandomUniqueId.class, Const.class, Deferred.class }) public final class TestUniqueId { - - private HBaseClient client = mock(HBaseClient.class); - private static final byte[] table = { 't', 'a', 'b', 'l', 'e' }; + private static final byte[] table = { 't', 's', 'd', 'b', '-', 'u', 'i', 'd' }; private static final byte[] ID = { 'i', 'd' }; - private UniqueId uid; - private static final String kind = "metric"; - private static final byte[] kind_array = { 'm', 'e', 't', 'r', 'i', 'c' }; - + private static final byte[] NAME = { 'n', 'a', 'm', 'e' }; + private static final String METRIC = "metric"; + private static final byte[] METRIC_ARRAY = { 'm', 'e', 't', 'r', 'i', 'c' }; + private static final String TAGK = "tagk"; + private static final byte[] TAGK_ARRAY = { 't', 'a', 'g', 'k' }; + private static final String TAGV = "tagv"; + private static final byte[] TAGV_ARRAY = { 't', 'a', 'g', 'v' }; + private static final byte[] UID = new byte[] { 0, 0, 1 }; + private TSDB tsdb; + private Config config; + private HBaseClient client; + private UniqueId uid; + private MockBase storage; + + @Before + public void before() throws Exception { + tsdb = mock(TSDB.class); + client = mock(HBaseClient.class); + config = new Config(false); + when(tsdb.getClient()).thenReturn(client); + when(tsdb.getConfig()).thenReturn(config); + } + @Test(expected=IllegalArgumentException.class) public void testCtorZeroWidth() { - uid = new UniqueId(client, table, kind, 0); + uid = new UniqueId(client, table, METRIC, 0); } @Test(expected=IllegalArgumentException.class) public void testCtorNegativeWidth() { - uid = new UniqueId(client, table, kind, -1); + uid = new UniqueId(client, table, METRIC, -1); } @Test(expected=IllegalArgumentException.class) @@ -93,36 +117,29 @@ public void testCtorEmptyKind() { @Test(expected=IllegalArgumentException.class) public void testCtorLargeWidth() { - uid = new UniqueId(client, table, kind, 9); + uid = new UniqueId(client, table, METRIC, 9); } @Test public void kindEqual() { - uid = new UniqueId(client, table, kind, 3); - assertEquals(kind, uid.kind()); + uid = new UniqueId(client, table, METRIC, 3); + assertEquals(METRIC, uid.kind()); } @Test public void widthEqual() { - uid = new UniqueId(client, table, kind, 3); + uid = new UniqueId(client, table, METRIC, 3); assertEquals(3, uid.width()); } - - @Test - public void testMaxPossibleId() { - assertEquals(255, (new UniqueId(client, table, kind, 1)).maxPossibleId()); - assertEquals(65535, (new UniqueId(client, table, kind, 2)).maxPossibleId()); - assertEquals(16777215L, (new UniqueId(client, table, kind, 3)).maxPossibleId()); - } @Test public void getNameSuccessfulHBaseLookup() { - uid = new UniqueId(client, table, kind, 3); + uid = new UniqueId(client, table, METRIC, 3); final byte[] id = { 0, 'a', 0x42 }; final byte[] byte_name = { 'f', 'o', 'o' }; ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); - kvs.add(new KeyValue(id, ID, kind_array, byte_name)); + kvs.add(new KeyValue(id, ID, METRIC_ARRAY, byte_name)); when(client.get(anyGet())) .thenReturn(Deferred.fromResult(kvs)); @@ -140,14 +157,14 @@ public void getNameSuccessfulHBaseLookup() { @Test public void getNameWithErrorDuringHBaseLookup() { - uid = new UniqueId(client, table, kind, 3); + uid = new UniqueId(client, table, METRIC, 3); final byte[] id = { 0, 'a', 0x42 }; final byte[] byte_name = { 'f', 'o', 'o' }; HBaseException hbe = mock(HBaseException.class); ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); - kvs.add(new KeyValue(id, ID, kind_array, byte_name)); + kvs.add(new KeyValue(id, ID, METRIC_ARRAY, byte_name)); when(client.get(anyGet())) .thenThrow(hbe) .thenReturn(Deferred.fromResult(kvs)); @@ -172,7 +189,7 @@ public void getNameWithErrorDuringHBaseLookup() { @Test(expected=NoSuchUniqueId.class) public void getNameForNonexistentId() { - uid = new UniqueId(client, table, kind, 3); + uid = new UniqueId(client, table, METRIC, 3); when(client.get(anyGet())) .thenReturn(Deferred.fromResult(new ArrayList<KeyValue>(0))); @@ -182,19 +199,19 @@ public void getNameForNonexistentId() { @Test(expected=IllegalArgumentException.class) public void getNameWithInvalidId() { - uid = new UniqueId(client, table, kind, 3); + uid = new UniqueId(client, table, METRIC, 3); uid.getName(new byte[] { 1 }); } @Test public void getIdSuccessfulHBaseLookup() { - uid = new UniqueId(client, table, kind, 3); + uid = new UniqueId(client, table, METRIC, 3); final byte[] id = { 0, 'a', 0x42 }; final byte[] byte_name = { 'f', 'o', 'o' }; ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); - kvs.add(new KeyValue(byte_name, ID, kind_array, id)); + kvs.add(new KeyValue(byte_name, ID, METRIC_ARRAY, id)); when(client.get(anyGet())) .thenReturn(Deferred.fromResult(kvs)); @@ -215,12 +232,12 @@ public void getIdSuccessfulHBaseLookup() { // The table contains IDs encoded on 2 bytes but the instance wants 3. @Test(expected=IllegalStateException.class) public void getIdMisconfiguredWidth() { - uid = new UniqueId(client, table, kind, 3); + uid = new UniqueId(client, table, METRIC, 3); final byte[] id = { 'a', 0x42 }; final byte[] byte_name = { 'f', 'o', 'o' }; ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); - kvs.add(new KeyValue(byte_name, ID, kind_array, id)); + kvs.add(new KeyValue(byte_name, ID, METRIC_ARRAY, id)); when(client.get(anyGet())) .thenReturn(Deferred.fromResult(kvs)); @@ -229,7 +246,7 @@ public void getIdMisconfiguredWidth() { @Test(expected=NoSuchUniqueName.class) public void getIdForNonexistentName() { - uid = new UniqueId(client, table, kind, 3); + uid = new UniqueId(client, table, METRIC, 3); when(client.get(anyGet())) // null => ID doesn't exist. .thenReturn(Deferred.<ArrayList<KeyValue>>fromResult(null)); @@ -240,12 +257,12 @@ public void getIdForNonexistentName() { @Test public void getOrCreateIdWithExistingId() { - uid = new UniqueId(client, table, kind, 3); + uid = new UniqueId(client, table, METRIC, 3); final byte[] id = { 0, 'a', 0x42 }; final byte[] byte_name = { 'f', 'o', 'o' }; ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); - kvs.add(new KeyValue(byte_name, ID, kind_array, id)); + kvs.add(new KeyValue(byte_name, ID, METRIC_ARRAY, id)); when(client.get(anyGet())) .thenReturn(Deferred.fromResult(kvs)); @@ -261,14 +278,20 @@ public void getOrCreateIdWithExistingId() { } @Test // Test the creation of an ID with no problem. - public void getOrCreateIdAssignIdWithSuccess() { - uid = new UniqueId(client, table, kind, 3); + public void getOrCreateIdAssignFilterOK() { + uid = new UniqueId(client, table, METRIC, 3); final byte[] id = { 0, 0, 5 }; final Config config = mock(Config.class); when(config.enable_realtime_uid()).thenReturn(false); final TSDB tsdb = mock(TSDB.class); when(tsdb.getConfig()).thenReturn(config); uid.setTSDB(tsdb); + final UniqueIdFilterPlugin filter = mock(UniqueIdFilterPlugin.class); + when(filter.fillterUIDAssignments()).thenReturn(true); + when(filter.allowUIDAssignment(any(UniqueIdType.class), anyString(), + anyString(), anyMapOf(String.class, String.class))) + .thenReturn(Deferred.fromResult(true)); + when(tsdb.getUidFilter()).thenReturn(filter); when(client.get(anyGet())) // null => ID doesn't exist. .thenReturn(Deferred.<ArrayList<KeyValue>>fromResult(null)); @@ -291,14 +314,223 @@ public void getOrCreateIdAssignIdWithSuccess() { verify(client).atomicIncrement(incrementForRow(MAXID)); // Reverse + forward mappings. verify(client, times(2)).compareAndSet(anyPut(), emptyArray()); + verify(filter, times(1)).allowUIDAssignment(any(UniqueIdType.class), anyString(), + anyString(), anyMapOf(String.class, String.class)); + } + + @Test (expected = FailedToAssignUniqueIdException.class) + public void getOrCreateIdAssignFilterBlocked() { + uid = new UniqueId(client, table, METRIC, 3); + final Config config = mock(Config.class); + when(config.enable_realtime_uid()).thenReturn(false); + final TSDB tsdb = mock(TSDB.class); + when(tsdb.getConfig()).thenReturn(config); + uid.setTSDB(tsdb); + final UniqueIdFilterPlugin filter = mock(UniqueIdFilterPlugin.class); + when(filter.fillterUIDAssignments()).thenReturn(true); + when(filter.allowUIDAssignment(any(UniqueIdType.class), anyString(), + anyString(), anyMapOf(String.class, String.class))) + .thenReturn(Deferred.fromResult(false)); + when(tsdb.getUidFilter()).thenReturn(filter); + + when(client.get(anyGet())) // null => ID doesn't exist. + .thenReturn(Deferred.<ArrayList<KeyValue>>fromResult(null)); + // Watch this! ______,^ I'm writing C++ in Java! + + when(client.atomicIncrement(incrementForRow(MAXID))) + .thenReturn(Deferred.fromResult(5L)); + + when(client.compareAndSet(anyPut(), emptyArray())) + .thenReturn(Deferred.fromResult(true)) + .thenReturn(Deferred.fromResult(true)); + + uid.getOrCreateId("foo"); + } + + @Test(expected = RuntimeException.class) + public void getOrCreateIdAssignFilterReturnException() { + uid = new UniqueId(client, table, METRIC, 3); + final Config config = mock(Config.class); + when(config.enable_realtime_uid()).thenReturn(false); + final TSDB tsdb = mock(TSDB.class); + when(tsdb.getConfig()).thenReturn(config); + uid.setTSDB(tsdb); + final UniqueIdFilterPlugin filter = mock(UniqueIdFilterPlugin.class); + when(filter.fillterUIDAssignments()).thenReturn(true); + when(filter.allowUIDAssignment(any(UniqueIdType.class), anyString(), + anyString(), anyMapOf(String.class, String.class))) + .thenReturn(Deferred.<Boolean>fromError(new UnitTestException())); + when(tsdb.getUidFilter()).thenReturn(filter); + + when(client.get(anyGet())) // null => ID doesn't exist. + .thenReturn(Deferred.<ArrayList<KeyValue>>fromResult(null)); + // Watch this! ______,^ I'm writing C++ in Java! + + when(client.atomicIncrement(incrementForRow(MAXID))) + .thenReturn(Deferred.fromResult(5L)); + + when(client.compareAndSet(anyPut(), emptyArray())) + .thenReturn(Deferred.fromResult(true)) + .thenReturn(Deferred.fromResult(true)); + + uid.getOrCreateId("foo"); } + + @Test(expected = RuntimeException.class) + public void getOrCreateIdAssignFilterThrowsException() { + uid = new UniqueId(client, table, METRIC, 3); + final Config config = mock(Config.class); + when(config.enable_realtime_uid()).thenReturn(false); + final TSDB tsdb = mock(TSDB.class); + when(tsdb.getConfig()).thenReturn(config); + uid.setTSDB(tsdb); + final UniqueIdFilterPlugin filter = mock(UniqueIdFilterPlugin.class); + when(filter.fillterUIDAssignments()).thenReturn(true); + when(filter.allowUIDAssignment(any(UniqueIdType.class), anyString(), + anyString(), anyMapOf(String.class, String.class))) + .thenThrow(new UnitTestException()); + when(tsdb.getUidFilter()).thenReturn(filter); + + when(client.get(anyGet())) // null => ID doesn't exist. + .thenReturn(Deferred.<ArrayList<KeyValue>>fromResult(null)); + // Watch this! ______,^ I'm writing C++ in Java! - @PrepareForTest({HBaseClient.class, UniqueId.class}) + when(client.atomicIncrement(incrementForRow(MAXID))) + .thenReturn(Deferred.fromResult(5L)); + + when(client.compareAndSet(anyPut(), emptyArray())) + .thenReturn(Deferred.fromResult(true)) + .thenReturn(Deferred.fromResult(true)); + + uid.getOrCreateId("foo"); + } + + @Test + public void getOrCreateIdAsyncAssignFilterOK() throws Exception { + uid = new UniqueId(client, table, METRIC, 3); + final byte[] id = { 0, 0, 5 }; + final Config config = mock(Config.class); + when(config.enable_realtime_uid()).thenReturn(false); + final TSDB tsdb = mock(TSDB.class); + when(tsdb.getConfig()).thenReturn(config); + uid.setTSDB(tsdb); + final UniqueIdFilterPlugin filter = mock(UniqueIdFilterPlugin.class); + when(filter.fillterUIDAssignments()).thenReturn(true); + when(filter.allowUIDAssignment(any(UniqueIdType.class), anyString(), + anyString(), anyMapOf(String.class, String.class))) + .thenReturn(Deferred.fromResult(true)); + when(tsdb.getUidFilter()).thenReturn(filter); + when(client.get(anyGet())) // null => ID doesn't exist. + .thenReturn(Deferred.<ArrayList<KeyValue>>fromResult(null)); + // Watch this! ______,^ I'm writing C++ in Java! + + when(client.atomicIncrement(incrementForRow(MAXID))) + .thenReturn(Deferred.fromResult(5L)); + + when(client.compareAndSet(anyPut(), emptyArray())) + .thenReturn(Deferred.fromResult(true)) + .thenReturn(Deferred.fromResult(true)); + + assertArrayEquals(id, uid.getOrCreateIdAsync("foo").join()); + // Should be a cache hit since we created that entry. + assertArrayEquals(id, uid.getOrCreateIdAsync("foo").join()); + // Should be a cache hit too for the same reason. + assertEquals("foo", uid.getName(id)); + + verify(client).get(anyGet()); // Initial Get. + verify(client).atomicIncrement(incrementForRow(MAXID)); + // Reverse + forward mappings. + verify(client, times(2)).compareAndSet(anyPut(), emptyArray()); + verify(filter, times(1)).allowUIDAssignment(any(UniqueIdType.class), anyString(), + anyString(), anyMapOf(String.class, String.class)); + } + + @Test (expected = FailedToAssignUniqueIdException.class) + public void getOrCreateIdAsyncAssignFilterBlocked() throws Exception { + uid = new UniqueId(client, table, METRIC, 3); + final Config config = mock(Config.class); + when(config.enable_realtime_uid()).thenReturn(false); + final TSDB tsdb = mock(TSDB.class); + when(tsdb.getConfig()).thenReturn(config); + uid.setTSDB(tsdb); + final UniqueIdFilterPlugin filter = mock(UniqueIdFilterPlugin.class); + when(filter.fillterUIDAssignments()).thenReturn(true); + when(filter.allowUIDAssignment(any(UniqueIdType.class), anyString(), + anyString(), anyMapOf(String.class, String.class))) + .thenReturn(Deferred.fromResult(false)); + when(tsdb.getUidFilter()).thenReturn(filter); + when(client.get(anyGet())) // null => ID doesn't exist. + .thenReturn(Deferred.<ArrayList<KeyValue>>fromResult(null)); + + when(client.atomicIncrement(incrementForRow(MAXID))) + .thenReturn(Deferred.fromResult(5L)); + + when(client.compareAndSet(anyPut(), emptyArray())) + .thenReturn(Deferred.fromResult(true)) + .thenReturn(Deferred.fromResult(true)); + + uid.getOrCreateIdAsync("foo").join(); + } + + @Test (expected = UnitTestException.class) + public void getOrCreateIdAsyncAssignFilterReturnException() throws Exception { + uid = new UniqueId(client, table, METRIC, 3); + final Config config = mock(Config.class); + when(config.enable_realtime_uid()).thenReturn(false); + final TSDB tsdb = mock(TSDB.class); + when(tsdb.getConfig()).thenReturn(config); + uid.setTSDB(tsdb); + final UniqueIdFilterPlugin filter = mock(UniqueIdFilterPlugin.class); + when(filter.fillterUIDAssignments()).thenReturn(true); + when(filter.allowUIDAssignment(any(UniqueIdType.class), anyString(), + anyString(), anyMapOf(String.class, String.class))) + .thenReturn(Deferred.<Boolean>fromError(new UnitTestException())); + when(tsdb.getUidFilter()).thenReturn(filter); + when(client.get(anyGet())) // null => ID doesn't exist. + .thenReturn(Deferred.<ArrayList<KeyValue>>fromResult(null)); + + when(client.atomicIncrement(incrementForRow(MAXID))) + .thenReturn(Deferred.fromResult(5L)); + + when(client.compareAndSet(anyPut(), emptyArray())) + .thenReturn(Deferred.fromResult(true)) + .thenReturn(Deferred.fromResult(true)); + + uid.getOrCreateIdAsync("foo").join(); + } + + @Test (expected = UnitTestException.class) + public void getOrCreateIdAsyncAssignFilterThrowsException() throws Exception { + uid = new UniqueId(client, table, METRIC, 3); + final Config config = mock(Config.class); + when(config.enable_realtime_uid()).thenReturn(false); + final TSDB tsdb = mock(TSDB.class); + when(tsdb.getConfig()).thenReturn(config); + uid.setTSDB(tsdb); + final UniqueIdFilterPlugin filter = mock(UniqueIdFilterPlugin.class); + when(filter.fillterUIDAssignments()).thenReturn(true); + when(filter.allowUIDAssignment(any(UniqueIdType.class), anyString(), + anyString(),anyMapOf(String.class, String.class))) + .thenThrow(new UnitTestException()); + when(tsdb.getUidFilter()).thenReturn(filter); + when(client.get(anyGet())) // null => ID doesn't exist. + .thenReturn(Deferred.<ArrayList<KeyValue>>fromResult(null)); + + when(client.atomicIncrement(incrementForRow(MAXID))) + .thenReturn(Deferred.fromResult(5L)); + + when(client.compareAndSet(anyPut(), emptyArray())) + .thenReturn(Deferred.fromResult(true)) + .thenReturn(Deferred.fromResult(true)); + + uid.getOrCreateIdAsync("foo").join(); + } + @Test // Test the creation of an ID when unable to increment MAXID public void getOrCreateIdUnableToIncrementMaxId() throws Exception { PowerMockito.mockStatic(Thread.class); - uid = new UniqueId(client, table, kind, 3); + uid = new UniqueId(client, table, METRIC, 3); when(client.get(anyGet())) // null => ID doesn't exist. .thenReturn(Deferred.<ArrayList<KeyValue>>fromResult(null)); @@ -318,8 +550,7 @@ public void getOrCreateIdUnableToIncrementMaxId() throws Exception { } @Test // Test the creation of an ID with a race condition. - @PrepareForTest({HBaseClient.class, Deferred.class}) - public void getOrCreateIdAssignIdWithRaceCondition() { + public void getOrCreateIdAssignIdWithRaceCondition() { // Simulate a race between client A and client B. // A does a Get and sees that there's no ID for this name. // B does a Get and sees that there's no ID too, and B actually goes @@ -327,17 +558,17 @@ public void getOrCreateIdAssignIdWithRaceCondition() { // Then A attempts to go through the process and should discover that the // ID has already been assigned. - uid = new UniqueId(client, table, kind, 3); // Used by client A. + uid = new UniqueId(client, table, METRIC, 3); // Used by client A. HBaseClient client_b = mock(HBaseClient.class); // For client B. - final UniqueId uid_b = new UniqueId(client_b, table, kind, 3); + final UniqueId uid_b = new UniqueId(client_b, table, METRIC, 3); final byte[] id = { 0, 0, 5 }; final byte[] byte_name = { 'f', 'o', 'o' }; final ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); - kvs.add(new KeyValue(byte_name, ID, kind_array, id)); - - @SuppressWarnings("unchecked") - final Deferred<ArrayList<KeyValue>> d = PowerMockito.spy(new Deferred<ArrayList<KeyValue>>()); + kvs.add(new KeyValue(byte_name, ID, METRIC_ARRAY, id)); + + final Deferred<ArrayList<KeyValue>> d = + PowerMockito.spy(new Deferred<ArrayList<KeyValue>>()); when(client.get(anyGet())) .thenReturn(d) .thenReturn(Deferred.fromResult(kvs)); @@ -398,7 +629,7 @@ public byte[] answer(final InvocationOnMock unused_invocation) throws Exception @Test // Test the creation of an ID when all possible IDs are already in use public void getOrCreateIdWithOverflow() { - uid = new UniqueId(client, table, kind, 1); // IDs are only on 1 byte. + uid = new UniqueId(client, table, METRIC, 1); // IDs are only on 1 byte. when(client.get(anyGet())) // null => ID doesn't exist. .thenReturn(Deferred.<ArrayList<KeyValue>>fromResult(null)); @@ -422,7 +653,7 @@ public void getOrCreateIdWithOverflow() { @Test // ICV throws an exception, we can't get an ID. public void getOrCreateIdWithICVFailure() { - uid = new UniqueId(client, table, kind, 3); + uid = new UniqueId(client, table, METRIC, 3); final Config config = mock(Config.class); when(config.enable_realtime_uid()).thenReturn(false); final TSDB tsdb = mock(TSDB.class); @@ -454,7 +685,7 @@ public void getOrCreateIdWithICVFailure() { @Test // Test that the reverse mapping is created before the forward one. public void getOrCreateIdPutsReverseMappingFirst() { - uid = new UniqueId(client, table, kind, 3); + uid = new UniqueId(client, table, METRIC, 3); final Config config = mock(Config.class); when(config.enable_realtime_uid()).thenReturn(false); final TSDB tsdb = mock(TSDB.class); @@ -482,11 +713,135 @@ public void getOrCreateIdPutsReverseMappingFirst() { order.verify(client).compareAndSet(putForRow(id), emptyArray()); order.verify(client).compareAndSet(putForRow(row), emptyArray()); } + + @Test + public void getOrCreateIdRandom() { + PowerMockito.mockStatic(RandomUniqueId.class); + uid = new UniqueId(client, table, METRIC, 3, true); + final long id = 42L; + final byte[] id_array = { 0, 0, 0x2A }; + + when(RandomUniqueId.getRandomUID()).thenReturn(id); + when(client.get(any(GetRequest.class))) + .thenReturn(Deferred.<ArrayList<KeyValue>>fromResult(null)); + + when(client.compareAndSet(any(PutRequest.class), any(byte[].class))) + .thenReturn(Deferred.fromResult(true)) + .thenReturn(Deferred.fromResult(true)); + + assertArrayEquals(id_array, uid.getOrCreateId("foo")); + // Should be a cache hit ... + assertArrayEquals(id_array, uid.getOrCreateId("foo")); + assertEquals(1, uid.cacheHits()); + assertEquals(1, uid.cacheMisses()); + assertEquals(2, uid.cacheSize()); + assertEquals(0, uid.randomIdCollisions()); + // ... so verify there was only one HBase Get. + verify(client).get(any(GetRequest.class)); + } + + @Test + public void getOrCreateIdRandomCollision() { + PowerMockito.mockStatic(RandomUniqueId.class); + uid = new UniqueId(client, table, METRIC, 3, true); + final long id = 42L; + final byte[] id_array = { 0, 0, 0x2A }; + + when(RandomUniqueId.getRandomUID()).thenReturn(24L).thenReturn(id); + + when(client.get(any(GetRequest.class))) + .thenReturn(Deferred.fromResult((ArrayList<KeyValue>)null)); + + when(client.compareAndSet(anyPut(), any(byte[].class))) + .thenReturn(Deferred.fromResult(false)) + .thenReturn(Deferred.fromResult(true)) + .thenReturn(Deferred.fromResult(true)); + + assertArrayEquals(id_array, uid.getOrCreateId("foo")); + // Should be a cache hit ... + assertArrayEquals(id_array, uid.getOrCreateId("foo")); + assertEquals(1, uid.cacheHits()); + assertEquals(1, uid.cacheMisses()); + assertEquals(2, uid.cacheSize()); + assertEquals(1, uid.randomIdCollisions()); + + // ... so verify there was only one HBase Get. + verify(client).get(anyGet()); + } + + @Test + public void getOrCreateIdRandomCollisionTooManyAttempts() { + PowerMockito.mockStatic(RandomUniqueId.class); + uid = new UniqueId(client, table, METRIC, 3, true); + final long id = 42L; + + when(RandomUniqueId.getRandomUID()).thenReturn(24L).thenReturn(id); + + when(client.get(any(GetRequest.class))) + .thenReturn(Deferred.fromResult((ArrayList<KeyValue>)null)); + + when(client.compareAndSet(any(PutRequest.class), any(byte[].class))) + .thenReturn(Deferred.fromResult(false)) + .thenReturn(Deferred.fromResult(false)) + .thenReturn(Deferred.fromResult(false)) + .thenReturn(Deferred.fromResult(false)) + .thenReturn(Deferred.fromResult(false)) + .thenReturn(Deferred.fromResult(false)) + .thenReturn(Deferred.fromResult(false)) + .thenReturn(Deferred.fromResult(false)) + .thenReturn(Deferred.fromResult(false)) + .thenReturn(Deferred.fromResult(false)); + + try { + final byte[] assigned_id = uid.getOrCreateId("foo"); + fail("FailedToAssignUniqueIdException should have been thrown but instead " + + " this was returned id=" + Arrays.toString(assigned_id)); + } catch (FailedToAssignUniqueIdException e) { + // OK + } + assertEquals(0, uid.cacheHits()); + assertEquals(1, uid.cacheMisses()); + assertEquals(0, uid.cacheSize()); + assertEquals(9, uid.randomIdCollisions()); + + // ... so verify there was only one HBase Get. + verify(client).get(any(GetRequest.class)); + } + + @Test + public void getOrCreateIdRandomWithRaceCondition() { + PowerMockito.mockStatic(RandomUniqueId.class); + uid = new UniqueId(client, table, METRIC, 3, true); + final long id = 24L; + final byte[] id_array = { 0, 0, 0x2A }; + final byte[] byte_name = { 'f', 'o', 'o' }; + + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); + kvs.add(new KeyValue(byte_name, ID, METRIC_ARRAY, id_array)); + + when(RandomUniqueId.getRandomUID()).thenReturn(id); + + when(client.get(any(GetRequest.class))) + .thenReturn(Deferred.fromResult((ArrayList<KeyValue>)null)) + .thenReturn(Deferred.fromResult(kvs)); + + when(client.compareAndSet(any(PutRequest.class), any(byte[].class))) + .thenReturn(Deferred.fromResult(true)) + .thenReturn(Deferred.fromResult(false)); + + assertArrayEquals(id_array, uid.getOrCreateId("foo")); + assertEquals(0, uid.cacheHits()); + assertEquals(2, uid.cacheMisses()); + assertEquals(2, uid.cacheSize()); + assertEquals(1, uid.randomIdCollisions()); - @PrepareForTest({HBaseClient.class, Scanner.class}) + // ... so verify there was only one HBase Get. + verify(client, times(2)).get(any(GetRequest.class)); + } + @Test public void suggestWithNoMatch() { - uid = new UniqueId(client, table, kind, 3); + uid = new UniqueId(client, table, METRIC, 3); final Scanner fake_scanner = mock(Scanner.class); when(client.newScanner(table)) @@ -502,13 +857,12 @@ public void suggestWithNoMatch() { verify(fake_scanner).setStartKey("nomatch".getBytes()); verify(fake_scanner).setStopKey("nomatci".getBytes()); verify(fake_scanner).setFamily(ID); - verify(fake_scanner).setQualifier(kind_array); + verify(fake_scanner).setQualifier(METRIC_ARRAY); } - - @PrepareForTest({HBaseClient.class, Scanner.class}) + @Test public void suggestWithMatches() { - uid = new UniqueId(client, table, kind, 3); + uid = new UniqueId(client, table, METRIC, 3); final Scanner fake_scanner = mock(Scanner.class); when(client.newScanner(table)) @@ -518,10 +872,10 @@ public void suggestWithMatches() { final byte[] foo_bar_id = { 0, 0, 1 }; { ArrayList<KeyValue> row = new ArrayList<KeyValue>(1); - row.add(new KeyValue("foo.bar".getBytes(), ID, kind_array, foo_bar_id)); + row.add(new KeyValue("foo.bar".getBytes(), ID, METRIC_ARRAY, foo_bar_id)); rows.add(row); row = new ArrayList<KeyValue>(1); - row.add(new KeyValue("foo.baz".getBytes(), ID, kind_array, + row.add(new KeyValue("foo.baz".getBytes(), ID, METRIC_ARRAY, new byte[] { 0, 0, 2 })); rows.add(row); } @@ -634,11 +988,57 @@ public void getTSUIDFromKey() { } @Test + public void getTSUIDFromKeySalted() { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + + final byte[] expected = { 0, 0, 1, 0, 0, 2, 0, 0, 3 }; + byte[] tsuid = UniqueId.getTSUIDFromKey(new byte[] + { 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 2, 0, 0, 3 }, (short)3, (short)4); + assertArrayEquals(expected, tsuid); + + tsuid = UniqueId.getTSUIDFromKey(new byte[] + { 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 2, 0, 0, 3 }, (short)3, (short)4); + assertArrayEquals(expected, tsuid); + + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(4); + tsuid = UniqueId.getTSUIDFromKey(new byte[] + { 1, 2, 3, 4, 0, 0, 1, 1, 1, 1, 1, 0, 0, 2, 0, 0, 3 }, (short)3, (short)4); + assertArrayEquals(expected, tsuid); + + tsuid = UniqueId.getTSUIDFromKey(new byte[] + { 4, 3, 2, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 2, 0, 0, 3 }, (short)3, (short)4); + assertArrayEquals(expected, tsuid); + } + + @Test (expected = IllegalArgumentException.class) public void getTSUIDFromKeyMissingTags() { - final byte[] tsuid = UniqueId.getTSUIDFromKey(new byte[] + UniqueId.getTSUIDFromKey(new byte[] { 0, 0, 1, 1, 1, 1, 1 }, (short)3, (short)4); - assertArrayEquals(new byte[] { 0, 0, 1 }, - tsuid); + } + + @Test (expected = IllegalArgumentException.class) + public void getTSUIDFromKeyMissingTagsSalted() { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + + UniqueId.getTSUIDFromKey(new byte[] + { 0, 0, 0, 1, 1, 1, 1, 1 }, (short)3, (short)4); + } + + @Test (expected = IllegalArgumentException.class) + public void getTSUIDFromKeyMissingSalt() { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + + UniqueId.getTSUIDFromKey(new byte[] + { 0, 0, 1, 1, 1, 1, 1, 0, 0, 2, 0, 0, 3 }, (short)3, (short)4); + } + + @Test (expected = IllegalArgumentException.class) + public void getTSUIDFromKeySaltButShouldntBe() { + UniqueId.getTSUIDFromKey(new byte[] + { 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 2, 0, 0, 3 }, (short)3, (short)4); } @Test @@ -651,7 +1051,6 @@ public void getTagPairsFromTSUIDString() { assertArrayEquals(new byte[] { 0, 0, 3, 0, 0, 4 }, tags.get(1)); } - @Test public void getTagPairsFromTSUIDStringNonStandardWidth() { PowerMockito.mockStatic(TSDB.class); @@ -707,7 +1106,6 @@ public void getTagPairsFromTSUIDBytes() { assertArrayEquals(new byte[] { 0, 0, 3, 0, 0, 4 }, tags.get(1)); } - @Test public void getTagPairsFromTSUIDBytesNonStandardWidth() { PowerMockito.mockStatic(TSDB.class); @@ -890,10 +1288,457 @@ public void longToUIDTooBig() throws Exception { UniqueId.longToUID(257, (short)1); } + @Test + public void rename() throws Exception { + uid = new UniqueId(client, table, METRIC, 3); + final byte[] foo_id = { 0, 'a', 0x42 }; + final byte[] foo_name = { 'f', 'o', 'o' }; + + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); + kvs.add(new KeyValue(foo_name, ID, METRIC_ARRAY, foo_id)); + when(client.get(anyGet())) + .thenReturn(Deferred.fromResult(kvs)) + .thenReturn(Deferred.<ArrayList<KeyValue>>fromResult(null)); + when(client.put(anyPut())).thenAnswer(answerTrue()); + when(client.delete(anyDelete())).thenAnswer(answerTrue()); + + uid.rename("foo", "bar"); + } + + @Test (expected = IllegalArgumentException.class) + public void renameNewNameExists() throws Exception { + uid = new UniqueId(client, table, METRIC, 3); + final byte[] foo_id = { 0, 'a', 0x42 }; + final byte[] foo_name = { 'f', 'o', 'o' }; + final byte[] bar_id = { 1, 'b', 0x43 }; + final byte[] bar_name = { 'b', 'a', 'r' }; + + ArrayList<KeyValue> foo_kvs = new ArrayList<KeyValue>(1); + ArrayList<KeyValue> bar_kvs = new ArrayList<KeyValue>(1); + foo_kvs.add(new KeyValue(foo_name, ID, METRIC_ARRAY, foo_id)); + bar_kvs.add(new KeyValue(bar_name, ID, METRIC_ARRAY, bar_id)); + when(client.get(anyGet())) + .thenReturn(Deferred.fromResult(foo_kvs)) + .thenReturn(Deferred.fromResult(bar_kvs)); + when(client.put(anyPut())).thenAnswer(answerTrue()); + when(client.delete(anyDelete())).thenAnswer(answerTrue()); + + uid.rename("foo", "bar"); + } + + @Test (expected = IllegalStateException.class) + public void renameRaceCondition() throws Exception { + // Simulate a race between client A(default) and client B. + // A and B rename same UID to different name. + // B waits till A start to invoke PutRequest to start. + + uid = new UniqueId(client, table, METRIC, 3); + HBaseClient client_b = mock(HBaseClient.class); + final UniqueId uid_b = new UniqueId(client_b, table, METRIC, 3); + + final byte[] foo_id = { 0, 'a', 0x42 }; + final byte[] foo_name = { 'f', 'o', 'o' }; + + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); + kvs.add(new KeyValue(foo_name, ID, METRIC_ARRAY, foo_id)); + + when(client_b.get(anyGet())) + .thenReturn(Deferred.fromResult(kvs)) + .thenReturn(Deferred.<ArrayList<KeyValue>>fromResult(null)); + when(client_b.put(anyPut())).thenAnswer(answerTrue()); + when(client_b.delete(anyDelete())).thenAnswer(answerTrue()); + + final Answer<Deferred<Boolean>> the_race = new Answer<Deferred<Boolean>>() { + public Deferred<Boolean> answer(final InvocationOnMock inv) throws Exception { + uid_b.rename("foo", "xyz"); + return Deferred.fromResult(true); + } + }; + + when(client.get(anyGet())) + .thenReturn(Deferred.fromResult(kvs)) + .thenReturn(Deferred.<ArrayList<KeyValue>>fromResult(null)); + when(client.put(anyPut())).thenAnswer(the_race); + when(client.delete(anyDelete())).thenAnswer(answerTrue()); + + uid.rename("foo", "bar"); + } + + @Test + public void deleteCached() throws Exception { + setupStorage(); + uid = new UniqueId(client, table, METRIC, 3); + uid.setTSDB(tsdb); + assertArrayEquals(UID, uid.getId("sys.cpu.user")); + assertEquals("sys.cpu.user", uid.getName(UID)); + + uid.deleteAsync("sys.cpu.user").join(); + try { + uid.getId("sys.cpu.user"); + fail("Expected a NoSuchUniqueName"); + } catch (NoSuchUniqueName nsun) { } + + try { + uid.getName(UID); + fail("Expected a NoSuchUniqueId"); + } catch (NoSuchUniqueId nsui) { } + + uid = new UniqueId(client, table, TAGK, 3); + uid.setTSDB(tsdb); + assertArrayEquals(UID, uid.getId("host")); + assertEquals("host", uid.getName(UID)); + + uid = new UniqueId(client, table, TAGV, 3); + uid.setTSDB(tsdb); + assertArrayEquals(UID, uid.getId("web01")); + assertEquals("web01", uid.getName(UID)); + } + + @Test + public void deleteNotCached() throws Exception { + setupStorage(); + uid = new UniqueId(client, table, METRIC, 3); + uid.setTSDB(tsdb); + uid.deleteAsync("sys.cpu.user").join(); + try { + uid.getId("sys.cpu.user"); + fail("Expected a NoSuchUniqueName"); + } catch (NoSuchUniqueName nsun) { } + + try { + uid.getName(UID); + fail("Expected a NoSuchUniqueId"); + } catch (NoSuchUniqueId nsui) { } + + uid = new UniqueId(client, table, TAGK, 3); + uid.setTSDB(tsdb); + assertArrayEquals(UID, uid.getId("host")); + assertEquals("host", uid.getName(UID)); + + uid = new UniqueId(client, table, TAGV, 3); + uid.setTSDB(tsdb); + assertArrayEquals(UID, uid.getId("web01")); + assertEquals("web01", uid.getName(UID)); + } + + @Test + public void deleteFailForwardDelete() throws Exception { + setupStorage(); + uid = new UniqueId(client, table, METRIC, 3); + uid.setTSDB(tsdb); + assertArrayEquals(UID, uid.getId("sys.cpu.user")); + assertEquals("sys.cpu.user", uid.getName(UID)); + + storage.throwException("sys.cpu.user".getBytes(), fakeHBaseException()); + try { + uid.deleteAsync("sys.cpu.user").join(); + fail("Expected HBaseException"); + } catch (HBaseException e) { } + catch (Exception e) { } + storage.clearExceptions(); + try { + uid.getName(UID); + fail("Expected a NoSuchUniqueId"); + } catch (NoSuchUniqueId nsui) { } + assertArrayEquals(UID, uid.getId("sys.cpu.user")); + // now it pollutes the cache + assertEquals("sys.cpu.user", uid.getName(UID)); + } + + @Test + public void deleteFailReverseDelete() throws Exception { + setupStorage(); + storage.throwException(UID, fakeHBaseException()); + uid = new UniqueId(client, table, METRIC, 3); + uid.setTSDB(tsdb); + try { + uid.deleteAsync("sys.cpu.user").join(); + fail("Expected HBaseException"); + } catch (HBaseException e) { } + catch (Exception e) { } + storage.clearExceptions(); + try { + uid.getId("sys.cpu.user"); + fail("Expected a NoSuchUniqueName"); + } catch (NoSuchUniqueName nsun) { } + assertEquals("sys.cpu.user", uid.getName(UID)); + } + + @Test + public void deleteNoSuchUniqueName() throws Exception { + setupStorage(); + uid = new UniqueId(client, table, METRIC, 3); + uid.setTSDB(tsdb); + storage.flushRow(table, "sys.cpu.user".getBytes()); + try { + uid.deleteAsync("sys.cpu.user").join(); + fail("Expected NoSuchUniqueName"); + } catch (NoSuchUniqueName e) { } + assertEquals("sys.cpu.user", uid.getName(UID)); + } + + @Test + public void useLru() throws Exception { + config.overrideConfig("tsd.uid.lru.enable", "true"); + uid = new UniqueId(tsdb, table, METRIC, 3, false); + final byte[] id = { 0, 'a', 0x42 }; + final byte[] byte_name = { 'f', 'o', 'o' }; + + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); + kvs.add(new KeyValue(id, ID, METRIC_ARRAY, byte_name)); + when(client.get(anyGet())) + .thenReturn(Deferred.fromResult(kvs)); + + assertEquals("foo", uid.getName(id)); + // Should be a cache hit ... + assertEquals("foo", uid.getName(id)); + + assertEquals(1, uid.cacheHits()); + assertEquals(1, uid.cacheMisses()); + assertEquals(2, uid.cacheSize()); + + // ... so verify there was only one HBase Get. + verify(client).get(anyGet()); + assertNotNull(uid.lruNameCache()); + assertNotNull(uid.lruIdCache()); + } + + @Test + public void useLruLimit() throws Exception { + config.overrideConfig("tsd.uid.lru.name.size", "2"); + config.overrideConfig("tsd.uid.lru.id.size", "2"); + config.overrideConfig("tsd.uid.lru.enable", "true"); + uid = new UniqueId(tsdb, table, METRIC, 3, false); + byte[] id = { 0, 'a', 0x42 }; + byte[] byte_name = { 'f', 'o', 'o' }; + + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); + kvs.add(new KeyValue(id, ID, METRIC_ARRAY, byte_name)); + when(client.get(anyGet())) + .thenReturn(Deferred.fromResult(kvs)); + + assertEquals("foo", uid.getName(id)); + // Should be a cache hit ... + assertEquals("foo", uid.getName(id)); + + assertEquals(1, uid.cacheHits()); + assertEquals(1, uid.cacheMisses()); + assertEquals(2, uid.cacheSize()); + + // ... so verify there was only one HBase Get. + verify(client).get(anyGet()); + + id = new byte[] { 0, 0, 1 }; + byte_name = new byte[] { 'b', 'a', 'r' }; + + kvs = new ArrayList<KeyValue>(1); + kvs.add(new KeyValue(id, ID, METRIC_ARRAY, byte_name)); + when(client.get(anyGet())) + .thenReturn(Deferred.fromResult(kvs)); + + assertEquals("bar", uid.getName(id)); + assertEquals(1, uid.cacheHits()); + assertEquals(2, uid.cacheMisses()); + assertEquals(4, uid.cacheSize()); + verify(client, times(2)).get(anyGet()); + + // now one should be bumped out + id = new byte[] { 0, 0, 2 }; + byte_name = new byte[] { 'd', 'o', 'g' }; + + kvs = new ArrayList<KeyValue>(1); + kvs.add(new KeyValue(id, ID, METRIC_ARRAY, byte_name)); + when(client.get(anyGet())) + .thenReturn(Deferred.fromResult(kvs)); + + assertEquals("dog", uid.getName(id)); + assertEquals(1, uid.cacheHits()); + assertEquals(3, uid.cacheMisses()); + assertEquals(4, uid.cacheSize()); + verify(client, times(3)).get(anyGet()); + } + + @Test + public void useModeRWGetName() throws Exception { + when(tsdb.getMode()).thenReturn(OperationMode.READWRITE); + config.overrideConfig("tsd.uid.use_mode", "true"); + uid = new UniqueId(tsdb, table, METRIC, 3, false); + final byte[] id = { 0, 'a', 0x42 }; + final byte[] byte_name = { 'f', 'o', 'o' }; + + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); + kvs.add(new KeyValue(id, ID, METRIC_ARRAY, byte_name)); + when(client.get(anyGet())) + .thenReturn(Deferred.fromResult(kvs)); + + assertEquals("foo", uid.getName(id)); + // Should be a cache hit ... + assertEquals("foo", uid.getName(id)); + + assertEquals(1, uid.cacheHits()); + assertEquals(1, uid.cacheMisses()); + assertEquals(2, uid.cacheSize()); + + // ... so verify there was only one HBase Get. + verify(client).get(anyGet()); + } + + @Test + public void useModeROGetName() throws Exception { + when(tsdb.getMode()).thenReturn(OperationMode.READONLY); + config.overrideConfig("tsd.uid.use_mode", "true"); + uid = new UniqueId(tsdb, table, METRIC, 3, false); + final byte[] id = { 0, 'a', 0x42 }; + final byte[] byte_name = { 'f', 'o', 'o' }; + + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); + kvs.add(new KeyValue(id, ID, METRIC_ARRAY, byte_name)); + when(client.get(anyGet())) + .thenReturn(Deferred.fromResult(kvs)); + + assertEquals("foo", uid.getName(id)); + // Should be a cache hit ... + assertEquals("foo", uid.getName(id)); + + assertEquals(1, uid.cacheHits()); + assertEquals(1, uid.cacheMisses()); + assertEquals(1, uid.cacheSize()); + + // ... so verify there was only one HBase Get. + verify(client).get(anyGet()); + assertTrue(uid.nameCache().isEmpty()); + assertEquals(1, uid.idCache().size()); + } + + @Test + public void useModeWOGetName() throws Exception { + when(tsdb.getMode()).thenReturn(OperationMode.WRITEONLY); + config.overrideConfig("tsd.uid.use_mode", "true"); + uid = new UniqueId(tsdb, table, METRIC, 3, false); + final byte[] id = { 0, 'a', 0x42 }; + final byte[] byte_name = { 'f', 'o', 'o' }; + + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); + kvs.add(new KeyValue(id, ID, METRIC_ARRAY, byte_name)); + when(client.get(anyGet())) + .thenReturn(Deferred.fromResult(kvs)) + .thenReturn(Deferred.fromResult(kvs)); + + assertEquals("foo", uid.getName(id)); + // NOT a cache hit since we didn't cache the first result. + assertEquals("foo", uid.getName(id)); + + assertEquals(0, uid.cacheHits()); + assertEquals(2, uid.cacheMisses()); + assertEquals(0, uid.cacheSize()); + + // 2 hbase hits this time + verify(client, times(2)).get(anyGet()); + assertTrue(uid.nameCache().isEmpty()); + assertTrue(uid.idCache().isEmpty()); + } + + @Test + public void useModeRWGetId() throws Exception { + when(tsdb.getMode()).thenReturn(OperationMode.READWRITE); + config.overrideConfig("tsd.uid.use_mode", "true"); + uid = new UniqueId(tsdb, table, METRIC, 3, false); + final byte[] id = { 0, 'a', 0x42 }; + final byte[] byte_name = { 'f', 'o', 'o' }; + + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); + kvs.add(new KeyValue(byte_name, ID, METRIC_ARRAY, id)); + when(client.get(anyGet())) + .thenReturn(Deferred.fromResult(kvs)); + + assertArrayEquals(id, uid.getId("foo")); + // Should be a cache hit ... + assertArrayEquals(id, uid.getId("foo")); + + assertEquals(1, uid.cacheHits()); + assertEquals(1, uid.cacheMisses()); + assertEquals(2, uid.cacheSize()); + + // ... so verify there was only one HBase Get. + verify(client).get(anyGet()); + } + + @Test + public void useModeROGetId() throws Exception { + when(tsdb.getMode()).thenReturn(OperationMode.READONLY); + config.overrideConfig("tsd.uid.use_mode", "true"); + uid = new UniqueId(tsdb, table, METRIC, 3, false); + final byte[] id = { 0, 'a', 0x42 }; + final byte[] byte_name = { 'f', 'o', 'o' }; + + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); + kvs.add(new KeyValue(byte_name, ID, METRIC_ARRAY, id)); + when(client.get(anyGet())) + .thenReturn(Deferred.fromResult(kvs)) + .thenReturn(Deferred.fromResult(kvs)); + + assertArrayEquals(id, uid.getId("foo")); + // NOT a cache hit since we didn't cache the first time. + assertArrayEquals(id, uid.getId("foo")); + + assertEquals(0, uid.cacheHits()); + assertEquals(2, uid.cacheMisses()); + assertEquals(0, uid.cacheSize()); + + // two HBase hits + verify(client, times(2)).get(anyGet()); + assertTrue(uid.nameCache().isEmpty()); + assertTrue(uid.idCache().isEmpty()); + } + + @Test + public void useModeWOGetId() throws Exception { + when(tsdb.getMode()).thenReturn(OperationMode.WRITEONLY); + config.overrideConfig("tsd.uid.use_mode", "true"); + uid = new UniqueId(tsdb, table, METRIC, 3, false); + final byte[] id = { 0, 'a', 0x42 }; + final byte[] byte_name = { 'f', 'o', 'o' }; + + ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1); + kvs.add(new KeyValue(byte_name, ID, METRIC_ARRAY, id)); + when(client.get(anyGet())) + .thenReturn(Deferred.fromResult(kvs)) + .thenReturn(Deferred.fromResult(kvs)); + + assertArrayEquals(id, uid.getId("foo")); + // Should be a cache hit ... + assertArrayEquals(id, uid.getId("foo")); + + assertEquals(1, uid.cacheHits()); + assertEquals(1, uid.cacheMisses()); + assertEquals(1, uid.cacheSize()); + + // ... so verify there was only one HBase Get. + verify(client, times(1)).get(anyGet()); + assertEquals(1, uid.nameCache().size()); + assertTrue(uid.idCache().isEmpty()); + } + // ----------------- // // Helper functions. // // ----------------- // + private void setupStorage() throws Exception { + storage = new MockBase(tsdb, client, true, true, true, true); + + final List<byte[]> families = new ArrayList<byte[]>(); + families.add(ID); + families.add(NAME); + storage.addTable(table, families); + + storage.addColumn(table, "sys.cpu.user".getBytes(), ID, METRIC_ARRAY, UID); + storage.addColumn(table, UID, NAME, METRIC_ARRAY, "sys.cpu.user".getBytes()); + storage.addColumn(table, "host".getBytes(), ID, TAGK_ARRAY, UID); + storage.addColumn(table, UID, NAME, TAGK_ARRAY, "host".getBytes()); + storage.addColumn(table, "web01".getBytes(), ID, TAGV_ARRAY, UID); + storage.addColumn(table, UID, NAME,TAGV_ARRAY, "web01".getBytes()); + } + private static byte[] emptyArray() { return eq(HBaseClient.EMPTY_ARRAY); } @@ -918,6 +1763,18 @@ private static PutRequest anyPut() { return any(PutRequest.class); } + private static DeleteRequest anyDelete() { + return any(DeleteRequest.class); + } + + private static Answer<Deferred<Boolean>> answerTrue() { + return new Answer<Deferred<Boolean>>() { + public Deferred<Boolean> answer(final InvocationOnMock inv) { + return Deferred.fromResult(true); + } + }; + } + @SuppressWarnings("unchecked") private static Callback<byte[], ArrayList<KeyValue>> anyByteCB() { return any(Callback.class); diff --git a/test/uid/TestUniqueIdWhitelistFilter.java b/test/uid/TestUniqueIdWhitelistFilter.java new file mode 100644 index 0000000000..c07f9c26e2 --- /dev/null +++ b/test/uid/TestUniqueIdWhitelistFilter.java @@ -0,0 +1,165 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.uid; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import net.opentsdb.core.TSDB; +import net.opentsdb.uid.UniqueId.UniqueIdType; +import net.opentsdb.utils.Config; + +@RunWith(PowerMockRunner.class) +//"Classloader hell"... It's real. Tell PowerMock to ignore these classes +//because they fiddle with the class loader. We don't test them anyway. +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", "com.sum.*", "org.xml.*"}) +@PrepareForTest({ TSDB.class, Config.class }) +public class TestUniqueIdWhitelistFilter { + + private TSDB tsdb; + private Config config; + private UniqueIdWhitelistFilter filter; + + @Before + public void before() throws Exception { + tsdb = PowerMockito.mock(TSDB.class); + config = new Config(false); + when(tsdb.getConfig()).thenReturn(config); + filter = new UniqueIdWhitelistFilter(); + + config.overrideConfig("tsd.uidfilter.whitelist.metric_patterns", ".*"); + config.overrideConfig("tsd.uidfilter.whitelist.tagk_patterns", ".*"); + config.overrideConfig("tsd.uidfilter.whitelist.tagv_patterns", ".*"); + } + + @Test + public void ctor() throws Exception { + assertNull(filter.metricPatterns()); + assertNull(filter.tagkPatterns()); + assertNull(filter.tagvPatterns()); + } + + @Test + public void initalize() throws Exception { + filter.initialize(tsdb); + assertEquals(1, filter.metricPatterns().size()); + assertEquals(".*", filter.metricPatterns().get(0).pattern()); + assertEquals(1, filter.tagkPatterns().size()); + assertEquals(".*", filter.tagkPatterns().get(0).pattern()); + assertEquals(1, filter.tagvPatterns().size()); + assertEquals(".*", filter.tagvPatterns().get(0).pattern()); + } + + @Test + public void initalizeMultiplePatterns() throws Exception { + config.overrideConfig("tsd.uidfilter.whitelist.metric_patterns", ".*,^test.*"); + config.overrideConfig("tsd.uidfilter.whitelist.tagk_patterns", ".*,^test.*"); + config.overrideConfig("tsd.uidfilter.whitelist.tagv_patterns", ".*,^test.*"); + filter.initialize(tsdb); + assertEquals(2, filter.metricPatterns().size()); + assertEquals(".*", filter.metricPatterns().get(0).pattern()); + assertEquals("^test.*", filter.metricPatterns().get(1).pattern()); + assertEquals(2, filter.tagkPatterns().size()); + assertEquals(".*", filter.tagkPatterns().get(0).pattern()); + assertEquals("^test.*", filter.tagkPatterns().get(1).pattern()); + assertEquals(2, filter.tagvPatterns().size()); + assertEquals(".*", filter.tagvPatterns().get(0).pattern()); + assertEquals("^test.*", filter.tagvPatterns().get(1).pattern()); + } + + @Test + public void initalizeMultiplePatternsAlternateDelimiter() throws Exception { + config.overrideConfig("tsd.uidfilter.whitelist.delimiter", "\\|"); + config.overrideConfig("tsd.uidfilter.whitelist.metric_patterns", ".*|^test.*"); + config.overrideConfig("tsd.uidfilter.whitelist.tagk_patterns", ".*|^test.*"); + config.overrideConfig("tsd.uidfilter.whitelist.tagv_patterns", ".*|^test.*"); + filter.initialize(tsdb); + assertEquals(2, filter.metricPatterns().size()); + assertEquals(".*", filter.metricPatterns().get(0).pattern()); + assertEquals("^test.*", filter.metricPatterns().get(1).pattern()); + assertEquals(2, filter.tagkPatterns().size()); + assertEquals(".*", filter.tagkPatterns().get(0).pattern()); + assertEquals("^test.*", filter.tagkPatterns().get(1).pattern()); + assertEquals(2, filter.tagvPatterns().size()); + assertEquals(".*", filter.tagvPatterns().get(0).pattern()); + assertEquals("^test.*", filter.tagvPatterns().get(1).pattern()); + } + + @Test (expected = IllegalArgumentException.class) + public void initalizeBadRegex() throws Exception { + config.overrideConfig("tsd.uidfilter.whitelist.metric_patterns", "grp[start"); + filter.initialize(tsdb); + } + + @Test + public void shutdown() throws Exception { + assertNull(filter.shutdown().join()); + } + + @Test + public void allowUIDAssignment() throws Exception { + config.overrideConfig("tsd.uidfilter.whitelist.metric_patterns", "^test.*"); + config.overrideConfig("tsd.uidfilter.whitelist.tagk_patterns", "^test.*"); + config.overrideConfig("tsd.uidfilter.whitelist.tagv_patterns", "^test.*"); + filter.initialize(tsdb); + assertTrue(filter.allowUIDAssignment(UniqueIdType.METRIC, "test_metric", + null, null).join()); + assertFalse(filter.allowUIDAssignment(UniqueIdType.METRIC, "metric", + null, null).join()); + assertTrue(filter.allowUIDAssignment(UniqueIdType.TAGK, "test_tagk", + null, null).join()); + assertFalse(filter.allowUIDAssignment(UniqueIdType.TAGK, "tagk", + null, null).join()); + assertTrue(filter.allowUIDAssignment(UniqueIdType.TAGV, "test_tagv", + null, null).join()); + assertFalse(filter.allowUIDAssignment(UniqueIdType.TAGV, "tagv", + null, null).join()); + } + + @Test + public void allowUIDAssignmentMultiplePaterns() throws Exception { + config.overrideConfig("tsd.uidfilter.whitelist.metric_patterns", ".*,^test.*"); + config.overrideConfig("tsd.uidfilter.whitelist.tagk_patterns", ".*,^test.*"); + config.overrideConfig("tsd.uidfilter.whitelist.tagv_patterns", ".*,^test.*"); + filter.initialize(tsdb); + assertTrue(filter.allowUIDAssignment(UniqueIdType.METRIC, "test_metric", + null, null).join()); + assertFalse(filter.allowUIDAssignment(UniqueIdType.METRIC, "metric", + null, null).join()); + assertTrue(filter.allowUIDAssignment(UniqueIdType.TAGK, "test_tagk", + null, null).join()); + assertFalse(filter.allowUIDAssignment(UniqueIdType.TAGK, "tagk", + null, null).join()); + assertTrue(filter.allowUIDAssignment(UniqueIdType.TAGV, "test_tagv", + null, null).join()); + assertFalse(filter.allowUIDAssignment(UniqueIdType.TAGV, "tagv", + null, null).join()); + } + + @Test + public void fillterUIDAssignments() throws Exception { + assertTrue(filter.fillterUIDAssignments()); + } +} diff --git a/test/utils/TestByteSet.java b/test/utils/TestByteSet.java new file mode 100644 index 0000000000..5cd83a240c --- /dev/null +++ b/test/utils/TestByteSet.java @@ -0,0 +1,71 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.utils; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.util.Iterator; + +import org.junit.Test; + +public class TestByteSet { + + private static final byte[] V1 = new byte[] { 0, 0, 1 }; + private static final byte[] V2 = new byte[] { 0, 0, 2 }; + private static final byte[] V3 = new byte[] { 0, 0, 3 }; + private static final byte[] V4 = new byte[] { 0, 0, 4 }; + + @Test + public void ctor() { + final ByteSet set = new ByteSet(); + assertNotNull(set); + assertEquals(0, set.size()); + assertTrue(set.isEmpty()); + } + + @Test + public void goodOperations() { + final ByteSet set = new ByteSet(); + set.add(V3); + set.add(V2); + set.add(V1); + + assertEquals(3, set.size()); + assertFalse(set.isEmpty()); + + // should come out in order + final Iterator<byte[]> it = set.iterator(); + assertArrayEquals(V1, it.next()); + assertArrayEquals(V2, it.next()); + assertArrayEquals(V3, it.next()); + assertFalse(it.hasNext()); + + assertEquals("[[0, 0, 1],[0, 0, 2],[0, 0, 3]]", set.toString()); + + assertTrue(set.contains(V1)); + assertFalse(set.contains(V4)); + + assertTrue(set.remove(V1)); + assertFalse(set.contains(V1)); + assertFalse(set.remove(V4)); + + set.clear(); + assertFalse(set.contains(V2)); + assertFalse(set.contains(V3)); + assertTrue(set.isEmpty()); + } +} diff --git a/test/utils/TestConfig.java b/test/utils/TestConfig.java index 189f81f742..604777d1cb 100644 --- a/test/utils/TestConfig.java +++ b/test/utils/TestConfig.java @@ -584,7 +584,7 @@ public void getDirectoryNameWindowsOnLinuxException() throws Exception { } } - @Test (expected = NullPointerException.class) + @Test public void getDirectoryNameNull() throws Exception { final Config config = new Config(false); assertNull(config.getDirectoryName("tsd.unitest")); diff --git a/test/utils/TestDateTime.java b/test/utils/TestDateTime.java index 1e72b34ccd..cee9866487 100644 --- a/test/utils/TestDateTime.java +++ b/test/utils/TestDateTime.java @@ -17,22 +17,44 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.mockito.Mockito.when; import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Locale; import java.util.TimeZone; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +//"Classloader hell"... It's real. Tell PowerMock to ignore these classes +//because they fiddle with the class loader. We don't test them anyway. +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", + "com.sum.*", "org.xml.*"}) @RunWith(PowerMockRunner.class) -@PrepareForTest({ DateTime.class }) +@PrepareForTest({ DateTime.class, System.class }) public final class TestDateTime { + //30 minute offset + final static TimeZone AF = DateTime.timezones.get("Asia/Kabul"); + // 45 minute offset w DST + final static TimeZone NZ = DateTime.timezones.get("Pacific/Chatham"); + // 12h offset w/o DST + final static TimeZone TV = DateTime.timezones.get("Pacific/Funafuti"); + // 12h offset w DST + final static TimeZone FJ = DateTime.timezones.get("Pacific/Fiji"); + // Fri, 15 May 2015 14:21:13.432 UTC + final static long NON_DST_TS = 1431699673432L; + // Tue, 15 Dec 2015 04:02:25.123 UTC + final static long DST_TS = 1450152145123L; + @Before public void before() { PowerMockito.mockStatic(System.class); @@ -49,6 +71,12 @@ public void getTimezoneNull() { assertNull(DateTime.timezones.get("Nothere")); } + @Test + public void parseDateTimeStringNow() { + long t = DateTime.parseDateTimeString("now", null); + assertEquals(t, 1357300800000L); + } + @Test public void parseDateTimeStringRelativeS() { long t = DateTime.parseDateTimeString("60s-ago", null); @@ -122,6 +150,29 @@ public void parseDateTimeStringUnixSecondsZero() { public void parseDateTimeStringUnixSecondsNegative() { DateTime.parseDateTimeString("-135596160", null); } + + /* + 1234567890.418 - match + 1234567890.1235 - no match + 1234567890.12 - matches + 1234.56789.003 - no match + 1234567890.3 - match + */ + + @Test(expected = IllegalArgumentException.class) + public void parseDateTimeStringMultipleDots() { + DateTime.parseDateTimeString("1234567890.2.4", null); + } + + @Test(expected = IllegalArgumentException.class) + public void parseDateTimeStringMultipleDotsEarlyDot() { + DateTime.parseDateTimeString("1234.56789.123", null); + } + + @Test(expected = IllegalArgumentException.class) + public void parseDateTimeStringEarlyandExtraDots() { + DateTime.parseDateTimeString("1234.56789.0.3", null); + } @Test public void parseDateTimeStringUnixSecondsInvalidLong() { @@ -135,12 +186,36 @@ public void parseDateTimeStringUnixMS() { long t = DateTime.parseDateTimeString("1355961603418", null); assertEquals(1355961603418L, t); } + + @Test + public void parseDateTimeStringShortExplicitMS() { + long t = DateTime.parseDateTimeString("123123ms", null); + assertEquals(123123L, t); + } + + @Test + public void parseDateTimeStringExplicitMS() { + long t = DateTime.parseDateTimeString("1234567890123ms", null); + assertEquals(1234567890123L, t); + } @Test public void parseDateTimeStringUnixMSDot() { long t = DateTime.parseDateTimeString("1355961603.418", null); assertEquals(1355961603418L, t); } + + @Test + public void parseDateTimeStringUnixMSDotShorter() { + long t = DateTime.parseDateTimeString("1355961603.41", null); + assertEquals(135596160341L, t); + } + + @Test + public void parseDateTimeStringUnixMSDotShortest() { + long t = DateTime.parseDateTimeString("1355961603.4", null); + assertEquals(13559616034L, t); + } @Test (expected = IllegalArgumentException.class) public void parseDateTimeStringUnixMSDotInvalid() { @@ -296,6 +371,97 @@ public void parseDurationTooBig() { DateTime.parseDuration("6393590450230209347573980s"); } + @Test + public void getDurationUnits() { + assertEquals("ms", DateTime.getDurationUnits("5ms")); + assertEquals("s", DateTime.getDurationUnits("30s")); + assertEquals("m", DateTime.getDurationUnits("60m")); + assertEquals("h", DateTime.getDurationUnits("42h")); + assertEquals("d", DateTime.getDurationUnits("9d")); + assertEquals("w", DateTime.getDurationUnits("4w")); + assertEquals("n", DateTime.getDurationUnits("12n")); + assertEquals("y", DateTime.getDurationUnits("4y")); + + // zeros ok + assertEquals("d", DateTime.getDurationUnits("0d")); + + // biggies + assertEquals("s", DateTime.getDurationUnits("86400s")); + assertEquals("s", DateTime.getDurationUnits("8641816584387483775236188168735700s")); + } + + @Test (expected = IllegalArgumentException.class) + public void getDurationUnitsNegative() { + DateTime.getDurationUnits("-1d"); + } + + @Test (expected = IllegalArgumentException.class) + public void getDurationUnitsFloat() { + DateTime.getDurationUnits("0.42d"); + } + + @Test (expected = IllegalArgumentException.class) + public void getDurationUnitsNoUnits() { + DateTime.getDurationUnits("42"); + } + + @Test (expected = IllegalArgumentException.class) + public void getDurationUnitsNull() { + DateTime.getDurationUnits(null); + } + + @Test (expected = IllegalArgumentException.class) + public void getDurationUnitsEmpty() { + DateTime.getDurationUnits(""); + } + + @Test (expected = IllegalArgumentException.class) + public void getDurationIsNull() { + DateTime.getDurationUnits(null); + } + + @Test + public void getDurationInterval() { + assertEquals(1, DateTime.getDurationInterval("1s")); + assertEquals(1, DateTime.getDurationInterval("1ms")); + assertEquals(42, DateTime.getDurationInterval("42d")); + assertEquals(86400, DateTime.getDurationInterval("86400s")); + assertEquals(Integer.MAX_VALUE, DateTime.getDurationInterval( + Integer.MAX_VALUE + "s")); + } + + @Test (expected = IllegalArgumentException.class) + public void getDurationIntervalTooBig() { + long value = Integer.MAX_VALUE; + value++; + DateTime.getDurationInterval(Long.toString(value) + "s"); + } + + @Test (expected = IllegalArgumentException.class) + public void getDurationIntervalNegative() { + DateTime.getDurationInterval("-1s"); + } + + @Test (expected = IllegalArgumentException.class) + public void getDurationIntervalFloat() { + DateTime.getDurationInterval("1.42s"); + } + + @Test (expected = IllegalArgumentException.class) + public void getDurationIntervalNoInt() { + DateTime.getDurationInterval("s"); + } + + @Test (expected = IllegalArgumentException.class) + public void getDurationIntervalNull() { + DateTime.getDurationInterval(null); + } + + @Test (expected = IllegalArgumentException.class) + public void getDurationIntervalEmpty() { + DateTime.getDurationInterval(""); + } + @Test public void setTimeZone() { SimpleDateFormat fmt = new SimpleDateFormat("yyyy/MM/dd"); @@ -364,4 +530,511 @@ public void setDefaultTimezone() { public void setDefaultTimezoneNull() { DateTime.setDefaultTimezone(null); } + + @Test + public void currentTimeMillis() { + PowerMockito.mockStatic(System.class); + when(System.currentTimeMillis()).thenReturn(1388534400000L); + assertEquals(1388534400000L, DateTime.currentTimeMillis()); + } + + @Test + public void nanoTime() { + PowerMockito.mockStatic(System.class); + when(System.nanoTime()).thenReturn(1388534400000000000L); + assertEquals(1388534400000000000L, DateTime.nanoTime()); + } + + @Test + public void previousIntervalMilliseconds() { + // interval 1 + assertEquals(DST_TS, DateTime.previousInterval(DST_TS, + 1, Calendar.MILLISECOND).getTimeInMillis()); + assertEquals(NON_DST_TS, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.MILLISECOND).getTimeInMillis()); + + // interval 100 + assertEquals(1450152145100L, DateTime.previousInterval(DST_TS, + 100, Calendar.MILLISECOND).getTimeInMillis()); + assertEquals(1450152145000L, DateTime.previousInterval(1450152145000L, + 100, Calendar.MILLISECOND).getTimeInMillis()); + + // odd interval + assertEquals(1450152144769L, DateTime.previousInterval(DST_TS, + 799, Calendar.MILLISECOND).getTimeInMillis()); + + // TZs - all the same for ms + assertEquals(1450152145100L, DateTime.previousInterval(DST_TS, + 100, Calendar.MILLISECOND, AF).getTimeInMillis()); + assertEquals(1431699673400L, DateTime.previousInterval(NON_DST_TS, + 100, Calendar.MILLISECOND, AF).getTimeInMillis()); + assertEquals(1450152145100L, DateTime.previousInterval(DST_TS, + 100, Calendar.MILLISECOND, NZ).getTimeInMillis()); + assertEquals(1431699673400L, DateTime.previousInterval(NON_DST_TS, + 100, Calendar.MILLISECOND, NZ).getTimeInMillis()); + assertEquals(1450152145100L, DateTime.previousInterval(DST_TS, + 100, Calendar.MILLISECOND, TV).getTimeInMillis()); + assertEquals(1431699673400L, DateTime.previousInterval(NON_DST_TS, + 100, Calendar.MILLISECOND, TV).getTimeInMillis()); + assertEquals(1450152145100L, DateTime.previousInterval(DST_TS, + 100, Calendar.MILLISECOND, FJ).getTimeInMillis()); + assertEquals(1431699673400L, DateTime.previousInterval(NON_DST_TS, + 100, Calendar.MILLISECOND, FJ).getTimeInMillis()); + + // multiples + assertEquals(1450152120000L, DateTime.previousInterval(DST_TS, + 60000, Calendar.MILLISECOND).getTimeInMillis()); + assertEquals(1431699660000L, DateTime.previousInterval(NON_DST_TS, + 60000, Calendar.MILLISECOND).getTimeInMillis()); + } + + @Test + public void previousIntervalSeconds() { + // interval 1 + assertEquals(1450152145000L, DateTime.previousInterval(DST_TS, + 1, Calendar.SECOND).getTimeInMillis()); + assertEquals(1431699673000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.SECOND).getTimeInMillis()); + + // interval 30 + assertEquals(1450152120000L, DateTime.previousInterval(DST_TS, + 30, Calendar.SECOND).getTimeInMillis()); + assertEquals(1431699660000L, DateTime.previousInterval(NON_DST_TS, + 30, Calendar.SECOND).getTimeInMillis()); + assertEquals(1450152120000L, DateTime.previousInterval(1450152120000L, + 30, Calendar.SECOND).getTimeInMillis()); + + // odd interval + assertEquals(1431699647000L, DateTime.previousInterval(NON_DST_TS, + 29, Calendar.SECOND).getTimeInMillis()); + assertEquals(1450152145000L, DateTime.previousInterval(DST_TS, + 29, Calendar.SECOND).getTimeInMillis()); + + // TZs - all the same for seconds + assertEquals(1450152120000L, DateTime.previousInterval(DST_TS, + 30, Calendar.SECOND, AF).getTimeInMillis()); + assertEquals(1431699660000L, DateTime.previousInterval(NON_DST_TS, + 30, Calendar.SECOND, AF).getTimeInMillis()); + assertEquals(1450152120000L, DateTime.previousInterval(DST_TS, + 30, Calendar.SECOND, NZ).getTimeInMillis()); + assertEquals(1431699660000L, DateTime.previousInterval(NON_DST_TS, + 30, Calendar.SECOND, NZ).getTimeInMillis()); + assertEquals(1450152120000L, DateTime.previousInterval(DST_TS, + 30, Calendar.SECOND, TV).getTimeInMillis()); + assertEquals(1431699660000L, DateTime.previousInterval(NON_DST_TS, + 30, Calendar.SECOND, TV).getTimeInMillis()); + assertEquals(1450152120000L, DateTime.previousInterval(DST_TS, + 30, Calendar.SECOND, FJ).getTimeInMillis()); + assertEquals(1431699660000L, DateTime.previousInterval(NON_DST_TS, + 30, Calendar.SECOND, FJ).getTimeInMillis()); + + // multiples + assertEquals(1450152000000L, DateTime.previousInterval(DST_TS, + 60000, Calendar.SECOND).getTimeInMillis()); + assertEquals(1431698400000L, DateTime.previousInterval(NON_DST_TS, + 60000, Calendar.SECOND).getTimeInMillis()); + } + + @Test + public void previousIntervalMinutes() { + // interval 1 + assertEquals(1450152120000L, DateTime.previousInterval(DST_TS, + 1, Calendar.MINUTE).getTimeInMillis()); + assertEquals(1431699660000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.MINUTE).getTimeInMillis()); + + // interval 30 + assertEquals(1450152000000L, DateTime.previousInterval(DST_TS, + 30, Calendar.MINUTE).getTimeInMillis()); + assertEquals(1431698400000L, DateTime.previousInterval(NON_DST_TS, + 30, Calendar.MINUTE).getTimeInMillis()); + assertEquals(1431698400000L, DateTime.previousInterval(1431698400000L, + 30, Calendar.MINUTE).getTimeInMillis()); + + // odd interval + assertEquals(1431698460000L, DateTime.previousInterval(NON_DST_TS, + 29, Calendar.MINUTE).getTimeInMillis()); + assertEquals(1450151520000L, DateTime.previousInterval(DST_TS, + 29, Calendar.MINUTE).getTimeInMillis()); + + // TZs + assertEquals(1450152000000L, DateTime.previousInterval(DST_TS, + 30, Calendar.MINUTE, AF).getTimeInMillis()); + assertEquals(1431698400000L, DateTime.previousInterval(NON_DST_TS, + 30, Calendar.MINUTE, AF).getTimeInMillis()); + // 15 min diff + assertEquals(1450152000000L, DateTime.previousInterval(DST_TS, + 15, Calendar.MINUTE, AF).getTimeInMillis()); + assertEquals(1431699300000L, DateTime.previousInterval(NON_DST_TS, + 15, Calendar.MINUTE, AF).getTimeInMillis()); + // outliers @ 45 minutes + assertEquals(1450151100000L, DateTime.previousInterval(DST_TS, + 30, Calendar.MINUTE, NZ).getTimeInMillis()); + assertEquals(1431699300000L, DateTime.previousInterval(NON_DST_TS, + 30, Calendar.MINUTE, NZ).getTimeInMillis()); + // back to normal + assertEquals(1450152000000L, DateTime.previousInterval(DST_TS, + 30, Calendar.MINUTE, TV).getTimeInMillis()); + assertEquals(1431698400000L, DateTime.previousInterval(NON_DST_TS, + 30, Calendar.MINUTE, TV).getTimeInMillis()); + assertEquals(1450152000000L, DateTime.previousInterval(DST_TS, + 30, Calendar.MINUTE, FJ).getTimeInMillis()); + assertEquals(1431698400000L, DateTime.previousInterval(NON_DST_TS, + 30, Calendar.MINUTE, FJ).getTimeInMillis()); + + // multiples + assertEquals(1450152000000L, DateTime.previousInterval(DST_TS, + 120, Calendar.MINUTE).getTimeInMillis()); + assertEquals(1431698400000L, DateTime.previousInterval(NON_DST_TS, + 120, Calendar.MINUTE).getTimeInMillis()); + } + + @Test + public void previousIntervalHours() { + // interval 1 + assertEquals(1450152000000L, DateTime.previousInterval(DST_TS, + 1, Calendar.HOUR_OF_DAY).getTimeInMillis()); + assertEquals(1431698400000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.HOUR_OF_DAY).getTimeInMillis()); + + // interval 12 + assertEquals(1450137600000L, DateTime.previousInterval(DST_TS, + 12, Calendar.HOUR_OF_DAY).getTimeInMillis()); + assertEquals(1431691200000L, DateTime.previousInterval(NON_DST_TS, + 12, Calendar.HOUR_OF_DAY).getTimeInMillis()); + assertEquals(1450137600000L, DateTime.previousInterval(1450137600000L, + 12, Calendar.HOUR_OF_DAY).getTimeInMillis()); + + // odd interval + assertEquals(1431680400000L, DateTime.previousInterval(NON_DST_TS, + 15, Calendar.HOUR_OF_DAY).getTimeInMillis()); + assertEquals(1450116000000L, DateTime.previousInterval(DST_TS, + 15, Calendar.HOUR_OF_DAY).getTimeInMillis()); + + // TZs - 30m offset here + assertEquals(1450121400000L, DateTime.previousInterval(DST_TS, + 12, Calendar.HOUR_OF_DAY, AF).getTimeInMillis()); + assertEquals(1431675000000L, DateTime.previousInterval(NON_DST_TS, + 12, Calendar.HOUR_OF_DAY, AF).getTimeInMillis()); + // outliers @ 45 minutes + assertEquals(1450131300000L, DateTime.previousInterval(DST_TS, + 12, Calendar.HOUR_OF_DAY, NZ).getTimeInMillis()); + assertEquals(1431688500000L, DateTime.previousInterval(NON_DST_TS, + 12, Calendar.HOUR_OF_DAY, NZ).getTimeInMillis()); + // back to normal + assertEquals(1450137600000L, DateTime.previousInterval(DST_TS, + 12, Calendar.HOUR_OF_DAY, TV).getTimeInMillis()); + assertEquals(1431691200000L, DateTime.previousInterval(NON_DST_TS, + 12, Calendar.HOUR_OF_DAY, TV).getTimeInMillis()); + assertEquals(1450134000000L, DateTime.previousInterval(DST_TS, + 12, Calendar.HOUR_OF_DAY, FJ).getTimeInMillis()); + assertEquals(1431691200000L, DateTime.previousInterval(NON_DST_TS, + 12, Calendar.HOUR_OF_DAY, FJ).getTimeInMillis()); + + // multiples + assertEquals(1450094400000L, DateTime.previousInterval(DST_TS, + 36, Calendar.HOUR_OF_DAY).getTimeInMillis()); + assertEquals(1431604800000L, DateTime.previousInterval(NON_DST_TS, + 36, Calendar.HOUR_OF_DAY).getTimeInMillis()); + } + + @Test + public void previousIntervalDays() { + // interval 1 + assertEquals(1450137600000L, DateTime.previousInterval(DST_TS, + 1, Calendar.DAY_OF_MONTH).getTimeInMillis()); + assertEquals(1431648000000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.DAY_OF_MONTH).getTimeInMillis()); + + // interval 7 - since days aren't consistent, the only thing we can + // do is pick a starting day, i.e. start of the year + assertEquals(1449705600000L, DateTime.previousInterval(DST_TS, + 7, Calendar.DAY_OF_MONTH).getTimeInMillis()); + assertEquals(1431561600000L, DateTime.previousInterval(NON_DST_TS, + 7, Calendar.DAY_OF_MONTH).getTimeInMillis()); + assertEquals(1449705600000L, DateTime.previousInterval(1449705600000L, + 7, Calendar.DAY_OF_MONTH).getTimeInMillis()); + + // leap year + assertEquals(1330473600000L, DateTime.previousInterval(1330516800000L, + 1, Calendar.DAY_OF_MONTH).getTimeInMillis()); + + // TZs - 30m offset here + assertEquals(1450121400000L, DateTime.previousInterval(DST_TS, + 1, Calendar.DAY_OF_MONTH, AF).getTimeInMillis()); + assertEquals(1431631800000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.DAY_OF_MONTH, AF).getTimeInMillis()); + // outliers @ 45 minutes + assertEquals(1450088100000L, DateTime.previousInterval(DST_TS, + 1, Calendar.DAY_OF_MONTH, NZ).getTimeInMillis()); + assertEquals(1431688500000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.DAY_OF_MONTH, NZ).getTimeInMillis()); + // back to normal + assertEquals(1450094400000L, DateTime.previousInterval(DST_TS, + 1, Calendar.DAY_OF_MONTH, TV).getTimeInMillis()); + assertEquals(1431691200000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.DAY_OF_MONTH, TV).getTimeInMillis()); + assertEquals(1450090800000L, DateTime.previousInterval(DST_TS, + 1, Calendar.DAY_OF_MONTH, FJ).getTimeInMillis()); + assertEquals(1431691200000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.DAY_OF_MONTH, FJ).getTimeInMillis()); + + // multiples + assertEquals(1445990400000L, DateTime.previousInterval(DST_TS, + 60, Calendar.DAY_OF_MONTH).getTimeInMillis()); + assertEquals(1430438400000L, DateTime.previousInterval(NON_DST_TS, + 60, Calendar.DAY_OF_MONTH).getTimeInMillis()); + } + + @Test + public void previousIntervalWeeks() { + // Test assumes Sunday is first day of week. + Locale.setDefault(Locale.US); + + // interval 1 DST_TS starts on 13th of Dec, NON starts on the 10th of May + assertEquals(1449964800000L, DateTime.previousInterval(DST_TS, + 1, Calendar.DAY_OF_WEEK).getTimeInMillis()); + assertEquals(1431216000000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.DAY_OF_WEEK).getTimeInMillis()); + + // interval 2 + assertEquals(1449964800000L, DateTime.previousInterval(DST_TS, + 2, Calendar.DAY_OF_WEEK).getTimeInMillis()); + assertEquals(1431216000000L, DateTime.previousInterval(NON_DST_TS, + 2, Calendar.DAY_OF_WEEK).getTimeInMillis()); + assertEquals(1435449600000L, DateTime.previousInterval(1435795200000L, + 2, Calendar.DAY_OF_WEEK).getTimeInMillis()); + + // TZs - 30m offset here + assertEquals(1449948600000L, DateTime.previousInterval(DST_TS, + 1, Calendar.DAY_OF_WEEK, AF).getTimeInMillis()); + assertEquals(1431199800000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.DAY_OF_WEEK, AF).getTimeInMillis()); + // outliers @ 45 minutes + assertEquals(1449915300000L, DateTime.previousInterval(DST_TS, + 1, Calendar.DAY_OF_WEEK, NZ).getTimeInMillis()); + assertEquals(1431170100000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.DAY_OF_WEEK, NZ).getTimeInMillis()); + // back to normal + assertEquals(1449921600000L, DateTime.previousInterval(DST_TS, + 1, Calendar.DAY_OF_WEEK, TV).getTimeInMillis()); + assertEquals(1431172800000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.DAY_OF_WEEK, TV).getTimeInMillis()); + assertEquals(1449918000000L, DateTime.previousInterval(DST_TS, + 1, Calendar.DAY_OF_WEEK, FJ).getTimeInMillis()); + assertEquals(1431172800000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.DAY_OF_WEEK, FJ).getTimeInMillis()); + + // multiples - still from start of the week + assertEquals(1449964800000L, DateTime.previousInterval(DST_TS, + 104, Calendar.DAY_OF_WEEK).getTimeInMillis()); + assertEquals(1431216000000L, DateTime.previousInterval(NON_DST_TS, + 104, Calendar.DAY_OF_WEEK).getTimeInMillis()); + } + + @Test + public void previousIntervalWeekOfYear() { + // interval 1 DST_TS starts on 10th of Dec, NON starts on the 14th of May + assertEquals(1449705600000L, DateTime.previousInterval(DST_TS, + 1, Calendar.WEEK_OF_YEAR).getTimeInMillis()); + assertEquals(1431561600000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.WEEK_OF_YEAR).getTimeInMillis()); + + // interval 26 + assertEquals(1435795200000L, DateTime.previousInterval(DST_TS, + 26, Calendar.WEEK_OF_YEAR).getTimeInMillis()); + assertEquals(1420070400000L, DateTime.previousInterval(NON_DST_TS, + 26, Calendar.WEEK_OF_YEAR).getTimeInMillis()); + assertEquals(1435795200000L, DateTime.previousInterval(1435795200000L, + 26, Calendar.WEEK_OF_YEAR).getTimeInMillis()); + + // TZs - 30m offset here + assertEquals(1449689400000L, DateTime.previousInterval(DST_TS, + 1, Calendar.WEEK_OF_YEAR, AF).getTimeInMillis()); + assertEquals(1431545400000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.WEEK_OF_YEAR, AF).getTimeInMillis()); + // outliers @ 45 minutes + assertEquals(1449656100000L, DateTime.previousInterval(DST_TS, + 1, Calendar.WEEK_OF_YEAR, NZ).getTimeInMillis()); + assertEquals(1431515700000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.WEEK_OF_YEAR, NZ).getTimeInMillis()); + // back to normal + assertEquals(1449662400000L, DateTime.previousInterval(DST_TS, + 1, Calendar.WEEK_OF_YEAR, TV).getTimeInMillis()); + assertEquals(1431518400000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.WEEK_OF_YEAR, TV).getTimeInMillis()); + assertEquals(1449658800000L, DateTime.previousInterval(DST_TS, + 1, Calendar.WEEK_OF_YEAR, FJ).getTimeInMillis()); + assertEquals(1431518400000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.WEEK_OF_YEAR, FJ).getTimeInMillis()); + + // multiples + assertEquals(1420070400000L, DateTime.previousInterval(DST_TS, + 104, Calendar.WEEK_OF_YEAR).getTimeInMillis()); + assertEquals(1420070400000L, DateTime.previousInterval(NON_DST_TS, + 104, Calendar.WEEK_OF_YEAR).getTimeInMillis()); + } + + @Test + public void previousIntervalMonths() { + // interval 1 + assertEquals(1448928000000L, DateTime.previousInterval(DST_TS, + 1, Calendar.MONTH).getTimeInMillis()); + assertEquals(1430438400000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.MONTH).getTimeInMillis()); + + // interval 3 (quarters) + assertEquals(1443657600000L, DateTime.previousInterval(DST_TS, + 3, Calendar.MONTH).getTimeInMillis()); + assertEquals(1427846400000L, DateTime.previousInterval(NON_DST_TS, + 3, Calendar.MONTH).getTimeInMillis()); + assertEquals(1443657600000L, DateTime.previousInterval(1443657600000L, + 3, Calendar.MONTH).getTimeInMillis()); + + // odd intervals + assertEquals(1446336000000L, DateTime.previousInterval(DST_TS, + 5, Calendar.MONTH).getTimeInMillis()); + assertEquals(1420070400000L, DateTime.previousInterval(NON_DST_TS, + 5, Calendar.MONTH).getTimeInMillis()); + + // TZs - 30m offset here + assertEquals(1448911800000L, DateTime.previousInterval(DST_TS, + 1, Calendar.MONTH, AF).getTimeInMillis()); + assertEquals(1430422200000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.MONTH, AF).getTimeInMillis()); + // outliers @ 45 minutes + assertEquals(1448878500000L, DateTime.previousInterval(DST_TS, + 1, Calendar.MONTH, NZ).getTimeInMillis()); + assertEquals(1430392500000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.MONTH, NZ).getTimeInMillis()); + // back to normal + assertEquals(1448884800000L, DateTime.previousInterval(DST_TS, + 1, Calendar.MONTH, TV).getTimeInMillis()); + assertEquals(1430395200000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.MONTH, TV).getTimeInMillis()); + assertEquals(1448881200000L, DateTime.previousInterval(DST_TS, + 1, Calendar.MONTH, FJ).getTimeInMillis()); + assertEquals(1430395200000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.MONTH, FJ).getTimeInMillis()); + + // multiples + assertEquals(1420070400000L, DateTime.previousInterval(DST_TS, + 24, Calendar.MONTH).getTimeInMillis()); + assertEquals(1420070400000L, DateTime.previousInterval(NON_DST_TS, + 24, Calendar.MONTH).getTimeInMillis()); + } + + @Test + public void previousIntervalYears() { + // interval 1 + assertEquals(1420070400000L, DateTime.previousInterval(DST_TS, + 1, Calendar.YEAR).getTimeInMillis()); + assertEquals(1420070400000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.YEAR).getTimeInMillis()); + + // interval 5 + assertEquals(1420070400000L, DateTime.previousInterval(DST_TS, + 5, Calendar.YEAR).getTimeInMillis()); + assertEquals(1420070400000L, DateTime.previousInterval(NON_DST_TS, + 5, Calendar.YEAR).getTimeInMillis()); + assertEquals(1420070400000L, DateTime.previousInterval(1420070400000L, + 5, Calendar.YEAR).getTimeInMillis()); + + // TZs - 30m offset here + assertEquals(1420054200000L, DateTime.previousInterval(DST_TS, + 1, Calendar.YEAR, AF).getTimeInMillis()); + assertEquals(1420054200000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.YEAR, AF).getTimeInMillis()); + // outliers @ 45 minutes + assertEquals(1420020900000L, DateTime.previousInterval(DST_TS, + 1, Calendar.YEAR, NZ).getTimeInMillis()); + assertEquals(1420020900000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.YEAR, NZ).getTimeInMillis()); + // back to normal + assertEquals(1420027200000L, DateTime.previousInterval(DST_TS, + 1, Calendar.YEAR, TV).getTimeInMillis()); + assertEquals(1420027200000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.YEAR, TV).getTimeInMillis()); + assertEquals(1420023600000L, DateTime.previousInterval(DST_TS, + 1, Calendar.YEAR, FJ).getTimeInMillis()); + assertEquals(1420023600000L, DateTime.previousInterval(NON_DST_TS, + 1, Calendar.YEAR, FJ).getTimeInMillis()); + } + + @Test (expected = IllegalArgumentException.class) + public void previousIntervalNegativeTs() { + DateTime.previousInterval(-42, 1, Calendar.MINUTE); + } + + @Test (expected = IllegalArgumentException.class) + public void previousIntervalNegativeInterval() { + DateTime.previousInterval(1355961600000L, -1, Calendar.MINUTE); + } + + @Test (expected = IllegalArgumentException.class) + public void previousIntervalZeroInterval() { + DateTime.previousInterval(1355961600000L, 0, Calendar.MINUTE); + } + + @Test (expected = IllegalArgumentException.class) + public void previousIntervalNegativeUnit() { + DateTime.previousInterval(1355961600000L, 1, -1); + } + + @Test (expected = IllegalArgumentException.class) + public void previousIntervalUnsupportedUnit() { + DateTime.previousInterval(1355961600000L, 1, Calendar.HOUR); + } + + @Test (expected = IllegalArgumentException.class) + public void previousIntervalMassiveUnit() { + DateTime.previousInterval(1355961600000L, 1, 6048); + } + + @Test + public void unitsToCalendarType() { + assertEquals(Calendar.MILLISECOND, DateTime.unitsToCalendarType("ms")); + assertEquals(Calendar.SECOND, DateTime.unitsToCalendarType("s")); + assertEquals(Calendar.MINUTE, DateTime.unitsToCalendarType("m")); + assertEquals(Calendar.HOUR_OF_DAY, DateTime.unitsToCalendarType("h")); + assertEquals(Calendar.DAY_OF_MONTH, DateTime.unitsToCalendarType("d")); + assertEquals(Calendar.DAY_OF_WEEK, DateTime.unitsToCalendarType("w")); + assertEquals(Calendar.MONTH, DateTime.unitsToCalendarType("n")); + assertEquals(Calendar.YEAR, DateTime.unitsToCalendarType("y")); + try { + DateTime.unitsToCalendarType("j"); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + try { + DateTime.unitsToCalendarType(null); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + try { + DateTime.unitsToCalendarType(""); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { } + } + + + @Test + public void msFromNano() { + assertEquals(0, DateTime.msFromNano(0), 0.0001); + assertEquals(1, DateTime.msFromNano(1000000), 0.0001); + assertEquals(-1, DateTime.msFromNano(-1000000), 0.0001); + assertEquals(1.5, DateTime.msFromNano(1500000), 0.0001); + assertEquals(1.123, DateTime.msFromNano(1123000), 0.0001); + } + + @Test + public void msFromNanoDiff() { + assertEquals(0, DateTime.msFromNanoDiff(1000000, 1000000), 0.0001); + assertEquals(0.5, DateTime.msFromNanoDiff(1500000, 1000000), 0.0001); + assertEquals(1.5, DateTime.msFromNanoDiff(1500000, 0), 0.0001); + assertEquals(0.5, DateTime.msFromNanoDiff(-1000000, -1500000), 0.0001); + try { + assertEquals(0.5, DateTime.msFromNanoDiff(1000000, 1500000), 0.0001); + fail("Expected an IllegalArgumentException"); + } catch (IllegalArgumentException e) {} + } + } diff --git a/test/utils/TestExceptions.java b/test/utils/TestExceptions.java new file mode 100644 index 0000000000..d83ffa715a --- /dev/null +++ b/test/utils/TestExceptions.java @@ -0,0 +1,73 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see <http://www.gnu.org/licenses/>. +package net.opentsdb.utils; + +import static org.junit.Assert.assertSame; +import static org.junit.Assert.fail; + +import java.util.ArrayList; + +import org.junit.Before; +import org.junit.Test; + +import com.stumbleupon.async.Callback; +import com.stumbleupon.async.Deferred; +import com.stumbleupon.async.DeferredGroupException; + +public class TestExceptions { + private ArrayList<Deferred<Object>> deferreds; + + @Before + public void before() { + deferreds = new ArrayList<Deferred<Object>>(1); + } + + @Test + public void oneLevel() throws Exception { + final RuntimeException ex = new RuntimeException("Boo!"); + deferreds.add(Deferred.fromError(ex)); + try { + Deferred.group(deferreds).join(); + fail("Expected a DeferredGroupException"); + } catch (DeferredGroupException dge) { + assertSame(ex, Exceptions.getCause(dge)); + } + } + + @Test + public void nested() throws Exception { + final RuntimeException ex = new RuntimeException("Boo!"); + deferreds.add(Deferred.fromError(ex)); + + final ArrayList<Deferred<Object>> deferreds2 = + new ArrayList<Deferred<Object>>(1); + deferreds2.add(Deferred.fromResult(null)); + + class LOne implements + Callback<Deferred<ArrayList<Object>>, ArrayList<Object>> { + @Override + public Deferred<ArrayList<Object>> call(final ArrayList<Object> piff) + throws Exception { + return Deferred.group(deferreds); + } + } + + try { + Deferred.group(deferreds2).addCallbackDeferring(new LOne()).join(); + fail("Expected a DeferredGroupException"); + } catch (DeferredGroupException dge) { + assertSame(ex, Exceptions.getCause(dge)); + } + } + +} diff --git a/test/utils/TestPluginLoader.java b/test/utils/TestPluginLoader.java index 393a524a64..bf6450e2fa 100644 --- a/test/utils/TestPluginLoader.java +++ b/test/utils/TestPluginLoader.java @@ -20,6 +20,7 @@ import java.util.List; import net.opentsdb.plugin.DummyPlugin; +import net.opentsdb.tsd.HttpRpcPlugin; import net.opentsdb.utils.PluginLoader; import org.junit.Test; @@ -109,6 +110,14 @@ public void loadPlugins() throws Exception { assertEquals(2, plugins.size()); } + @Test + public void loadHttpRpcPlugin() throws Exception { + PluginLoader.loadJAR("plugin_test.jar"); + HttpRpcPlugin plugin = PluginLoader.loadSpecificPlugin("net.opentsdb.tsd.DummyHttpRpcPlugin", HttpRpcPlugin.class); + assertNotNull(plugin); + assertEquals("/dummy/test", plugin.getPath()); + } + @Test public void loadPluginsNotFound() throws Exception { List<DummyPluginBad> plugins = PluginLoader.loadPlugins( diff --git a/third_party/apache/commons-math3-3.4.1.jar.md5 b/third_party/apache/commons-math3-3.4.1.jar.md5 new file mode 100644 index 0000000000..a26a157f84 --- /dev/null +++ b/third_party/apache/commons-math3-3.4.1.jar.md5 @@ -0,0 +1 @@ +14a218d0ee57907dd2c7ef944b6c0afd \ No newline at end of file diff --git a/third_party/apache/commons-math3-3.6.1.jar.md5 b/third_party/apache/commons-math3-3.6.1.jar.md5 new file mode 100644 index 0000000000..2a5ecc945b --- /dev/null +++ b/third_party/apache/commons-math3-3.6.1.jar.md5 @@ -0,0 +1 @@ +5b730d97e4e6368069de1983937c508e diff --git a/third_party/apache/include.mk b/third_party/apache/include.mk new file mode 100644 index 0000000000..991f95bb8c --- /dev/null +++ b/third_party/apache/include.mk @@ -0,0 +1,32 @@ +# Copyright (C) 2011-2022 The OpenTSDB Authors. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the StumbleUpon nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +APACHE_MATH_VERSION := 3.6.1 +APACHE_MATH := third_party/apache/commons-math3-$(APACHE_MATH_VERSION).jar +APACHE_MATH_BASE_URL := https://repo1.maven.org/maven2/org/apache/commons/commons-math3/$(APACHE_MATH_VERSION) + +$(APACHE_MATH): $(APACHE_MATH).md5 + set dummy "$(APACHE_MATH_BASE_URL)" "$(APACHE_MATH)"; shift; $(FETCH_DEPENDENCY) + +THIRD_PARTY += $(APACHE_MATH) diff --git a/third_party/asyncbigtable/asyncbigtable-0.3.1-20170903.031804-2-jar-with-dependencies.jar.md5 b/third_party/asyncbigtable/asyncbigtable-0.3.1-20170903.031804-2-jar-with-dependencies.jar.md5 new file mode 100644 index 0000000000..c6f344387e --- /dev/null +++ b/third_party/asyncbigtable/asyncbigtable-0.3.1-20170903.031804-2-jar-with-dependencies.jar.md5 @@ -0,0 +1 @@ +0c3d205142ae73fd2b72614fa4d7fde0 \ No newline at end of file diff --git a/third_party/asyncbigtable/asyncbigtable-0.4.3-jar-with-dependencies.jar.md5 b/third_party/asyncbigtable/asyncbigtable-0.4.3-jar-with-dependencies.jar.md5 new file mode 100644 index 0000000000..ae03a41a90 --- /dev/null +++ b/third_party/asyncbigtable/asyncbigtable-0.4.3-jar-with-dependencies.jar.md5 @@ -0,0 +1 @@ +0e8c09c99998241ce539f8a500692d50 \ No newline at end of file diff --git a/third_party/asyncbigtable/include.mk b/third_party/asyncbigtable/include.mk new file mode 100644 index 0000000000..d05fe1cd06 --- /dev/null +++ b/third_party/asyncbigtable/include.mk @@ -0,0 +1,23 @@ +# Copyright (C) 2015-2021 The OpenTSDB Authors. +# +# This library is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 2.1 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. + +ASYNCBIGTABLE_VERSION := 0.4.3 +ASYNCBIGTABLE := third_party/asyncbigtable/asyncbigtable-$(ASYNCBIGTABLE_VERSION)-jar-with-dependencies.jar +ASYNCBIGTABLE_BASE_URL := https://repo1.maven.org/maven2/com/pythian/opentsdb/asyncbigtable/$(ASYNCBIGTABLE_VERSION) + +$(ASYNCBIGTABLE): $(ASYNCBIGTABLE).md5 + set dummy "$(ASYNCBIGTABLE_BASE_URL)" "$(ASYNCBIGTABLE)"; shift; $(FETCH_DEPENDENCY) + +THIRD_PARTY += $(ASYNCBIGTABLE) diff --git a/third_party/asynccassandra/asynccassandra-0.0.1-20151104.191228-3-jar-with-dependencies.jar.md5 b/third_party/asynccassandra/asynccassandra-0.0.1-20151104.191228-3-jar-with-dependencies.jar.md5 new file mode 100644 index 0000000000..6469b18da4 --- /dev/null +++ b/third_party/asynccassandra/asynccassandra-0.0.1-20151104.191228-3-jar-with-dependencies.jar.md5 @@ -0,0 +1 @@ +0dd29195cdb9ca4467d0fc32bfffb98c \ No newline at end of file diff --git a/third_party/asynccassandra/asynccassandra-0.0.1-20160229.001338-4-jar-with-dependencies.jar.md5 b/third_party/asynccassandra/asynccassandra-0.0.1-20160229.001338-4-jar-with-dependencies.jar.md5 new file mode 100644 index 0000000000..cbd7d36bb3 --- /dev/null +++ b/third_party/asynccassandra/asynccassandra-0.0.1-20160229.001338-4-jar-with-dependencies.jar.md5 @@ -0,0 +1 @@ +cb857f54223905d744fafa475d82623f \ No newline at end of file diff --git a/third_party/asynccassandra/include.mk b/third_party/asynccassandra/include.mk new file mode 100644 index 0000000000..f98aab5050 --- /dev/null +++ b/third_party/asynccassandra/include.mk @@ -0,0 +1,23 @@ +# Copyright (C) 2015 The OpenTSDB Authors. +# +# This library is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 2.1 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. + +ASYNCCASSANDRA_VERSION := 0.0.1-20160229.001338-4 +ASYNCCASSANDRA := third_party/asynccassandra/asynccassandra-$(ASYNCCASSANDRA_VERSION)-jar-with-dependencies.jar +ASYNCCASSANDRA_BASE_URL := https://oss.sonatype.org/content/repositories/snapshots/net/opentsdb/asynccassandra/0.0.1-SNAPSHOT/ + +$(ASYNCCASSANDRA): $(ASYNCCASSANDRA).md5 + set dummy "$(ASYNCCASSANDRA_BASE_URL)" "$(ASYNCCASSANDRA)"; shift; $(FETCH_DEPENDENCY) + +THIRD_PARTY += $(ASYNCCASSANDRA) diff --git a/third_party/guava/include.mk b/third_party/guava/include.mk index b686739a88..53ba9e2638 100644 --- a/third_party/guava/include.mk +++ b/third_party/guava/include.mk @@ -25,7 +25,7 @@ GUAVA_VERSION := 18.0 GUAVA := third_party/guava/guava-$(GUAVA_VERSION).jar -GUAVA_BASE_URL := http://central.maven.org/maven2/com/google/guava/guava/$(GUAVA_VERSION) +GUAVA_BASE_URL := https://repo1.maven.org/maven2/com/google/guava/guava/$(GUAVA_VERSION) $(GUAVA): $(GUAVA).md5 set dummy "$(GUAVA_BASE_URL)" "$(GUAVA)"; shift; $(FETCH_DEPENDENCY) diff --git a/third_party/gwt/gwt-dev-2.5.0.jar.md5 b/third_party/gwt/gwt-dev-2.5.0.jar.md5 deleted file mode 100644 index 757d50e319..0000000000 --- a/third_party/gwt/gwt-dev-2.5.0.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -538df8db77bacc863f34b82c59d5632d diff --git a/third_party/gwt/gwt-dev-2.6.1.jar.md5 b/third_party/gwt/gwt-dev-2.6.1.jar.md5 new file mode 100644 index 0000000000..b5847e6a60 --- /dev/null +++ b/third_party/gwt/gwt-dev-2.6.1.jar.md5 @@ -0,0 +1 @@ +2f8df1f3b021315775506a1e0a4ee4b8 diff --git a/third_party/gwt/gwt-user-2.5.0.jar.md5 b/third_party/gwt/gwt-user-2.5.0.jar.md5 deleted file mode 100644 index 7a290c8be2..0000000000 --- a/third_party/gwt/gwt-user-2.5.0.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -c58e29c1132ee6694e8f780e045f4d13 diff --git a/third_party/gwt/gwt-user-2.6.1.jar.md5 b/third_party/gwt/gwt-user-2.6.1.jar.md5 new file mode 100644 index 0000000000..822626d3e7 --- /dev/null +++ b/third_party/gwt/gwt-user-2.6.1.jar.md5 @@ -0,0 +1 @@ +ce17f82bb92e3a7416a9be5659cbcc89 diff --git a/third_party/gwt/include.mk b/third_party/gwt/include.mk index c78c3a7951..a3c911d71b 100644 --- a/third_party/gwt/include.mk +++ b/third_party/gwt/include.mk @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2012 The OpenTSDB Authors. +# Copyright (C) 2011-2023 The OpenTSDB Authors. # # This library is free software: you can redistribute it and/or modify it # under the terms of the GNU Lesser General Public License as published @@ -13,11 +13,11 @@ # You should have received a copy of the GNU Lesser General Public License # along with this library. If not, see <http://www.gnu.org/licenses/>. -GWT_VERSION := 2.6.0 +GWT_VERSION := 2.6.1 GWT_DEV_VERSION := $(GWT_VERSION) GWT_DEV := third_party/gwt/gwt-dev-$(GWT_DEV_VERSION).jar -GWT_DEV_BASE_URL := http://central.maven.org/maven2/com/google/gwt/gwt-dev/$(GWT_DEV_VERSION) +GWT_DEV_BASE_URL := https://repo1.maven.org/maven2/com/google/gwt/gwt-dev/$(GWT_DEV_VERSION) $(GWT_DEV): $(GWT_DEV).md5 set dummy "$(GWT_DEV_BASE_URL)" "$(GWT_DEV)"; shift; $(FETCH_DEPENDENCY) @@ -25,9 +25,16 @@ $(GWT_DEV): $(GWT_DEV).md5 GWT_USER_VERSION := $(GWT_VERSION) GWT_USER := third_party/gwt/gwt-user-$(GWT_USER_VERSION).jar -GWT_USER_BASE_URL := http://central.maven.org/maven2/com/google/gwt/gwt-user/$(GWT_USER_VERSION) +GWT_USER_BASE_URL := https://repo1.maven.org/maven2/com/google/gwt/gwt-user/$(GWT_USER_VERSION) $(GWT_USER): $(GWT_USER).md5 set dummy "$(GWT_USER_BASE_URL)" "$(GWT_USER)"; shift; $(FETCH_DEPENDENCY) -THIRD_PARTY += $(GWT_DEV) $(GWT_USER) +GWT_THEME_VERSION := 1.0.0 +GWT_THEME := third_party/gwt/opentsdb-gwt-theme-$(GWT_THEME_VERSION).jar +GWT_THEME_BASE_URL := https://repo1.maven.org/maven2/net/opentsdb/opentsdb-gwt-theme/$(GWT_THEME_VERSION) + +$(GWT_THEME): $(GWT_THEME).md5 + set dummy "$(GWT_THEME_BASE_URL)" "$(GWT_THEME)"; shift; $(FETCH_DEPENDENCY) + +THIRD_PARTY += $(GWT_DEV) $(GWT_USER) $(GWT_THEME) diff --git a/third_party/gwt/opentsdb-gwt-theme-1.0.0.jar.md5 b/third_party/gwt/opentsdb-gwt-theme-1.0.0.jar.md5 new file mode 100644 index 0000000000..17b7f7547c --- /dev/null +++ b/third_party/gwt/opentsdb-gwt-theme-1.0.0.jar.md5 @@ -0,0 +1 @@ +458540cf39138f1ad566c2eabf930699 diff --git a/third_party/hamcrest/include.mk b/third_party/hamcrest/include.mk index b643b87743..c4e8b2151b 100644 --- a/third_party/hamcrest/include.mk +++ b/third_party/hamcrest/include.mk @@ -15,7 +15,7 @@ HAMCREST_VERSION := 1.3 HAMCREST := third_party/hamcrest/hamcrest-core-$(HAMCREST_VERSION).jar -HAMCREST_BASE_URL := http://central.maven.org/maven2/org/hamcrest/hamcrest-core/$(HAMCREST_VERSION) +HAMCREST_BASE_URL := https://repo1.maven.org/maven2/org/hamcrest/hamcrest-core/$(HAMCREST_VERSION) $(HAMCREST): $(HAMCREST).md5 set dummy "$(HAMCREST_BASE_URL)" "$(HAMCREST)"; shift; $(FETCH_DEPENDENCY) diff --git a/third_party/hbase/asynchbase-1.5.0.jar.md5 b/third_party/hbase/asynchbase-1.5.0.jar.md5 deleted file mode 100644 index e20d2ff219..0000000000 --- a/third_party/hbase/asynchbase-1.5.0.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -12c61569f04eb88229c90dde9fa51848 diff --git a/third_party/hbase/asynchbase-1.6.0.jar.md5 b/third_party/hbase/asynchbase-1.6.0.jar.md5 deleted file mode 100644 index 7fcfbd8ec4..0000000000 --- a/third_party/hbase/asynchbase-1.6.0.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -6738dd73fd48d30cbf5c78f62bc18852 diff --git a/third_party/hbase/asynchbase-1.8.0-20161103.193100-4.jar.md5 b/third_party/hbase/asynchbase-1.8.0-20161103.193100-4.jar.md5 new file mode 100644 index 0000000000..97ce7b7a6d --- /dev/null +++ b/third_party/hbase/asynchbase-1.8.0-20161103.193100-4.jar.md5 @@ -0,0 +1 @@ +f4cd05231ee2604e59f65667e4cc7634 \ No newline at end of file diff --git a/third_party/hbase/asynchbase-1.8.0.jar.md5 b/third_party/hbase/asynchbase-1.8.0.jar.md5 new file mode 100644 index 0000000000..daf07749fd --- /dev/null +++ b/third_party/hbase/asynchbase-1.8.0.jar.md5 @@ -0,0 +1 @@ +84ce0ce7f048a80755105f001352e14e diff --git a/third_party/hbase/asynchbase-1.8.2.jar.md5 b/third_party/hbase/asynchbase-1.8.2.jar.md5 new file mode 100644 index 0000000000..73b20d5c91 --- /dev/null +++ b/third_party/hbase/asynchbase-1.8.2.jar.md5 @@ -0,0 +1 @@ +aa7df5b8b3c77d3671eff8bfa12a87c4 \ No newline at end of file diff --git a/third_party/hbase/include.mk b/third_party/hbase/include.mk index eb7ca564ab..01a5407ff7 100644 --- a/third_party/hbase/include.mk +++ b/third_party/hbase/include.mk @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2014 The OpenTSDB Authors. +# Copyright (C) 2011-2017 The OpenTSDB Authors. # # This library is free software: you can redistribute it and/or modify it # under the terms of the GNU Lesser General Public License as published @@ -13,9 +13,9 @@ # You should have received a copy of the GNU Lesser General Public License # along with this library. If not, see <http://www.gnu.org/licenses/>. -ASYNCHBASE_VERSION := 1.6.0 +ASYNCHBASE_VERSION := 1.8.2 ASYNCHBASE := third_party/hbase/asynchbase-$(ASYNCHBASE_VERSION).jar -ASYNCHBASE_BASE_URL := http://central.maven.org/maven2/org/hbase/asynchbase/$(ASYNCHBASE_VERSION) +ASYNCHBASE_BASE_URL := https://repo1.maven.org/maven2/org/hbase/asynchbase/$(ASYNCHBASE_VERSION) $(ASYNCHBASE): $(ASYNCHBASE).md5 set dummy "$(ASYNCHBASE_BASE_URL)" "$(ASYNCHBASE)"; shift; $(FETCH_DEPENDENCY) diff --git a/third_party/include.mk b/third_party/include.mk index c6b7fe2326..b44b12416a 100644 --- a/third_party/include.mk +++ b/third_party/include.mk @@ -21,17 +21,39 @@ THIRD_PARTY = include third_party/guava/include.mk include third_party/gwt/include.mk include third_party/hamcrest/include.mk -include third_party/hbase/include.mk include third_party/jackson/include.mk +include third_party/javacc/include.mk include third_party/javassist/include.mk +include third_party/jexl/include.mk +include third_party/jgrapht/include.mk include third_party/junit/include.mk +include third_party/kryo/include.mk include third_party/logback/include.mk include third_party/mockito/include.mk include third_party/netty/include.mk include third_party/objenesis/include.mk include third_party/powermock/include.mk -include third_party/protobuf/include.mk include third_party/slf4j/include.mk include third_party/suasync/include.mk include third_party/validation-api/include.mk +include third_party/apache/include.mk + +if BIGTABLE +include third_party/asyncbigtable/include.mk +ASYNCCASSANDRA_VERSION = 0.0 +ASYNCHBASE_VERSION = 0.0 +ZOOKEEPER_VERSION = 0.0 +else +if CASSANDRA +include third_party/asynccassandra/include.mk +ASYNCBIGTABLE_VERSION = 0.0 +ASYNCHBASE_VERSION = 0.0 +ZOOKEEPER_VERSION = 0.0 +else +include third_party/hbase/include.mk +include third_party/protobuf/include.mk include third_party/zookeeper/include.mk +ASYNCBIGTABLE_VERSION = 0.0 +ASYNCCASSANDRA_VERSION = 0.0 +endif +endif diff --git a/third_party/jackson/include.mk b/third_party/jackson/include.mk index c74fb7fa39..57002ac30b 100644 --- a/third_party/jackson/include.mk +++ b/third_party/jackson/include.mk @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2014 The OpenTSDB Authors. +# Copyright (C) 2011-2022 The OpenTSDB Authors. # # This library is free software: you can redistribute it and/or modify it # under the terms of the GNU Lesser General Public License as published @@ -13,25 +13,25 @@ # You should have received a copy of the GNU Lesser General Public License # along with this library. If not, see <http://www.gnu.org/licenses/>. -JACKSON_VERSION := 2.4.3 +JACKSON_VERSION := 2.14.1 JACKSON_ANNOTATIONS_VERSION = $(JACKSON_VERSION) JACKSON_ANNOTATIONS := third_party/jackson/jackson-annotations-$(JACKSON_ANNOTATIONS_VERSION).jar -JACKSON_ANNOTATIONS_BASE_URL := http://central.maven.org/maven2/com/fasterxml/jackson/core/jackson-annotations/$(JACKSON_VERSION) +JACKSON_ANNOTATIONS_BASE_URL := https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-annotations/$(JACKSON_VERSION) $(JACKSON_ANNOTATIONS): $(JACKSON_ANNOTATIONS).md5 set dummy "$(JACKSON_ANNOTATIONS_BASE_URL)" "$(JACKSON_ANNOTATIONS)"; shift; $(FETCH_DEPENDENCY) JACKSON_CORE_VERSION = $(JACKSON_VERSION) JACKSON_CORE := third_party/jackson/jackson-core-$(JACKSON_CORE_VERSION).jar -JACKSON_CORE_BASE_URL := http://central.maven.org/maven2/com/fasterxml/jackson/core/jackson-core/$(JACKSON_VERSION) +JACKSON_CORE_BASE_URL := https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-core/$(JACKSON_VERSION) $(JACKSON_CORE): $(JACKSON_CORE).md5 set dummy "$(JACKSON_CORE_BASE_URL)" "$(JACKSON_CORE)"; shift; $(FETCH_DEPENDENCY) JACKSON_DATABIND_VERSION = $(JACKSON_VERSION) JACKSON_DATABIND := third_party/jackson/jackson-databind-$(JACKSON_DATABIND_VERSION).jar -JACKSON_DATABIND_BASE_URL := http://central.maven.org/maven2/com/fasterxml/jackson/core/jackson-databind/$(JACKSON_VERSION) +JACKSON_DATABIND_BASE_URL := https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-databind/$(JACKSON_VERSION) $(JACKSON_DATABIND): $(JACKSON_DATABIND).md5 set dummy "$(JACKSON_DATABIND_BASE_URL)" "$(JACKSON_DATABIND)"; shift; $(FETCH_DEPENDENCY) diff --git a/third_party/jackson/jackson-annotations-2.1.5.jar.md5 b/third_party/jackson/jackson-annotations-2.1.5.jar.md5 deleted file mode 100644 index 5facae61a2..0000000000 --- a/third_party/jackson/jackson-annotations-2.1.5.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -bfe728a2d5f507e143ec41702a3dfc52 diff --git a/third_party/jackson/jackson-annotations-2.14.1.jar.md5 b/third_party/jackson/jackson-annotations-2.14.1.jar.md5 new file mode 100644 index 0000000000..3e09d564c8 --- /dev/null +++ b/third_party/jackson/jackson-annotations-2.14.1.jar.md5 @@ -0,0 +1 @@ +da4742ba6aeb24bf1bd29382fd95647b diff --git a/third_party/jackson/jackson-annotations-2.4.3.jar.md5 b/third_party/jackson/jackson-annotations-2.4.3.jar.md5 deleted file mode 100644 index b921a9846f..0000000000 --- a/third_party/jackson/jackson-annotations-2.4.3.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -31ef4fa866f9d24960a6807c9c299e98 diff --git a/third_party/jackson/jackson-annotations-2.9.10.jar.md5 b/third_party/jackson/jackson-annotations-2.9.10.jar.md5 new file mode 100644 index 0000000000..78c26afb02 --- /dev/null +++ b/third_party/jackson/jackson-annotations-2.9.10.jar.md5 @@ -0,0 +1 @@ +26c2b6f7bc704ccadc64c83995e0ff7f \ No newline at end of file diff --git a/third_party/jackson/jackson-annotations-2.9.5.jar.md5 b/third_party/jackson/jackson-annotations-2.9.5.jar.md5 new file mode 100644 index 0000000000..7f344feb82 --- /dev/null +++ b/third_party/jackson/jackson-annotations-2.9.5.jar.md5 @@ -0,0 +1 @@ +93ff99082f89beba3dd7b594a478df10 diff --git a/third_party/jackson/jackson-core-2.1.5.jar.md5 b/third_party/jackson/jackson-core-2.1.5.jar.md5 deleted file mode 100644 index 356d9b7a84..0000000000 --- a/third_party/jackson/jackson-core-2.1.5.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -25f14871629c6ed2408438f8285ad26d diff --git a/third_party/jackson/jackson-core-2.14.1.jar.md5 b/third_party/jackson/jackson-core-2.14.1.jar.md5 new file mode 100644 index 0000000000..72ffeeed96 --- /dev/null +++ b/third_party/jackson/jackson-core-2.14.1.jar.md5 @@ -0,0 +1 @@ +f9604a5f31129cdb7db4bdec90111850 \ No newline at end of file diff --git a/third_party/jackson/jackson-core-2.4.3.jar.md5 b/third_party/jackson/jackson-core-2.4.3.jar.md5 deleted file mode 100644 index d166db8725..0000000000 --- a/third_party/jackson/jackson-core-2.4.3.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -750ef3d86f04fe0d6d14d6ae904a6d2d diff --git a/third_party/jackson/jackson-core-2.9.10.jar.md5 b/third_party/jackson/jackson-core-2.9.10.jar.md5 new file mode 100644 index 0000000000..89a33946ea --- /dev/null +++ b/third_party/jackson/jackson-core-2.9.10.jar.md5 @@ -0,0 +1 @@ +d62d9b1d1d83dd553e678bc8fce8f809 \ No newline at end of file diff --git a/third_party/jackson/jackson-core-2.9.5.jar.md5 b/third_party/jackson/jackson-core-2.9.5.jar.md5 new file mode 100644 index 0000000000..74729af0ab --- /dev/null +++ b/third_party/jackson/jackson-core-2.9.5.jar.md5 @@ -0,0 +1 @@ +ec59f24f7f8d9acf53301c562722adf2 diff --git a/third_party/jackson/jackson-databind-2.1.5.jar.md5 b/third_party/jackson/jackson-databind-2.1.5.jar.md5 deleted file mode 100644 index 3e9e342bb5..0000000000 --- a/third_party/jackson/jackson-databind-2.1.5.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -18603628104fa90698bfd713ffc03beb diff --git a/third_party/jackson/jackson-databind-2.14.1.jar.md5 b/third_party/jackson/jackson-databind-2.14.1.jar.md5 new file mode 100644 index 0000000000..fe98f6bf13 --- /dev/null +++ b/third_party/jackson/jackson-databind-2.14.1.jar.md5 @@ -0,0 +1 @@ +3e3e7aab8799ccc169b10f244e6fb5b4 diff --git a/third_party/jackson/jackson-databind-2.4.3.jar.md5 b/third_party/jackson/jackson-databind-2.4.3.jar.md5 deleted file mode 100644 index 52f07b826c..0000000000 --- a/third_party/jackson/jackson-databind-2.4.3.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -4fcb9f74280eaa21de10191212c65b11 diff --git a/third_party/jackson/jackson-databind-2.9.10.jar.md5 b/third_party/jackson/jackson-databind-2.9.10.jar.md5 new file mode 100644 index 0000000000..a8536777d9 --- /dev/null +++ b/third_party/jackson/jackson-databind-2.9.10.jar.md5 @@ -0,0 +1 @@ +ff43d79c624b0f7d465542fee6648474 \ No newline at end of file diff --git a/third_party/jackson/jackson-databind-2.9.5.jar.md5 b/third_party/jackson/jackson-databind-2.9.5.jar.md5 new file mode 100644 index 0000000000..273471bcf0 --- /dev/null +++ b/third_party/jackson/jackson-databind-2.9.5.jar.md5 @@ -0,0 +1 @@ +34b37affbf74f5d199be10622ddc83cd diff --git a/third_party/javacc/include.mk b/third_party/javacc/include.mk new file mode 100644 index 0000000000..aa022bbe72 --- /dev/null +++ b/third_party/javacc/include.mk @@ -0,0 +1,23 @@ +# Copyright (C) 2015 The OpenTSDB Authors. +# +# This library is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 2.1 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. + +JAVACC_VERSION := 6.1.2 +JAVACC := third_party/javacc/javacc-$(JAVACC_VERSION).jar +JAVACC_BASE_URL := https://repo1.maven.org/maven2/net/java/dev/javacc/javacc/$(JAVACC_VERSION) + +$(JAVACC): $(JAVACC).md5 + set dummy "$(JAVACC_BASE_URL)" "$(JAVACC)"; shift; $(FETCH_DEPENDENCY) + +THIRD_PARTY += $(JAVACC) diff --git a/third_party/javacc/javacc-6.1.2.jar.md5 b/third_party/javacc/javacc-6.1.2.jar.md5 new file mode 100644 index 0000000000..84d9e28cfc --- /dev/null +++ b/third_party/javacc/javacc-6.1.2.jar.md5 @@ -0,0 +1 @@ +c74b2df75b4c46209d6da22bf4dad976 diff --git a/third_party/javassist/include.mk b/third_party/javassist/include.mk index 2df2cf6063..c667a3e5c0 100644 --- a/third_party/javassist/include.mk +++ b/third_party/javassist/include.mk @@ -23,9 +23,9 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -JAVASSIST_VERSION := 3.18.1-GA +JAVASSIST_VERSION := 3.21.0-GA JAVASSIST := third_party/javassist/javassist-$(JAVASSIST_VERSION).jar -JAVASSIST_BASE_URL := http://central.maven.org/maven2/org/javassist/javassist/$(JAVASSIST_VERSION) +JAVASSIST_BASE_URL := https://repo1.maven.org/maven2/org/javassist/javassist/$(JAVASSIST_VERSION) $(JAVASSIST): $(JAVASSIST).md5 set dummy "$(JAVASSIST_BASE_URL)" "$(JAVASSIST)"; shift; $(FETCH_DEPENDENCY) diff --git a/third_party/javassist/javassist-3.21.0-GA.jar.md5 b/third_party/javassist/javassist-3.21.0-GA.jar.md5 new file mode 100644 index 0000000000..fba94d1657 --- /dev/null +++ b/third_party/javassist/javassist-3.21.0-GA.jar.md5 @@ -0,0 +1 @@ +3dba2305f842c2891df0a0926e18bcfa \ No newline at end of file diff --git a/third_party/jexl/commons-jexl-2.1.1.jar.md5 b/third_party/jexl/commons-jexl-2.1.1.jar.md5 new file mode 100644 index 0000000000..866f0e175a --- /dev/null +++ b/third_party/jexl/commons-jexl-2.1.1.jar.md5 @@ -0,0 +1 @@ +4ad8f5c161dd3a50e190334555675db9 diff --git a/third_party/jexl/commons-logging-1.1.1.jar.md5 b/third_party/jexl/commons-logging-1.1.1.jar.md5 new file mode 100644 index 0000000000..00979c8fe9 --- /dev/null +++ b/third_party/jexl/commons-logging-1.1.1.jar.md5 @@ -0,0 +1 @@ +ed448347fc0104034aa14c8189bf37de diff --git a/third_party/jexl/commons-logging-1.2.jar.md5 b/third_party/jexl/commons-logging-1.2.jar.md5 new file mode 100644 index 0000000000..4d1ffb1050 --- /dev/null +++ b/third_party/jexl/commons-logging-1.2.jar.md5 @@ -0,0 +1 @@ +040b4b4d8eac886f6b4a2a3bd2f31b00 \ No newline at end of file diff --git a/third_party/jexl/include.mk b/third_party/jexl/include.mk new file mode 100644 index 0000000000..9e8d3e5343 --- /dev/null +++ b/third_party/jexl/include.mk @@ -0,0 +1,33 @@ +# Copyright (C) 2015-2023 The OpenTSDB Authors. +# +# This library is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 2.1 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. + +JEXL_VERSION := 2.1.1 +JEXL := third_party/jexl/commons-jexl-$(JEXL_VERSION).jar +JEXL_BASE_URL := https://repo1.maven.org/maven2/org/apache/commons/commons-jexl/$(JEXL_VERSION) + +$(JEXL): $(JEXL).md5 + set dummy "$(JEXL_BASE_URL)" "$(JEXL)"; shift; $(FETCH_DEPENDENCY) + +THIRD_PARTY += $(JEXL) + +# In here as Jexl depends on it and no one else (for now, I hope) +COMMONS_LOGGING_VERSION := 1.2 +COMMONS_LOGGING := third_party/jexl/commons-logging-$(COMMONS_LOGGING_VERSION).jar +COMMONS_LOGGING_BASE_URL := https://repo1.maven.org/maven2/commons-logging/commons-logging/$(COMMONS_LOGGING_VERSION) + +$(COMMONS_LOGGING): $(COMMONS_LOGGING).md5 + set dummy "$(COMMONS_LOGGING_BASE_URL)" "$(COMMONS_LOGGING)"; shift; $(FETCH_DEPENDENCY) + +THIRD_PARTY += $(COMMONS_LOGGING) diff --git a/third_party/jgrapht/include.mk b/third_party/jgrapht/include.mk new file mode 100644 index 0000000000..cce2048438 --- /dev/null +++ b/third_party/jgrapht/include.mk @@ -0,0 +1,23 @@ +# Copyright (C) 2015 The OpenTSDB Authors. +# +# This library is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 2.1 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. + +JGRAPHT_VERSION := 0.9.1 +JGRAPHT := third_party/jgrapht/jgrapht-core-$(JGRAPHT_VERSION).jar +JGRAPHT_BASE_URL := https://repo1.maven.org/maven2/org/jgrapht/jgrapht-core/$(JGRAPHT_VERSION) + +$(JGRAPHT): $(JGRAPHT).md5 + set dummy "$(JGRAPHT_BASE_URL)" "$(JGRAPHT)"; shift; $(FETCH_DEPENDENCY) + +THIRD_PARTY += $(JGRAPHT) diff --git a/third_party/jgrapht/jgrapht-core-0.9.1.jar.md5 b/third_party/jgrapht/jgrapht-core-0.9.1.jar.md5 new file mode 100644 index 0000000000..a0089aa304 --- /dev/null +++ b/third_party/jgrapht/jgrapht-core-0.9.1.jar.md5 @@ -0,0 +1 @@ +86e15da146c96430aef3e1de36df52c8 diff --git a/third_party/junit/include.mk b/third_party/junit/include.mk index 846953d64f..7f97540ace 100644 --- a/third_party/junit/include.mk +++ b/third_party/junit/include.mk @@ -15,7 +15,7 @@ JUNIT_VERSION := 4.11 JUNIT := third_party/junit/junit-$(JUNIT_VERSION).jar -JUNIT_BASE_URL := http://central.maven.org/maven2/junit/junit/$(JUNIT_VERSION) +JUNIT_BASE_URL := https://repo1.maven.org/maven2/junit/junit/$(JUNIT_VERSION) $(JUNIT): $(JUNIT).md5 set dummy "$(JUNIT_BASE_URL)" "$(JUNIT)"; shift; $(FETCH_DEPENDENCY) diff --git a/third_party/kryo/asm-4.0.jar.md5 b/third_party/kryo/asm-4.0.jar.md5 new file mode 100644 index 0000000000..2cfea76019 --- /dev/null +++ b/third_party/kryo/asm-4.0.jar.md5 @@ -0,0 +1 @@ +322d8f88c5111af612df838c0191cd7e diff --git a/third_party/kryo/asm-4.0.jar.md5.1 b/third_party/kryo/asm-4.0.jar.md5.1 new file mode 100644 index 0000000000..7a92e07c69 --- /dev/null +++ b/third_party/kryo/asm-4.0.jar.md5.1 @@ -0,0 +1 @@ +322d8f88c5111af612df838c0191cd7e \ No newline at end of file diff --git a/third_party/kryo/include.mk b/third_party/kryo/include.mk new file mode 100644 index 0000000000..f8d8d5f15e --- /dev/null +++ b/third_party/kryo/include.mk @@ -0,0 +1,46 @@ +# Copyright (C) 2017 The OpenTSDB Authors. +# +# This library is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 2.1 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. + +KRYO_VERSION := 3.0.0 +KRYO := third_party/kryo/kryo-$(KRYO_VERSION).jar +KRYO_BASE_URL := https://repo1.maven.org/maven2/com/esotericsoftware/kryo/$(KRYO_VERSION) + +$(KRYO): $(KRYO).md5 + set dummy "$(KRYO_BASE_URL)" "$(KRYO)"; shift; $(FETCH_DEPENDENCY) + +REFLECTASM_VERSION := 1.10.0 +REFLECTASM := third_party/kryo/reflectasm-$(REFLECTASM_VERSION)-shaded.jar +REFLECTASM_BASE_URL :=https://repo1.maven.org/maven2/com/esotericsoftware/reflectasm/$(REFLECTASM_VERSION) + +$(REFLECTASM): $(REFLECTASM).md5 + set dummy "$(REFLECTASM_BASE_URL)" "$(REFLECTASM)"; shift; $(FETCH_DEPENDENCY) + +ASM_VERSION := 4.0 +ASM := third_party/kryo/asm-$(ASM_VERSION).jar +ASM_BASE_URL := https://repo1.maven.org/maven2/org/ow2/asm/asm/$(ASM_VERSION) + +$(ASM): $(ASM).md5 + set dummy "$(ASM_BASE_URL)" "$(ASM)"; shift; $(FETCH_DEPENDENCY) + +MINLOG_VERSION := 1.3 +MINLOG := third_party/kryo/minlog-$(MINLOG_VERSION).jar +MINLOG_BASE_URL := https://repo1.maven.org/maven2/com/esotericsoftware/minlog/$(MINLOG_VERSION) + +$(MINLOG): $(MINLOG).md5 + set dummy "$(MINLOG_BASE_URL)" "$(MINLOG)"; shift; $(FETCH_DEPENDENCY) + +THIRD_PARTY += $(KRYO) $(REFLECTASM) $(ASM) $(MINLOG) + +https://repo1.maven.org/maven2/com/esotericsoftware/reflectasm/1.10.0/reflectasm-1.10.0-shaded.jar diff --git a/third_party/kryo/kryo-2.21.1.jar.md5 b/third_party/kryo/kryo-2.21.1.jar.md5 new file mode 100644 index 0000000000..7924c1ab25 --- /dev/null +++ b/third_party/kryo/kryo-2.21.1.jar.md5 @@ -0,0 +1 @@ +aa44f411a986ed6130dee951766bbba6 diff --git a/third_party/kryo/kryo-3.0.0.jar.md5 b/third_party/kryo/kryo-3.0.0.jar.md5 new file mode 100644 index 0000000000..28ce336010 --- /dev/null +++ b/third_party/kryo/kryo-3.0.0.jar.md5 @@ -0,0 +1 @@ +720adc0fa9b1ebfa789c6ceda3ffa990 \ No newline at end of file diff --git a/third_party/kryo/kryo-4.0.0.jar.md5 b/third_party/kryo/kryo-4.0.0.jar.md5 new file mode 100644 index 0000000000..ba8eac67ec --- /dev/null +++ b/third_party/kryo/kryo-4.0.0.jar.md5 @@ -0,0 +1 @@ +e817940f2e49280c3e5ad063f38e7884 \ No newline at end of file diff --git a/third_party/kryo/minlog-1.2.jar.md5 b/third_party/kryo/minlog-1.2.jar.md5 new file mode 100644 index 0000000000..6286bd611f --- /dev/null +++ b/third_party/kryo/minlog-1.2.jar.md5 @@ -0,0 +1 @@ +f7cfbdf63b67df0bbfa4c7cb260885bc diff --git a/third_party/kryo/minlog-1.3.jar.md5 b/third_party/kryo/minlog-1.3.jar.md5 new file mode 100644 index 0000000000..da08b47e84 --- /dev/null +++ b/third_party/kryo/minlog-1.3.jar.md5 @@ -0,0 +1 @@ +b4e9b84eaea9750fe58ac3e196c7ed9b \ No newline at end of file diff --git a/third_party/kryo/reflectasm-1.07-shaded.jar.md5 b/third_party/kryo/reflectasm-1.07-shaded.jar.md5 new file mode 100644 index 0000000000..ababaf5e81 --- /dev/null +++ b/third_party/kryo/reflectasm-1.07-shaded.jar.md5 @@ -0,0 +1 @@ +2042c222840fb21e3d13cef7c3965831 diff --git a/third_party/kryo/reflectasm-1.10.0-shaded.jar.md5 b/third_party/kryo/reflectasm-1.10.0-shaded.jar.md5 new file mode 100644 index 0000000000..9f1c813651 --- /dev/null +++ b/third_party/kryo/reflectasm-1.10.0-shaded.jar.md5 @@ -0,0 +1 @@ +779472dd799c5e9b1469e14b13c73061 \ No newline at end of file diff --git a/third_party/logback/include.mk b/third_party/logback/include.mk index de025c59ff..ba2f67e5c5 100644 --- a/third_party/logback/include.mk +++ b/third_party/logback/include.mk @@ -1,4 +1,4 @@ -# Copyright (C) 2015 The OpenTSDB Authors. +# Copyright (C) 2015-2022 The OpenTSDB Authors. # # This library is free software: you can redistribute it and/or modify it # under the terms of the GNU Lesser General Public License as published @@ -12,13 +12,11 @@ # # You should have received a copy of the GNU Lesser General Public License # along with this library. If not, see <http://www.gnu.org/licenses/>. - -http://central.maven.org/maven2/ch/qos/logback/logback-classic/1.0.13/logback-classic-1.0.13.jar -LOGBACK_VERSION := 1.0.13 +LOGBACK_VERSION := 1.3.4 LOGBACK_CLASSIC_VERSION := $(LOGBACK_VERSION) LOGBACK_CLASSIC := third_party/logback/logback-classic-$(LOGBACK_CLASSIC_VERSION).jar -LOGBACK_CLASSIC_BASE_URL := http://central.maven.org/maven2/ch/qos/logback/logback-classic/$(LOGBACK_VERSION) +LOGBACK_CLASSIC_BASE_URL := https://repo1.maven.org/maven2/ch/qos/logback/logback-classic/$(LOGBACK_VERSION) $(LOGBACK_CLASSIC): $(LOGBACK_CLASSIC).md5 set dummy "$(LOGBACK_CLASSIC_BASE_URL)" "$(LOGBACK_CLASSIC)"; shift; $(FETCH_DEPENDENCY) @@ -26,7 +24,7 @@ $(LOGBACK_CLASSIC): $(LOGBACK_CLASSIC).md5 LOGBACK_CORE_VERSION := $(LOGBACK_VERSION) LOGBACK_CORE := third_party/logback/logback-core-$(LOGBACK_CORE_VERSION).jar -LOGBACK_CORE_BASE_URL := http://central.maven.org/maven2/ch/qos/logback/logback-core/$(LOGBACK_VERSION) +LOGBACK_CORE_BASE_URL := https://repo1.maven.org/maven2/ch/qos/logback/logback-core/$(LOGBACK_VERSION) $(LOGBACK_CORE): $(LOGBACK_CORE).md5 set dummy "$(LOGBACK_CORE_BASE_URL)" "$(LOGBACK_CORE)"; shift; $(FETCH_DEPENDENCY) diff --git a/third_party/logback/logback-classic-1.0.13.jar.md5 b/third_party/logback/logback-classic-1.0.13.jar.md5 deleted file mode 100644 index ae8b69cb80..0000000000 --- a/third_party/logback/logback-classic-1.0.13.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -18586a078b51918942002ec085338e19 diff --git a/third_party/logback/logback-classic-1.0.9.jar.md5 b/third_party/logback/logback-classic-1.0.9.jar.md5 deleted file mode 100644 index 283adbfc00..0000000000 --- a/third_party/logback/logback-classic-1.0.9.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -ca99e6b10e9b2f46f264afc5cf1c13e1 diff --git a/third_party/logback/logback-classic-1.2.9.jar.md5 b/third_party/logback/logback-classic-1.2.9.jar.md5 new file mode 100644 index 0000000000..37d75e4e09 --- /dev/null +++ b/third_party/logback/logback-classic-1.2.9.jar.md5 @@ -0,0 +1 @@ +de212f6deebb4cf8b5c91853602db5ec \ No newline at end of file diff --git a/third_party/logback/logback-classic-1.3.4.jar.md5 b/third_party/logback/logback-classic-1.3.4.jar.md5 new file mode 100644 index 0000000000..cc7fdb0a1f --- /dev/null +++ b/third_party/logback/logback-classic-1.3.4.jar.md5 @@ -0,0 +1 @@ +0b98f1f2ad1caa97556b979598038d3d \ No newline at end of file diff --git a/third_party/logback/logback-core-1.0.13.jar.md5 b/third_party/logback/logback-core-1.0.13.jar.md5 deleted file mode 100644 index d4093eb2a6..0000000000 --- a/third_party/logback/logback-core-1.0.13.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -945c6dc3c10d3ce784d456a8bbbd0262 diff --git a/third_party/logback/logback-core-1.0.9.jar.md5 b/third_party/logback/logback-core-1.0.9.jar.md5 deleted file mode 100644 index 6a41d62be9..0000000000 --- a/third_party/logback/logback-core-1.0.9.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -d1647b6efc66bd38237e4d8c484aa7b4 diff --git a/third_party/logback/logback-core-1.2.9.jar.md5 b/third_party/logback/logback-core-1.2.9.jar.md5 new file mode 100644 index 0000000000..6d94ceb6bb --- /dev/null +++ b/third_party/logback/logback-core-1.2.9.jar.md5 @@ -0,0 +1 @@ +e0e576b4001e1c99e551655b3fcbacbf \ No newline at end of file diff --git a/third_party/logback/logback-core-1.3.4.jar.md5 b/third_party/logback/logback-core-1.3.4.jar.md5 new file mode 100644 index 0000000000..0e85214d09 --- /dev/null +++ b/third_party/logback/logback-core-1.3.4.jar.md5 @@ -0,0 +1 @@ +bbb33326f538cfd28186d5b4310b44fa \ No newline at end of file diff --git a/third_party/mockito/include.mk b/third_party/mockito/include.mk index aa99f81071..8f2e8591be 100644 --- a/third_party/mockito/include.mk +++ b/third_party/mockito/include.mk @@ -15,7 +15,7 @@ MOCKITO_VERSION := 1.9.5 MOCKITO := third_party/mockito/mockito-core-$(MOCKITO_VERSION).jar -MOCKITO_BASE_URL := http://central.maven.org/maven2/org/mockito/mockito-core/$(MOCKITO_VERSION) +MOCKITO_BASE_URL := https://repo1.maven.org/maven2/org/mockito/mockito-core/$(MOCKITO_VERSION) $(MOCKITO): $(MOCKITO).md5 set dummy "$(MOCKITO_BASE_URL)" "$(MOCKITO)"; shift; $(FETCH_DEPENDENCY) diff --git a/third_party/mockito/mockito-1.9.0.jar.md5 b/third_party/mockito/mockito-core-1.9.0.jar.md5 similarity index 100% rename from third_party/mockito/mockito-1.9.0.jar.md5 rename to third_party/mockito/mockito-core-1.9.0.jar.md5 diff --git a/third_party/netty/include.mk b/third_party/netty/include.mk index 638da86057..875ff72251 100644 --- a/third_party/netty/include.mk +++ b/third_party/netty/include.mk @@ -23,10 +23,10 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -NETTY_MAJOR_VERSION = 3.9 -NETTY_VERSION := 3.9.4.Final +NETTY_MAJOR_VERSION = 3.10 +NETTY_VERSION := 3.10.6.Final NETTY := third_party/netty/netty-$(NETTY_VERSION).jar -NETTY_BASE_URL := http://central.maven.org/maven2/io/netty/netty/$(NETTY_VERSION) +NETTY_BASE_URL := https://repo1.maven.org/maven2/io/netty/netty/$(NETTY_VERSION) $(NETTY): $(NETTY).md5 set dummy "$(NETTY_BASE_URL)" "$(NETTY)"; shift; $(FETCH_DEPENDENCY) diff --git a/third_party/netty/netty-3.10.6.Final.jar.md5 b/third_party/netty/netty-3.10.6.Final.jar.md5 new file mode 100644 index 0000000000..bc161edbc0 --- /dev/null +++ b/third_party/netty/netty-3.10.6.Final.jar.md5 @@ -0,0 +1 @@ +e9cdf01138257f48d796fb2cf67af53e diff --git a/third_party/netty/netty-3.9.1.Final.jar.md5 b/third_party/netty/netty-3.9.1.Final.jar.md5 deleted file mode 100644 index 0005a0f5fc..0000000000 --- a/third_party/netty/netty-3.9.1.Final.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -c1a35f5f1dbc6d8f693b836a66070d45 diff --git a/third_party/objenesis/include.mk b/third_party/objenesis/include.mk index 51396bf59c..53b6b4585f 100644 --- a/third_party/objenesis/include.mk +++ b/third_party/objenesis/include.mk @@ -15,7 +15,7 @@ OBJENESIS_VERSION := 1.3 OBJENESIS := third_party/objenesis/objenesis-$(OBJENESIS_VERSION).jar -OBJENESIS_BASE_URL := http://central.maven.org/maven2/org/objenesis/objenesis/$(OBJENESIS_VERSION) +OBJENESIS_BASE_URL := https://repo1.maven.org/maven2/org/objenesis/objenesis/$(OBJENESIS_VERSION) $(OBJENESIS): $(OBJENESIS).md5 set dummy "$(OBJENESIS_BASE_URL)" "$(OBJENESIS)"; shift; $(FETCH_DEPENDENCY) diff --git a/third_party/powermock/include.mk b/third_party/powermock/include.mk index 234235e355..9b3d4081c9 100644 --- a/third_party/powermock/include.mk +++ b/third_party/powermock/include.mk @@ -25,7 +25,7 @@ POWERMOCK_MOCKITO_VERSION := 1.5.4 POWERMOCK_MOCKITO := third_party/powermock/powermock-mockito-release-full-$(POWERMOCK_MOCKITO_VERSION)-full.jar -POWERMOCK_MOCKITO_BASE_URL := http://central.maven.org/maven2/org/powermock/powermock-mockito-release-full/$(POWERMOCK_MOCKITO_VERSION) +POWERMOCK_MOCKITO_BASE_URL := https://repo1.maven.org/maven2/org/powermock/powermock-mockito-release-full/$(POWERMOCK_MOCKITO_VERSION) $(POWERMOCK_MOCKITO): $(POWERMOCK_MOCKITO).md5 set dummy "$(POWERMOCK_MOCKITO_BASE_URL)" "$(POWERMOCK_MOCKITO)"; shift; $(FETCH_DEPENDENCY) diff --git a/third_party/protobuf/include.mk b/third_party/protobuf/include.mk index d7a9a01311..c521e130da 100644 --- a/third_party/protobuf/include.mk +++ b/third_party/protobuf/include.mk @@ -15,7 +15,7 @@ PROTOBUF_VERSION := 2.5.0 PROTOBUF := third_party/protobuf/protobuf-java-$(PROTOBUF_VERSION).jar -PROTOBUF_BASE_URL := http://central.maven.org/maven2/com/google/protobuf/protobuf-java/$(PROTOBUF_VERSION) +PROTOBUF_BASE_URL := https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/$(PROTOBUF_VERSION) $(PROTOBUF): $(PROTOBUF).md5 set dummy "$(PROTOBUF_BASE_URL)" "$(PROTOBUF)"; shift; $(FETCH_DEPENDENCY) diff --git a/third_party/slf4j/include.mk b/third_party/slf4j/include.mk index 48d686d80a..7fa319e9d3 100644 --- a/third_party/slf4j/include.mk +++ b/third_party/slf4j/include.mk @@ -13,12 +13,12 @@ # You should have received a copy of the GNU Lesser General Public License # along with this library. If not, see <http://www.gnu.org/licenses/>. -SLF4J_VERSION = 1.7.7 +SLF4J_VERSION = 2.0.6 LOG4J_OVER_SLF4J_VERSION := $(SLF4J_VERSION) LOG4J_OVER_SLF4J := third_party/slf4j/log4j-over-slf4j-$(LOG4J_OVER_SLF4J_VERSION).jar -LOG4J_OVER_SLF4J_BASE_URL := http://central.maven.org/maven2/org/slf4j/log4j-over-slf4j/$(LOG4J_OVER_SLF4J_VERSION) +LOG4J_OVER_SLF4J_BASE_URL := https://repo1.maven.org/maven2/org/slf4j/log4j-over-slf4j/$(LOG4J_OVER_SLF4J_VERSION) $(LOG4J_OVER_SLF4J): $(LOG4J_OVER_SLF4J).md5 set dummy "$(LOG4J_OVER_SLF4J_BASE_URL)" "$(LOG4J_OVER_SLF4J)"; shift; $(FETCH_DEPENDENCY) @@ -26,7 +26,7 @@ $(LOG4J_OVER_SLF4J): $(LOG4J_OVER_SLF4J).md5 SLF4J_API_VERSION := $(SLF4J_VERSION) SLF4J_API := third_party/slf4j/slf4j-api-$(SLF4J_API_VERSION).jar -SLF4J_API_BASE_URL := http://central.maven.org/maven2/org/slf4j/slf4j-api/$(SLF4J_API_VERSION) +SLF4J_API_BASE_URL := https://repo1.maven.org/maven2/org/slf4j/slf4j-api/$(SLF4J_API_VERSION) $(SLF4J_API): $(SLF4J_API).md5 set dummy "$(SLF4J_API_BASE_URL)" "$(SLF4J_API)"; shift; $(FETCH_DEPENDENCY) diff --git a/third_party/slf4j/log4j-over-slf4j-1.7.2.jar.md5 b/third_party/slf4j/log4j-over-slf4j-1.7.2.jar.md5 deleted file mode 100644 index 3b024cdfe3..0000000000 --- a/third_party/slf4j/log4j-over-slf4j-1.7.2.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -cbd407d8ff67a6d54fd233fc0e0d8228 diff --git a/third_party/slf4j/log4j-over-slf4j-2.0.6.jar.md5 b/third_party/slf4j/log4j-over-slf4j-2.0.6.jar.md5 new file mode 100644 index 0000000000..0608e24c10 --- /dev/null +++ b/third_party/slf4j/log4j-over-slf4j-2.0.6.jar.md5 @@ -0,0 +1 @@ +a5ccd131feef393a926dcdfc80436d08 \ No newline at end of file diff --git a/third_party/slf4j/slf4j-api-1.7.2.jar.md5 b/third_party/slf4j/slf4j-api-1.7.2.jar.md5 deleted file mode 100644 index 4853af80dc..0000000000 --- a/third_party/slf4j/slf4j-api-1.7.2.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -ebf348e2831a3b610860fa134ad6f67f diff --git a/third_party/slf4j/slf4j-api-2.0.6.jar.md5 b/third_party/slf4j/slf4j-api-2.0.6.jar.md5 new file mode 100644 index 0000000000..31ad90d856 --- /dev/null +++ b/third_party/slf4j/slf4j-api-2.0.6.jar.md5 @@ -0,0 +1 @@ +0dd65c386e8c5f4e6e014de3f7a7ae60 \ No newline at end of file diff --git a/third_party/suasync/include.mk b/third_party/suasync/include.mk index 599fe6e89d..1751dfedc2 100644 --- a/third_party/suasync/include.mk +++ b/third_party/suasync/include.mk @@ -15,7 +15,7 @@ SUASYNC_VERSION := 1.4.0 SUASYNC := third_party/suasync/async-$(SUASYNC_VERSION).jar -SUASYNC_BASE_URL := http://central.maven.org/maven2/com/stumbleupon/async/$(SUASYNC_VERSION) +SUASYNC_BASE_URL := https://repo1.maven.org/maven2/com/stumbleupon/async/$(SUASYNC_VERSION) $(SUASYNC): $(SUASYNC).md5 set dummy "$(SUASYNC_BASE_URL)" "$(SUASYNC)"; shift; $(FETCH_DEPENDENCY) diff --git a/third_party/validation-api/include.mk b/third_party/validation-api/include.mk index 3bd2f96f7d..e2bcd957ea 100644 --- a/third_party/validation-api/include.mk +++ b/third_party/validation-api/include.mk @@ -15,7 +15,7 @@ VALIDATION_API_VERSION := 1.0.0.GA VALIDATION_API := third_party/validation-api/validation-api-$(VALIDATION_API_VERSION).jar -VALIDATION_API_BASE_URL := http://central.maven.org/maven2/javax/validation/validation-api/$(VALIDATION_API_VERSION) +VALIDATION_API_BASE_URL := https://repo1.maven.org/maven2/javax/validation/validation-api/$(VALIDATION_API_VERSION) $(VALIDATION_API): $(VALIDATION_API).md5 set dummy "$(VALIDATION_API_BASE_URL)" "$(VALIDATION_API)"; shift; $(FETCH_DEPENDENCY) diff --git a/third_party/zookeeper/include.mk b/third_party/zookeeper/include.mk index 69368ea853..6ca1c99ce8 100644 --- a/third_party/zookeeper/include.mk +++ b/third_party/zookeeper/include.mk @@ -13,9 +13,9 @@ # You should have received a copy of the GNU Lesser General Public License # along with this library. If not, see <http://www.gnu.org/licenses/>. -ZOOKEEPER_VERSION := 3.3.6 +ZOOKEEPER_VERSION := 3.4.6 ZOOKEEPER := third_party/zookeeper/zookeeper-$(ZOOKEEPER_VERSION).jar -ZOOKEEPER_BASE_URL := http://central.maven.org/maven2/org/apache/zookeeper/zookeeper/$(ZOOKEEPER_VERSION) +ZOOKEEPER_BASE_URL := https://repo1.maven.org/maven2/org/apache/zookeeper/zookeeper/$(ZOOKEEPER_VERSION) $(ZOOKEEPER): $(ZOOKEEPER).md5 set dummy "$(ZOOKEEPER_BASE_URL)" "$(ZOOKEEPER)"; shift; $(FETCH_DEPENDENCY) diff --git a/third_party/zookeeper/zookeeper-3.4.5.jar.md5 b/third_party/zookeeper/zookeeper-3.4.5.jar.md5 new file mode 100644 index 0000000000..5b123bd33b --- /dev/null +++ b/third_party/zookeeper/zookeeper-3.4.5.jar.md5 @@ -0,0 +1 @@ +00b9db19ad7f18681761edc6db524ceb diff --git a/third_party/zookeeper/zookeeper-3.4.6.jar.md5 b/third_party/zookeeper/zookeeper-3.4.6.jar.md5 new file mode 100644 index 0000000000..ce5652c709 --- /dev/null +++ b/third_party/zookeeper/zookeeper-3.4.6.jar.md5 @@ -0,0 +1 @@ +7d01d317c717268725896cfb81b18152 \ No newline at end of file diff --git a/tools/check_tsd b/tools/check_tsd index ff07aefcb2..d6cd8b0f62 100755 --- a/tools/check_tsd +++ b/tools/check_tsd @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # # Script which queries TSDB with a given metric and alerts based on # supplied threshold. Compatible with Nagios output format, so can be @@ -29,6 +29,15 @@ import sys import time from optparse import OptionParser +AGGREGATORS = ('avg', 'count', 'dev', 'diff', + 'ep50r3', 'ep50r7', 'ep75r3', 'ep75r7', 'ep90r3', 'ep90r7', 'ep95r3', 'ep95r7', + 'ep99r3', 'ep99r7', 'ep999r3', 'ep999r7', + 'mimmin', 'mimmax', 'min', 'max', 'none', + 'p50', 'p75', 'p90', 'p95', 'p99', 'p999', + 'sum', 'zimsum') + +FILL_POLICIES = ('none','nan','null','zero') + def main(argv): """Pulls data out of the TSDB and do very simple alerting from Nagios.""" @@ -47,6 +56,8 @@ def main(argv): help='Downsample function, e.g. one of avg, min, sum, or max.') parser.add_option('-W', '--downsample-window', type='int', default=60, metavar='SECONDS', help='Window size over which to downsample.') + parser.add_option('-F', '--downsample-fill-policy', default='none', metavar='POLICY', + help='Fill Policies, e.g. one of none, nan, null, or zero.') parser.add_option('-a', '--aggregator', default='sum', metavar='METHOD', help='Aggregation method: avg, min, sum (default), max.') parser.add_option('-x', '--method', dest='comparator', default='gt', @@ -68,6 +79,15 @@ def main(argv): parser.add_option('-I', '--ignore-recent', default=0, type='int', metavar='SECONDS', help='Ignore data points that are that' ' are that recent.') + parser.add_option('-P', '--percent-over', dest='percent_over', default=0, + metavar='PERCENT', type='float', help='Only alarm if PERCENT of the data' + ' points violate the threshold.') + parser.add_option('-N', '--now', type='int', default=None, + metavar='UTC', + help='Set unix timestamp for "now", for testing') + parser.add_option('-B', '--bad_percent', dest='bad_percent', default=None, + metavar='PERCENT', type='float', help='Ignore alarm if PERCENT of the data' + ' points is bad') parser.add_option('-S', '--ssl', default=False, action='store_true', help='Make queries to OpenTSDB via SSL (https)') (options, args) = parser.parse_args(args=argv[1:]) @@ -75,9 +95,11 @@ def main(argv): # argument validation if options.comparator not in ('gt', 'ge', 'lt', 'le', 'eq', 'ne'): parser.error("Comparator '%s' not valid." % options.comparator) - elif options.downsample not in ('none', 'avg', 'min', 'sum', 'max'): + elif options.downsample not in ('none',)+AGGREGATORS: parser.error("Downsample '%s' not valid." % options.downsample) - elif options.aggregator not in ('avg', 'min', 'sum', 'max'): + elif options.downsample_fill_policy not in FILL_POLICIES: + parser.error("Downsample Fill policy '%s' not valid." % options.downsample_fill_policy) + elif options.aggregator not in AGGREGATORS: parser.error("Aggregator '%s' not valid." % options.aggregator) elif not options.metric: parser.error('You must specify a metric (option -m).') @@ -90,10 +112,14 @@ def main(argv): ' critical threshold (-c).') elif options.ignore_recent < 0: parser.error('--ignore-recent must be positive.') + elif options.percent_over < 0 or options.percent_over > 100: + parser.error('--percent-over must be in the range 0..100.') - if not options.critical: + options.percent_over /= 100.0 # Convert to range 0-1 + + if options.critical is None: options.critical = options.warning - elif not options.warning: + elif options.warning is None: options.warning = options.critical # argument construction @@ -105,14 +131,22 @@ def main(argv): if options.downsample == 'none': downsampling = '' else: - downsampling = '%ds-%s:' % (options.downsample_window, - options.downsample) + downsampling = '%ds-%s-%s:' % (options.downsample_window, + options.downsample, options.downsample_fill_policy) if options.rate: rate = 'rate:' else: rate = '' - url = ('/q?start=%ss-ago&m=%s:%s%s%s%s&ascii&nagios' - % (options.duration, options.aggregator, downsampling, rate, + + if options.now: + now = options.now + start = '%s' % (now - int(options.duration)) + else: + now = int(time.time()) + start = '%ss-ago' % options.duration + + url = ('/q?start=%s&m=%s:%s%s%s%s&ascii&nagios' + % (start, options.aggregator, downsampling, rate, options.metric, tags)) tsd = '%s:%d' % (options.host, options.port) if options.ssl: # Pick the class to instantiate first. @@ -132,7 +166,7 @@ def main(argv): peer = conn.sock.getpeername() print ('Connected to %s:%d' % (peer[0], peer[1])) conn.set_debuglevel(1) - now = int(time.time()) + try: conn.request('GET', url) res = conn.getresponse() @@ -152,8 +186,6 @@ def main(argv): return 2 # but we won! - if options.verbose: - print (datapoints) datapoints = datapoints.splitlines() def no_data_point(): @@ -173,12 +205,22 @@ def main(argv): badval = None # Value of the bad value we found, if any. npoints = 0 # How many values have we seen? nbad = 0 # How many bad values have we seen? - for datapoint in datapoints: - datapoint = datapoint.split() + ncrit = 0 # How many critical values have we seen? + nwarn = 0 # How many warning values have we seen? + for datapoint_str in datapoints: + datapoint = datapoint_str.split() ts = int(datapoint[1]) delta = now - ts if delta > options.duration or delta <= options.ignore_recent: + if options.verbose: + print "%s (ignored, delta %ds)" % (datapoint_str, delta) + if delta < 0: + break # Skip the rest, we got what we came for. continue # Ignore data points outside of our range. + + if options.verbose: + print datapoint_str + npoints += 1 val = datapoint[2] if '.' in val: @@ -188,19 +230,25 @@ def main(argv): bad = False # Is the current value bad? # compare to warning/crit if comparator(val, options.critical): - rv = 2 bad = True - nbad += 1 + ncrit += 1 + nwarn += 1 elif rv < 2 and comparator(val, options.warning): - rv = 1 bad = True - nbad += 1 + nwarn += 1 if (bad and (badval is None # First bad value we find. or comparator(val, badval))): # Worse value. badval = val badts = ts - + if ncrit > 0 and (float(ncrit) / npoints > options.percent_over): + rv = 2 + nbad = ncrit + elif nwarn > 0 and (float(nwarn) / npoints > options.percent_over): + rv = 1 + nbad = nwarn + else: + rv = 0 if options.verbose and len(datapoints) != npoints: print ('ignored %d/%d data points for being more than %ds old' % (len(datapoints) - npoints, len(datapoints), options.duration)) @@ -211,12 +259,21 @@ def main(argv): print ('worse data point value=%s at ts=%s' % (badval, badts)) badts = time.asctime(time.localtime(badts)) + bad_pct = nbad * 100.0 / npoints + # in nrpe, pipe character is something special, but it's used in tag # searches. Translate it to something else for the purposes of output. ttags = tags.replace("|",":") + + # Retrieve metric name for performance data label. + perf_label = options.metric.split(".")[-1] + if not rv: - print ('OK: %s%s: %d values OK, last=%r' - % (options.metric, ttags, npoints, val)) + status = 'OK: %s%s: %d values OK, last=%r' \ + % (options.metric, ttags, npoints, val) + status += ' | {0}={1};{2};{3};0;{3}'.format( + perf_label, npoints, options.warning, options.critical) + print(status) else: if rv == 1: level = 'WARNING' @@ -224,9 +281,12 @@ def main(argv): elif rv == 2: level = 'CRITICAL' threshold = options.critical - print ('%s: %s%s %s %s: %d/%d bad values (%.1f%%) worst: %r @ %s' - % (level, options.metric, ttags, options.comparator, threshold, - nbad, npoints, nbad * 100.0 / npoints, badval, badts)) + status = '%s: %s%s %s %s: %d/%d bad values (%.1f%%) worst: %r @ %s' \ + % (level, options.metric, ttags, options.comparator, threshold, + nbad, npoints, bad_pct, badval, badts) + status += ' | {0}={1};{2};{3};0;{3}'.format( + perf_label, npoints, options.warning, options.critical) + print(status) return rv diff --git a/tools/check_tsd_v2 b/tools/check_tsd_v2 new file mode 100755 index 0000000000..f93c93fe5e --- /dev/null +++ b/tools/check_tsd_v2 @@ -0,0 +1,312 @@ +#!/usr/bin/env python3 + +from urllib import request +import json +import operator +import time +import logging +from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter + +AGGREGATORS = ("avg", "count", "dev", "diff", + "ep50r3", "ep50r7", "ep75r3", "ep75r7", "ep90r3", "ep90r7", "ep95r3", + "ep95r7", "ep99r3", "ep99r7", "ep999r3", "ep999r7", + "mimmin", "mimmax", "min", "max", "none", + "p50", "p75", "p90", "p95", "p99", "p999", + "sum", "zimsum") +FILL_POLICIES = ("none", "nan", "null", "zero") +METHODS = ("gt", "ge", "lt", "le", "eq", "ne") +ALARMS = ("warn", "crit") +log = logging.getLogger("check_tsd_v2") +log.setLevel(logging.INFO) +ch = logging.StreamHandler() +logformat = "%(message)s" +formatter = logging.Formatter(logformat) +ch.setFormatter(formatter) +log.addHandler(ch) + + +def _get_metrics(query, timeout): + """ + Actually get data from OpenTSDB + + :param str query: the query string + :param int timeout: how long to wait for OpenTSDB to respond + :returns: yields metrics from the resulting list one at a time + :rtype: dict (generator) + """ + try: + res = request.urlopen(query, timeout=timeout) + metrics = json.loads(res.read().decode("utf-8")) + except Exception as e: + log.error("Failed to collect metrics: {}".format(e)) + exit(1) + for m in metrics: + yield m + + +def build_query(args): + """ + Format the query we'll be sending to OpenTSDB + + :param dict args: All arguments needed to format query + :returns: formatted query + :rtype: str + """ + if args.ssl: + query = "https" + else: + query = "http" + query += "://{}:{}/api/query?".format(args.host, args.port) + query += "start={}s-ago&noAnnotations=true&m={}:".format(args.duration, + args.aggregator) + query += "start={}s-ago&".format(args.duration) + if args.ignore_recent > 0: + query +="end={}s-ago&".format(args.ignore_recent) + query +="noAnnotations=true&m={}:".format(args.aggregator) + + if args.rate: + query += "rate" + if args.rate_counter or args.rate_reset_value: + if args.rate_counter: + query += "{counter,," + else: + query += "{,," + if args.rate_reset_value: + query += "{}}".format(args.rate_reset_value) + else: + query += "}" + query += ":" + if args.downsample: + query += "{}s-{}".format(args.downsample_window, args.downsample) + if args.downsample_fill_policy: + query += "-{}".format(args.downsample_fill_policy) + query += ":" + query += args.metric + if args.tag: + tags = ",".join(args.tag) + query += "{" + query += tags + query += "}" + return query + + +def build_comparisons(expressions): + """ + Turn a string object like 'gt,100,crit' into a tuple that + python can use to evaluate state. + Also ensure critical checks are put first in the list + so we don't evaluate a datapoint as WARNING that should + be CRITICAL + + :param list expressions: all expression strings + :returns: formatted expressions + :rtype: list + """ + comparisons = [] + for expression in expressions: + comparator, value, alarm = expression.split(",") + if comparator not in METHODS: + log.error("Invalid comparison method.") + exit(1) + if alarm not in ALARMS: + log.error("Invalid alarm type.") + exit(1) + try: + value = float(value) + except ValueError: + log.error("Alarm value must be a number.") + exit(1) + comparator = operator.__dict__[comparator] + comparisons.append((comparator, value, alarm)) + # Ensure we check criticals first, since all comparisons are ORed + sorted_comp = [] + for comp in comparisons: + if comp[2] == "crit": + sorted_comp.insert(0, comp) + else: + sorted_comp.append(comp) + return sorted_comp + + +def _process_metric(m, args, comparisons, now): + """ + Evaluate a single metric from OpenTSDB. + In this case, a metric is a object containing a list + of tuples of (ts, value) and a separate group of tags related + to the metric. + + :param dict m: the actual metric data + :param dict args: all arguments needed to perform evaluations + :param list comparisons: all comparison tuples + :param float now: the current time (generated by time.time()) + :returns: object describing the metric evaluated and its state + :rtype: dict + """ + value_count = len(m["dps"]) + mresult = {"crit": 0, "crit_alarm": False, "warn_alarm": False, "warn": 0, + "crit_percent": 0, "warn_percent": 0, "empty": False, "metric_avg": 0} + if args.tag: + keys = [t.split("=")[0] for t in args.tag] + mresult["tags"] = [v for k, v in m["tags"].items() if k in keys] + if value_count < 1: + mresult["empty"] = True + return mresult + + avglist = [] + for ts, d in m["dps"].items(): + # handle out-of-time metrics + if args.ignore_recent: + try: + ts = float(ts) + except ValueError: + log.error("Bad timestamp for {}: {}".format(",".join(mresult["tags"]), ts)) + mresult["crit_alarm"] = True + break + delta = now - ts + if delta >= args.ignore_recent: + log.debug("Timestamp outside evaluation range: {}".format(ts)) + continue + avglist.append(d) + for comparison in comparisons: + comparator, value, alarm = comparison + if comparator(d, value): + mresult[alarm] += 1 + break + mresult["metric_avg"] = sum(avglist)/len(avglist) + mresult["crit_percent"] = mresult["crit"] / value_count * 100 + mresult["warn_percent"] = mresult["warn"] / value_count * 100 + if mresult["crit"] > 0: + if args.percent_over > 0 and mresult["crit_percent"] > args.percent_over: + mresult["crit_alarm"] = True + else: + mresult["crit_alarm"] = True + if mresult["warn"] > 0: + if args.percent_over > 0 and mresult["warn_percent"] > args.percent_over: + mresult["warn_alarm"] = True + else: + mresult["warn_alarm"] = True + return mresult + + +def process_metrics(query, args, comparisons): + """ + Because we may get multiple metric "groups" back (if a query like + system.load5{host=*} was sent in) we need to evaluate each individual + metric "group" that returns from _get_metrics(). This wrapper helps + us do just that. + + :param str query: The query to send to OpenTSDB + :param dict args: All potential evaluation arguments + :param list comparisons: all comparison tuples to use for evaluating state + :returns: yields each evaluated metric object as it compeletes + :rtype: dict (generator) + """ + now = time.time() + for m in _get_metrics(query, args.timeout): + yield _process_metric(m, args, comparisons, now) + + +def cli_opts(): + parser = ArgumentParser(description="check tsd query", + formatter_class=ArgumentDefaultsHelpFormatter) + parser.add_argument("-H", "--host", default="localhost", type=str, + help="host to check for stats") + parser.add_argument("-p", "--port", default=4242, type=int, + help="port to check for stats") + parser.add_argument("-m", "--metric", required=True, type=str, + help="Metric to query.") + parser.add_argument("-t", "--tag", action="append", default=[], + help="Tags to filter the metric on.") + parser.add_argument("-d", "--duration", type=int, default=3600, + help="How far back to look for data.") + parser.add_argument("-D", "--downsample", default=None, + help="Downsample function", choices=AGGREGATORS) + parser.add_argument("-W", "--downsample-window", type=int, default=60, + help="Window size over which to downsample.") + parser.add_argument("-F", "--downsample-fill-policy", default=None, + help="Downsample Fill Policies", choices=FILL_POLICIES) + parser.add_argument("-a", "--aggregator", default="sum", + help="Aggregation method", choices=AGGREGATORS) + parser.add_argument("-r", "--rate", default=False, + action="store_true", help="Use rate value as comparison operand.") + parser.add_argument("--rate-counter", default=False, + action="store_true", help="Use rate counter") + parser.add_argument("--rate-reset-value", default=0, + type=int, help="rate reset value") + parser.add_argument("-e", "--expression", action="append", required=True, + help="Comparison expression. e.g. gt,100,warn (multiple allowed)\n" + "Allowed methods: {}\nAllowed alarms: {}".format(",".join(METHODS), ",".join(ALARMS))) + parser.add_argument("-I", "--ignore-recent", default=0, type=int, + help="Ignore data points that are that >= seconds ago.") + parser.add_argument("-P", "--percent-over", dest="percent_over", default=0, + type=float, help="Only alarm if PERCENT of the data" + " points violate the threshold.") + parser.add_argument("-S", "--ssl", default=False, action="store_true", + help="Make queries to OpenTSDB via SSL (https)") + parser.add_argument("-T", "--timeout", type=int, default=30, + help="How long to wait for the response from TSD.") + parser.add_argument("-A", "--alarm-empty", default=False, + action="store_true", help="Alert when an emtpy series returns") + parser.add_argument("--debug", default=False, + action="store_true", help="Verbose logging") + return parser.parse_args() + + +def main(): + args = cli_opts() + if args.debug: + log.setLevel(logging.DEBUG) + if args.percent_over > 100 or args.percent_over < 0: + log.error("Percentage over must be a value from 0-100: {}".format(args.percent_over)) + exit(1) + if args.downsample_window < 0: + log.error("Downsample window must be positive: {}".format(args.percent_over)) + exit(1) + if args.downsample_window < 0: + log.error("Downsample window must be positive: {}".format(args.percent_over)) + exit(1) + if args.ignore_recent >= args.duration: + log.error("Ignore Recent parameter must be smaller than Duration: {}".format(args.ignore_recent)) + exit(1) + comparisons = build_comparisons(args.expression) + query = build_query(args) + + crit = False + warn = False + crits = [] + warns = [] + total = [] + for r in process_metrics(query, args, comparisons): + total.append(r["tags"]) + if not r["crit_alarm"] and not r["warn_alarm"]: + continue + if args.alarm_empty and r["empty"]: + log.info("{} => no data returned in range.".format(",".join(r["tags"]))) + crits.append(r["tags"]) + crit = True + continue + if r["crit_alarm"]: + crits.append(r["tags"]) + crit = True + elif r["warn_alarm"]: + warns.append(r["tags"]) + warn = True + alerts = r["crit"] + r["warn"] + perc = r["crit_percent"] + r["warn_percent"] + log.info("{} => alarmed {} times in range. ({}% alarms). Avg. Value: {}".format(",".join(r["tags"]), + alerts, perc, r["metric_avg"])) + log.info("{} total metrics processed".format(len(total))) + crit_count = len(crits) + warn_count = len(warns) + if crit_count > 0: + log.info("{} Critical Alarms".format(crit_count)) + if warn_count > 0: + log.info("{} Warning Alarms".format(warn_count)) + if crit: + exit(2) + elif warn: + exit(1) + + +if __name__ == "__main__": + main() diff --git a/tools/clean_cache.sh b/tools/clean_cache.sh index 6babdb44ad..1dad8046bf 100755 --- a/tools/clean_cache.sh +++ b/tools/clean_cache.sh @@ -8,5 +8,5 @@ diskSpaceIsShort() { } if diskSpaceIsShort; then - rm -rf "$CACHE_DIR"/* + ( cd ${CACHE_DIR} && find . -type f -exec rm {} \; ) fi diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile new file mode 100644 index 0000000000..c9410133e1 --- /dev/null +++ b/tools/docker/Dockerfile @@ -0,0 +1,43 @@ +FROM java:openjdk-8-alpine + +MAINTAINER jonathan.creasy@gmail.com + +ENV VERSION 2.3.0-RC1 +ENV WORKDIR /usr/share/opentsdb +ENV LOGDIR /var/log/opentsdb +ENV DATADIR /data/opentsdb +ENV ETCDIR /etc/opentsdb + +RUN mkdir -p $WORKDIR/static +RUN mkdir -p $WORKDIR/libs +RUN mkdir -p $WORKDIR/third_party +RUN mkdir -p $WORKDIR/resources +RUN mkdir -p $DATADIR/cache +RUN mkdir -p $LOGDIR +RUN mkdir -p $ETCDIR + +ENV CONFIG $ETCDIR/opentsdb.conf +ENV STATICROOT $WORKDIR/static +ENV CACHEDIR $DATADIR/cache + +ENV CLASSPATH $WORKDIR:$WORKDIR/tsdb-$VERSION.jar:$WORKDIR/libs/*:$WORKDIR/logback.xml +ENV CLASS net.opentsdb.tools.TSDMain + +# It is expected these might need to be passed in with the -e flag +ENV JAVA_OPTS="-Xms512m -Xmx2048m" +ENV ZKQUORUM zookeeper:2181 +ENV ZKBASEDIR /hbase +ENV TSDB_OPTS "--read-only --disable-ui" +ENV TSDB_PORT 4244 + +WORKDIR $WORKDIR + +ADD libs $WORKDIR/libs +ADD logback.xml $WORKDIR +ADD tsdb-$VERSION.jar $WORKDIR +ADD opentsdb.conf $ETCDIR/opentsdb.conf + +VOLUME ["/etc/openstsdb"] +VOLUME ["/data/opentsdb"] + +ENTRYPOINT java -enableassertions -enablesystemassertions -classpath ${CLASSPATH} ${CLASS} --config=${CONFIG} --staticroot=${STATICROOT} --cachedir=${CACHEDIR} --port=${TSDB_PORT} --zkquorum=${ZKQUORUM} --zkbasedir=${ZKBASEDIR} ${TSDB_OPTS} diff --git a/tools/docker/docker.sh b/tools/docker/docker.sh new file mode 100755 index 0000000000..17686a0e61 --- /dev/null +++ b/tools/docker/docker.sh @@ -0,0 +1,16 @@ +#!/bin/bash -x +BUILDROOT=./build; +TOOLS=./tools +DOCKER=$BUILDROOT/docker; +rm -r $DOCKER; +mkdir -p $DOCKER; +SOURCE_PATH=$BUILDROOT; +DEST_PATH=$DOCKER/libs; +mkdir -p $DEST_PATH; +cp ${TOOLS}/docker/Dockerfile ${DOCKER}; +cp ${BUILDROOT}/../src/opentsdb.conf ${DOCKER}; +cp ${BUILDROOT}/../src/logback.xml ${DOCKER}; +#cp ${BUILDROOT}/../src/mygnuplot.sh ${DOCKER}; +cp ${SOURCE_PATH}/tsdb-2.3.0-RC1.jar ${DOCKER}; +cp ${SOURCE_PATH}/third_party/*/*.jar ${DEST_PATH}; +docker build -t opentsdb/opentsdb $DOCKER diff --git a/tools/opentsdb_restart.py b/tools/opentsdb_restart.py index eaad7537f6..3c67f6f39e 100644 --- a/tools/opentsdb_restart.py +++ b/tools/opentsdb_restart.py @@ -1,15 +1,18 @@ -#!/usr/bin/python +#!/usr/bin/env python """Restart opentsdb. Called using -XX:OnOutOfMemoryError=<this script> -Because it's calling the 'service opentsdb' command, should be run as root. +Because it's calling the 'systemctl *action* opentsdb' command, should be run as root. This is known to work with python2.6 and above. """ import os import subprocess +service_name = "opentsdb" +if 'NAME' in os.environ: + service_name = os.environ['NAME'] -subprocess.call(["service", "opentsdb", "stop"]) +subprocess.call(["systemctl", "stop", service_name]) # Close any file handles we inherited from our parent JVM. We need # to do this before restarting so that the socket isn't held open. openfiles = [int(f) for f in os.listdir("/proc/self/fd")] @@ -17,4 +20,4 @@ # that there is less chance of errors with those standard streams. # Other files start at fd 3. os.closerange(3, max(openfiles)) -subprocess.call(["service", "opentsdb", "start"]) +subprocess.call(["systemctl", "start", service_name]) diff --git a/tools/osx_full_stack_install.sh b/tools/osx_full_stack_install.sh new file mode 100644 index 0000000000..a8f28418b2 --- /dev/null +++ b/tools/osx_full_stack_install.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# +# Script which installs HBase, OpenTSDB and TCollector on OSX +# +# This file is part of OpenTSDB. +# Copyright (C) 2010-2012 The OpenTSDB Authors. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 2.1 of the License, or (at your +# option) any later version. This program is distributed in the hope that it +# will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +# General Public License for more details. You should have received a copy +# of the GNU Lesser General Public License along with this program. If not, +# see <http://www.gnu.org/licenses/>. +# +# +if [ $# -eq 0 ] + then + echo "No arguments supplied, please suggest an installation path, ex. $HOME" + exit 1; +fi +BASE_DIR=$1 +SUBBASE_DIR=opentsdb_stack; +if [ ! -d "${BASE_DIR}" ] ; then + echo "$BASE_DIR is not a directory"; + exit 1; +fi +export INSTALL_DIR=$BASE_DIR/$SUBBASE_DIR; +/bin/echo "Installing into $INSTALL_DIR"; +/bin/mkdir -p $INSTALL_DIR; +cd $INSTALL_DIR; +/usr/bin/curl -q http://mirror.cogentco.com/pub/apache/hbase/1.2.5/hbase-1.2.5-bin.tar.gz -o $INSTALL_DIR/hbase-1.2.5-bin.tar.gz 2>/dev/null; +/usr/bin/tar -xzvf hbase-1.2.5-bin.tar.gz -C $INSTALL_DIR/; +cd $INSTALL_DIR/hbase-1.2.5; +/bin/mkdir -p $INSTALL_DIR/data/hbase; +/bin/mkdir -p $INSTALL_DIR/data/zookeeper; +/bin/cat <<EOF > conf/hbase-site.xml +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <property> + <name>hbase.rootdir</name> + <value>file://$INSTALL_DIR/data/hbase</value> + </property> + <property> + <name>hbase.zookeeper.property.dataDir</name> + <value>$INSTALL_DIR/data/zookeeper</value> + </property> +</configuration> +EOF +$INSTALL_DIR/hbase-1.2.5/bin/start-hbase.sh; +cd $INSTALL_DIR +/usr/bin/git clone https://github.com/OpenTSDB/opentsdb.git; +cd opentsdb +$INSTALL_DIR/opentsdb/build.sh clean; $INSTALL_DIR/opentsdb/build.sh; +/bin/mkdir $INSTALL_DIR/opentsdb/build/cache; +export HBASE_HOME=$INSTALL_DIR/hbase-1.2.5; +export COMPRESSION=NONE; +$INSTALL_DIR/opentsdb/src/create_table.sh; +$INSTALL_DIR/opentsdb/build/tsdb tsd --config=$INSTALL_DIR/opentsdb/src/opentsdb.conf --staticroot=$INSTALL_DIR/opentsdb/build/staticroot --cachedir=$INSTALL_DIR/opentsdb/build/cache --port=4242 --zkquorum=localhost:2181 --zkbasedir=/hbase --auto-metric & +cd $INSTALL_DIR +/usr/bin/git clone https://github.com/OpenTSDB/tcollector.git; +cd $INSTALL_DIR/tcollector/tcollector +/bin/rm -rf $INSTALL_DIR/tcollector/collectors/0/*; +/usr/bin/curl -q https://raw.githubusercontent.com/aalpern/tcollector-osx/master/dfstat.py -o $INSTALL_DIR/tcollector/collectors/0/dfstat.py 2>/dev/null; +/usr/bin/curl -q https://raw.githubusercontent.com/aalpern/tcollector-osx/master/iostat.py -o $INSTALL_DIR/tcollector/collectors/0/iostat.py 2>/dev/null; +/usr/bin/curl -q https://raw.githubusercontent.com/aalpern/tcollector-osx/master/vmstat.py -o $INSTALL_DIR/tcollector/collectors/0/vmstat.py 2>/dev/null; +/bin/chmod a+x collectors/0/*.py; +$INSTALL_DIR/tcollector/tcollector.py -L localhost:4242 -t host=`hostname` -t domain=dev -P $INSTALL_DIR/tcollector/tcollector.pid --logfile $INSTALL_DIR/tcollector/tcollector.log & +/bin/sleep 30; +/usr/bin/open http://localhost:4242/#start=10m-ago\&m=sum:df.inodes.free\&autoreload=15; diff --git a/tools/repair-tsd b/tools/repair-tsd new file mode 100755 index 0000000000..0d96742644 --- /dev/null +++ b/tools/repair-tsd @@ -0,0 +1,304 @@ +#!/usr/bin/env python3 + +from subprocess import Popen, PIPE, TimeoutExpired, check_output +from random import shuffle +from pprint import pformat +import signal +import os +from multiprocessing import Pool, Value, cpu_count +import copy +import time +from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter +import logging +import math + +log = logging.getLogger("repair-tsd") +log.setLevel(logging.INFO) +ch = logging.StreamHandler() +logformat = '%(asctime)s %(name)s %(levelname)s %(message)s' +formatter = logging.Formatter(logformat) +ch.setFormatter(formatter) +log.addHandler(ch) +metric_count = Value('i', 0) +failed_count = Value('i', 0) + + +def get_large_divisors(num): + """ + Get "large" divisors of num + i.e. only numbers that, when multiplied, are > than num + + e.g. 60 -> 10, 12, 15, 20, 30, 60 + + * Because while 1, 2, 3, 4, 6 are all divisors, they are too small + when mutiplied together + + :param int num: The number to check + :returns: all large divisors + :rtype: list + """ + return [int(num / i) for i in range(1, int(math.sqrt(num)) + 1) + if num % i == 0 and i * i != num] + + +def get_metrics(args): + """ + Collect all metrics from OpenTSDB + + :returns: all metrics + :rtype: list + """ + time_chunk = args.get("time_chunk", 60) + multiplier = int(60 / time_chunk) + time_range = args.get("time_range", 48) + tsd_path = args.get("tsd_path", "/usr/share/opentsdb/bin/tsdb") + cfg_path = args.get("cfg_path", "/etc/opentsdb/opentsdb.conf") + use_sudo = args.get("use_sudo", False) + sudo_user = args.get("sudo_user", "opentsdb") + base = "{} fsck --config={}".format(tsd_path, cfg_path) + check_cmd = "{} uid --config={} grep metrics".format(tsd_path, cfg_path) + if use_sudo: + base = "sudo -u {} {}".format(sudo_user, base) + check_cmd = "sudo -u {} {}".format(sudo_user, check_cmd) + cmd = '{} uid --config={} grep metrics ".*"'.format(tsd_path, + cfg_path) + proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) + results = proc.communicate() + metriclist = [m.split(" ")[1].strip(":") + for m in results[0].decode().split("\n") if m] + metriclist = [m for m in metriclist if m and m != "\x00"] + metricobj = {"time_chunk": time_chunk, + "timeout": int((time_chunk * 60) / 2), + "retries": args.get("retries", 1), + "compact": args.get("compact", False), + "multiplier": multiplier, + "metriccount": len(metriclist), + "chunk_count": time_range * multiplier, + "base": base, "check_cmd": check_cmd} + metrics = [] + for m in metriclist: + metric = copy.deepcopy(metricobj) + metric["metric"] = m + metrics.append(metric) + if args.get("shuffle", False): + shuffle(metrics) + log.info("There are {} metrics to process".format(len(metrics))) + return metrics + + +def _process_metric_chunk(metric, chunk, x, cmd, timeout): + """ + Actually run the cli command to repair this chunk. + we segment this so the calls to both the compact and + regular runs can be somewhat consistently managed + + :param str metric: name of the metric we're processing + :param int chunk: Which time segment we're processing + :param int x: which attempt for the time segment we're on + :param str cmd: The actual command we'll run + :param int timeout: How long the command is allowed to run + :returns: whether the command was successful + :rtype: bool + """ + log.debug("Running command: {}".format(cmd)) + metricproc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, + preexec_fn=os.setsid) + try: + results, err = metricproc.communicate(timeout=timeout) + except TimeoutExpired: + msg = "{}: chunk {} failed (timeout: {}) (run {})".format(metric, + chunk, + timeout, + x) + log.warning(msg) + try: + os.killpg(os.getpgid(metricproc.pid), signal.SIGTERM) + except Exception as e: + log.warning("Couldn't kill subprocess :: {}".format(e)) + return False + except Exception as e: + log.error("{} general exception :: {}".format(metric, e)) + try: + os.killpg(os.getpgid(metricproc.pid), signal.SIGTERM) + except Exception as e: + log.warning("Couldn't kill subprocess :: {}".format(e)) + return False + results = [r for r in results.decode().split("\n") if r][-26:] + final_results = [] + """ + We'll only collect results that are non-0 + since we're not super interested in stuff that didn't change. + """ + for r in results: + # Strip the timestamp from the log line + line = r.split(" ")[6:] + try: + if int(line[-1]) != 0: + final_results.append(" ".join(line)) + except Exception: + final_results.append(" ".join(line)) + result_str = "\n".join(final_results) + log.debug("{} results:\n{}".format(metric, result_str)) + return True + + +def repair_metric_chunk(metricobj, chunk): + """ + Repair one 'chunk' of data for a metric + """ + metric = metricobj["metric"] + time_chunk = metricobj["time_chunk"] + base = metricobj["base"] + timeout = metricobj["timeout"] + chunk_count = metricobj["chunk_count"] + compact = metricobj["compact"] + log.debug("Running chunk {} for {}".format(chunk, metric)) + if chunk < 2: + timestr = "{}m-ago".format(time_chunk) + else: + timestr = "{}m-ago {}m-ago".format((chunk + 1) * time_chunk, + chunk * time_chunk) + cmd = "{} {} sum".format(base, timestr) + fullcmd = "{} {} --delete-bad-compacts --delete-bad-rows \ + --delete-bad-values --delete-unknown-columns \ + --delete-orphans --resolve-duplicates --fix".format(cmd, + metric) + ccmd = "{} {} --fix --compact".format(cmd, metric) + """ + Even though we're chunking, it's worth trying things more than once + """ + for x in range(1, metricobj["retries"] + 2): + log.debug("Repair try {} for {}".format(x, timestr)) + if not _process_metric_chunk(metric, chunk, x, fullcmd, timeout * x): + continue + if compact: + if not _process_metric_chunk(metric, chunk, x, ccmd, timeout * x): + continue + if chunk % 20 == 0: + log.info("{} -> Chunk {} of {} finished".format(metric, + chunk, + chunk_count)) + return None + else: + log.error("Failed to completely repair {}".format(metric)) + return metric + + +def process_metric(metricobj): + """ + Run fsck on a list of metrics over a time range + """ + metric = metricobj["metric"] + chunk_count = metricobj["chunk_count"] + try: + check_output("{} \"^{}$\"".format(metricobj["check_cmd"], metric), + shell=True) + except Exception: + log.warning("{} doesn't exist! Skipping...".format(metric)) + return None + log.info("Repairing {} in {} chunks".format(metric, chunk_count)) + start_time = time.time() + for x in range(1, chunk_count + 1): + failed = repair_metric_chunk(metricobj, x) + if failed: + with failed_count.get_lock(): + failed_count.value += 1 + return metric + runtime = time.time() - start_time + with metric_count.get_lock(): + metric_count.value += 1 + line = "COMPLETE: {} repair took {} seconds".format(metric, + int(runtime)) + line += " ({} of {} metrics complete)".format(metric_count.value, + metricobj["metriccount"]) + line += " ({} failed)".format(failed_count.value) + log.info(line) + + +def process_metrics(metric_list, threads): + threads = Pool(threads) + failed_metrics = threads.map(process_metric, metric_list) + failed_metrics = [m for m in failed_metrics if m] + return failed_metrics + + +def cli_opts(): + parser = ArgumentParser(description="Repair all OpenTSDB metrics", + formatter_class=ArgumentDefaultsHelpFormatter) + parser.add_argument("--debug", action="store_true", default=False, + help="Show debug information") + parser.add_argument("--time-range", default="48", + help="How many hours of time we collect to repair") + parser.add_argument("--time-chunk", default="60", + help="How many minutes of data to scan per chunk") + parser.add_argument("--retries", default="1", + help="How many times we should try failed metrics") + parser.add_argument("--tsd-path", default="/usr/share/opentsdb/bin/tsdb", + help="Path to the OpenTSDB CLI binary") + parser.add_argument("--cfg-path", default="/etc/opentsdb/opentsdb.conf", + help="Path to OpenTSDB config") + parser.add_argument("--use-sudo", action="store_true", + default=False, + help="switch user when running repairs?") + parser.add_argument("--compact", action="store_true", + default=False, + help="Run compaction with repairs") + parser.add_argument("--shuffle", action="store_true", + default=False, + help="Mix up incoming metric order") + parser.add_argument("--threads", default="{}".format(int(cpu_count() / 2)), + help="Total number of metrics to process at once") + parser.add_argument("--sudo-user", default="opentsdb", + help="User to switch to...") + return parser.parse_args() + + +def main(): + args = cli_opts() + chunks = get_large_divisors(60) + if args.debug: + log.setLevel(logging.DEBUG) + try: + time_range = int(args.time_range) + except Exception as e: + log.error("Invalid time range {} :: {}".format(args.time_range, e)) + try: + retries = int(args.retries) + except Exception as e: + log.error("Invalid retry number {} :: {}".format(args.retries, e)) + try: + threads = int(args.threads) + except Exception as e: + log.error("Invalid thread count {} :: {}".format(args.threads, e)) + try: + time_chunk = int(args.time_chunk) + if time_chunk < 60: + if time_chunk not in chunks: + raise ArithmeticError + if time_chunk > 60: + if not any(n for n in chunks if time_chunk % chunk == 0): + raise ArithmeticError + except Exception as e: + log.error("Invalid time chunk {} :: {}".format(args.time_chunk, e)) + + metric_list = get_metrics({"time_range": time_range, + "use_sudo": args.use_sudo, + "sudo_user": args.sudo_user, + "time_chunk": time_chunk, + "tsd_path": args.tsd_path, + "cfg_path": args.cfg_path, + "shuffle": args.shuffle, + "compact": args.compact, + "retries": retries}) + stime = time.time() + failed = process_metrics(metric_list, threads) + etime = time.time() + log.info("Processed {} metrics in [{}] seconds".format(len(metric_list), + etime - stime)) + if len(failed) > 0: + log.warning("{} failed metrics:\n{}".format(len(failed), + pformat(sorted(failed)))) + + +if __name__ == "__main__": + main() diff --git a/tools/tsdb_list_running_queries.py b/tools/tsdb_list_running_queries.py new file mode 100755 index 0000000000..466e6caf17 --- /dev/null +++ b/tools/tsdb_list_running_queries.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python +# pylint: disable=line-too-long,missing-docstring +# +# List the running queries on the local TSD, sorted ascending by age, with normalized start time, end time, and time range in secs + +from __future__ import print_function +from __future__ import division + +import httplib +import json +import os +import socket +import sys +import time + + +class ConnectionException(Exception): + pass + + +class OpenTSDBListRunningQueries(object): + + format_string = '{:<19}\t{:<10}\t{:<19}\t{:<19}\t{:<16}\t{:<10}\t{:<10}\t{}' + + # used by ms_to_secs() method further down to compare the epoch and figure out if it is in secs or ms and normalize it + # pulling this out of the iteratively called method is a little more efficient to not recalculate this for every query in the list + now_length = len(str(int(time.time()))) + + timestamp_multiplier = { + 'ms': 0.001, + 's': 1, + 'm': 60, + 'h': 3600, + 'd': 86400, + 'w': 7 * 86400, + 'n': 30 * 86400, + 'y': 365 * 86400 + } + + def __init__(self): + self.host = 'localhost' + self.port = 4242 + self.uri = '/api/stats/query' + self.server = httplib.HTTPConnection(self.host, self.port) + self.server.auto_open = True + + def request(self): + try: + self.server.request('GET', self.uri) + resp = self.server.getresponse().read() + self.server.close() + except socket.error as _: + raise ConnectionException('Socket Error querying TSDB port: {}'.format(_)) + except httplib.HTTPException as _: + raise ConnectionException('HTTP Error querying TSDB port: {}'.format(_)) + return json.loads(resp) + + # reference_timestamp is for comparing N-ago timestamps + def convert_timestamp_to_epoch(self, timestamp, reference_timestamp): + timestamp = str(timestamp).split('.')[0] + reference_timestamp_struct = time.strptime(reference_timestamp, '%Y-%m-%d %H:%M:%S') # %z not supported in the C runtime, and no workaround until Python 3.2 + reference_timestamp_secs = time.mktime(reference_timestamp_struct) + if '/' in timestamp: + timestamp = time.strptime(timestamp, '%Y/%m/%d-%H:%M:%S') + timestamp = time.mktime(timestamp) + elif timestamp == 'now': + timestamp = reference_timestamp_secs + elif timestamp[-4:] == '-ago': + timestamp = timestamp[:-4] + (ago, multiplier) = (timestamp[:-1], timestamp[-1]) + secs_ago = int(ago) * self.timestamp_multiplier[multiplier] + timestamp = int(reference_timestamp_secs) - secs_ago + timestamp = int(float(timestamp)) + timestamp = self.ms_to_secs(timestamp) + return timestamp + + def convert_human_time(self, epoch): + epoch = self.ms_to_secs(epoch) + human_time = time.strftime('%F %T', time.gmtime(float(epoch))) + #print('converted epoch {} to {}'.format(epoch, human_time)) + return human_time + + def ms_to_secs(self, epoch): + # convert from ms to secs if epoch is in ms + if len(str(epoch)) > self.now_length: + epoch = int(int(epoch) / 1000) + return epoch + + def process_query(self, query): + if os.getenv('DEBUG'): + print(json.dumps(query, indent=4, sort_keys=True)) + user = query.get('headers', {}).get('X-WEBAUTH-USER', '') + querystart = query.get('queryStart', '') + querystart_secs = self.ms_to_secs(querystart) + querystart = self.convert_human_time(querystart_secs) + query = query['query'] + start = query['start'] + start_epoch = self.ms_to_secs(self.convert_timestamp_to_epoch(start, querystart)) + start = self.convert_human_time(start_epoch) + end = query.get('end', '') + if end: + end_epoch = self.ms_to_secs(self.convert_timestamp_to_epoch(end, querystart)) + else: + end_epoch = int(time.mktime(time.strptime(querystart, '%Y-%m-%d %H:%M:%S'))) + end = self.convert_human_time(end_epoch) + timerange_secs = end_epoch - start_epoch + running_subquery_count = 0 + for subquery in query.get('queries', []): + metric = subquery.get('metric', '') + aggregator = subquery.get('aggregator') + downsample = subquery.get('downsample') + print(self.format_string.format(querystart, user, start, end, timerange_secs, aggregator, downsample, metric)) + running_subquery_count += 1 + return running_subquery_count + + def main(self): + stats = self.request() + running_queries = stats.get('running', []) + print('='*160) + print(self.format_string.format('Date', 'User', 'Start', 'End', 'TimeRange (Secs)', 'Aggregator', 'Downsample', 'Metric')) + print('='*160 + '\n') + running_queries.sort(key=lambda x: int(x['queryStart']), reverse=False) + running_subquery_count = 0 + for query in running_queries: + running_subquery_count += self.process_query(query) + running_query_count = len(running_queries) + print('\nListed {} running queries, {} individual subqueries'.format(running_query_count, running_subquery_count)) + sys.stdout.flush() + return 0 + + +if __name__ == "__main__": + try: + OpenTSDBListRunningQueries().main() + except ConnectionException as _: + print(_, file=sys.stderr) + sys.exit(2) + except KeyboardInterrupt: + print("Control-C, aborting...") + sys.exit(3) diff --git a/tools/tsddrain.py b/tools/tsddrain.py index 9a0fcf9b59..8bd595447c 100755 --- a/tools/tsddrain.py +++ b/tools/tsddrain.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # # This little script can be used to replace TSDs while performing prolonged # HBase or HDFS maintenances. It runs a simple, low-end TCP server to accept diff --git a/tsdb.in b/tsdb.in index cfcdec556a..346ebbde8a 100644 --- a/tsdb.in +++ b/tsdb.in @@ -6,8 +6,8 @@ mydir=`dirname "$0"` # Either: # abs_srcdir and abs_builddir are set: we're running in a dev tree # or pkgdatadir is set: we've been installed, we respect that. -abs_srcdir='@abs_srcdir@' -abs_builddir='@abs_builddir@' +abs_srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.." +abs_builddir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" pkgdatadir='@pkgdatadir@' configdir='@configdir@' # Either we've been installed and pkgdatadir exists, or we haven't been @@ -62,7 +62,7 @@ CLASSPATH="${CLASSPATH#:}" usage() { echo >&2 "usage: $me <command> [args]" - echo 'Valid commands: fsck, import, mkmetric, query, tsd, scan, uid' + echo 'Valid commands: fsck, import, mkmetric, query, tsd, scan, search, uid, version' exit 1 } @@ -93,6 +93,9 @@ case $1 in (uid) MAINCLASS=UidManager ;; + (version) + MAINCLASS=BuildData + ;; (*) echo >&2 "$me: error: unknown command '$1'" usage @@ -103,4 +106,13 @@ shift JAVA=${JAVA-'java'} JVMARGS=${JVMARGS-'-enableassertions -enablesystemassertions'} test -r "$localdir/tsdb.local" && . "$localdir/tsdb.local" -exec $JAVA $JVMARGS -classpath "$CLASSPATH" net.opentsdb.tools.$MAINCLASS "$@" + +if [[ $CLASSPATH == *"asyncbigtable"* ]] +then + USE_BIGTABLE=1 + echo "Running OpenTSDB with Bigtable support" + + exec $JAVA $JVMARGS -classpath "$CLASSPATH:$HBASE_CONF" net.opentsdb.tools.$MAINCLASS "$@" +else + exec $JAVA $JVMARGS -classpath "$CLASSPATH" net.opentsdb.tools.$MAINCLASS "$@" +fi