Repository navigation
REQ: allocate compactor storage on first use, and add reserve() #575
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
4e42b3f
84f79fc
9b9287e
fd662dd
ccbcf6a
1267c75
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -173,6 +173,7 @@ template<typename T, typename C, typename A> | |
| template<typename FwdT> | ||
| void req_sketch<T, C, A>::update(FwdT&& item) { | ||
| if (!check_update_item(item)) { return; } | ||
| compactors_[0].reserve(); // first, so that a failure leaves the sketch unchanged | ||
| if (is_empty()) { | ||
| min_item_.emplace(item); | ||
| max_item_.emplace(item); | ||
|
|
@@ -183,33 +184,45 @@ void req_sketch<T, C, A>::update(FwdT&& item) { | |
| compactors_[0].append(std::forward<FwdT>(item)); | ||
| ++num_retained_; | ||
| ++n_; | ||
| if (num_retained_ == max_nom_size_) compress(); | ||
| reset_sorted_view(); | ||
| if (num_retained_ >= max_nom_size_) compress(); | ||
| } | ||
|
|
||
| template<typename T, typename C, typename A> | ||
| template<typename FwdSk> | ||
| void req_sketch<T, C, A>::merge(FwdSk&& other) { | ||
| if (is_HRA() != other.is_HRA()) throw std::invalid_argument("merging HRA and LRA is not valid"); | ||
| if (other.is_empty()) return; | ||
| if (is_empty()) { | ||
| min_item_.emplace(conditional_forward<FwdSk>(*other.min_item_)); | ||
| max_item_.emplace(conditional_forward<FwdSk>(*other.max_item_)); | ||
| } else { | ||
| if (comparator_(*other.min_item_, *min_item_)) *min_item_ = conditional_forward<FwdSk>(*other.min_item_); | ||
| if (comparator_(*max_item_, *other.max_item_)) *max_item_ = conditional_forward<FwdSk>(*other.max_item_); | ||
| } | ||
| // grow until this has at least as many compactors as other | ||
| while (get_num_levels() < other.get_num_levels()) grow(); | ||
| // merge the items in all height compactors | ||
| for (size_t i = 0; i < other.get_num_levels(); ++i) { | ||
| compactors_[i].merge(conditional_forward<FwdSk>(other.compactors_[i])); | ||
| reset_sorted_view(); | ||
| const size_t num_levels = compactors_.size(); | ||
| try { | ||
| // grow until this has at least as many compactors as other | ||
| while (get_num_levels() < other.get_num_levels()) grow(); | ||
| if (is_empty()) { | ||
| min_item_.emplace(conditional_forward<FwdSk>(*other.min_item_)); | ||
| max_item_.emplace(conditional_forward<FwdSk>(*other.max_item_)); | ||
| } else { | ||
| if (comparator_(*other.min_item_, *min_item_)) *min_item_ = conditional_forward<FwdSk>(*other.min_item_); | ||
| if (comparator_(*max_item_, *other.max_item_)) *max_item_ = conditional_forward<FwdSk>(*other.max_item_); | ||
| } | ||
| // merge the items in all height compactors | ||
| for (size_t i = 0; i < other.get_num_levels(); ++i) { | ||
| compactors_[i].merge(conditional_forward<FwdSk>(other.compactors_[i])); | ||
| } | ||
| } catch (...) { | ||
| remove_empty_levels_above(num_levels); | ||
| restore_counts(); // some levels may be merged already | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. In here, we should remove last compactor. When grow is success and merge failed, then sketch's state is invalid.
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You're right, thanks. If Fixed in ccbcf6a: on failure, |
||
| throw; | ||
| } | ||
| n_ += other.n_; | ||
| update_max_nom_size(); | ||
| update_num_retained(); | ||
| if (num_retained_ >= max_nom_size_) compress(); | ||
| reset_sorted_view(); | ||
| } | ||
|
|
||
| template<typename T, typename C, typename A> | ||
| void req_sketch<T, C, A>::reserve() { | ||
| compactors_[0].reserve(); | ||
| } | ||
|
|
||
| template<typename T, typename C, typename A> | ||
|
|
@@ -631,8 +644,34 @@ void req_sketch<T, C, A>::update_num_retained() { | |
| for (const auto& compactor: compactors_) num_retained_ += compactor.get_num_items(); | ||
| } | ||
|
|
||
| template<typename T, typename C, typename A> | ||
| void req_sketch<T, C, A>::restore_counts() { | ||
| // the total weight of the retained items is always n | ||
| n_ = 0; | ||
| for (const auto& compactor: compactors_) n_ += static_cast<uint64_t>(compactor.get_num_items()) << compactor.get_lg_weight(); | ||
| update_num_retained(); | ||
| update_max_nom_size(); | ||
| } | ||
|
|
||
| template<typename T, typename C, typename A> | ||
| void req_sketch<T, C, A>::compress() { | ||
| const size_t num_levels = compactors_.size(); | ||
| try { | ||
| compress_levels(); | ||
| } catch (...) { | ||
| remove_empty_levels_above(num_levels); // a level can be added, and then the compaction into it fail | ||
| restore_counts(); // a compactor can fail to grow after a compaction moved its items | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. In here we should also reset compactors. Because
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Right, same problem as in Fixed in 1267c75: on failure |
||
| throw; | ||
| } | ||
| } | ||
|
|
||
| template<typename T, typename C, typename A> | ||
| void req_sketch<T, C, A>::remove_empty_levels_above(size_t num_levels) { | ||
| while (compactors_.size() > num_levels && compactors_.back().get_num_items() == 0) compactors_.pop_back(); | ||
| } | ||
|
|
||
| template<typename T, typename C, typename A> | ||
| void req_sketch<T, C, A>::compress_levels() { | ||
| for (size_t h = 0; h < compactors_.size(); ++h) { | ||
| if (compactors_[h].get_num_items() >= compactors_[h].get_nom_capacity()) { | ||
| if (h == 0) compactors_[0].sort(); | ||
|
|
@@ -791,15 +830,23 @@ template<typename T, typename C, typename A> | |
| req_sketch<T, C, A>::const_iterator::const_iterator(LevelsIterator begin, LevelsIterator end): | ||
| levels_it_(begin), | ||
| levels_end_(end), | ||
| compactor_it_(begin == end ? nullptr : (*levels_it_).begin()) | ||
| {} | ||
| compactor_it_(nullptr) | ||
| { | ||
| skip_empty_levels(); | ||
| } | ||
|
|
||
| template<typename T, typename C, typename A> | ||
| void req_sketch<T, C, A>::const_iterator::skip_empty_levels() { | ||
| while (levels_it_ != levels_end_ && (*levels_it_).get_num_items() == 0) ++levels_it_; | ||
| if (levels_it_ != levels_end_) compactor_it_ = (*levels_it_).begin(); | ||
| } | ||
|
|
||
| template<typename T, typename C, typename A> | ||
| auto req_sketch<T, C, A>::const_iterator::operator++() -> const_iterator& { | ||
| ++compactor_it_; | ||
| if (compactor_it_ == (*levels_it_).end()) { | ||
| ++levels_it_; | ||
| if (levels_it_ != levels_end_) compactor_it_ = (*levels_it_).begin(); | ||
| skip_empty_levels(); | ||
| } | ||
| return *this; | ||
| } | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.