From 1212664deee412f71cbc44db8c2dc4168e73fc12 Mon Sep 17 00:00:00 2001 From: Jacob Faibussowitsch Date: Mon, 2 Nov 2026 13:29:55 -0500 Subject: [PATCH 1/5] Add cccl-hide-from-abi --- ci/matrix.yaml | 1 + cmake/clang_tidy_plugins/CMakeLists.txt | 1 + .../avoid_internal_linkage_in_headers.cpp | 105 +------ cmake/clang_tidy_plugins/common/lexer.h | 103 +++++++ cmake/clang_tidy_plugins/common/matchers.h | 25 ++ .../hide_from_abi/CMakeLists.txt | 5 + .../hide_from_abi/hide_from_abi.cpp | 257 +++++++++++++++++ .../hide_from_abi/test/CMakeLists.txt | 21 ++ .../hide_from_abi/test/execution_spaces.hpp | 97 +++++++ .../hide_from_abi/test/functions.hpp | 258 ++++++++++++++++++ 10 files changed, 772 insertions(+), 101 deletions(-) create mode 100644 cmake/clang_tidy_plugins/common/lexer.h create mode 100644 cmake/clang_tidy_plugins/common/matchers.h create mode 100644 cmake/clang_tidy_plugins/hide_from_abi/CMakeLists.txt create mode 100644 cmake/clang_tidy_plugins/hide_from_abi/hide_from_abi.cpp create mode 100644 cmake/clang_tidy_plugins/hide_from_abi/test/CMakeLists.txt create mode 100644 cmake/clang_tidy_plugins/hide_from_abi/test/execution_spaces.hpp create mode 100644 cmake/clang_tidy_plugins/hide_from_abi/test/functions.hpp diff --git a/ci/matrix.yaml b/ci/matrix.yaml index f49b76bd9602..6ff4837e2e3b 100644 --- a/ci/matrix.yaml +++ b/ci/matrix.yaml @@ -87,6 +87,7 @@ workflows: # # IMPORTANT: Do NOT delete or remove the `override:` key below, even when it is empty. override: + - { jobs: ['build'], project: 'tidy', std: 'min', cxx: ['clang_preview-cuda'], cudacxx: ['clang'], ctk: 'clang_preview-cuda', sm: '75' } pull_request_fork: - {jobs: ['build'], project: ['libcudacxx', 'thrust', 'cudax'], std: 'max', cxx: ['gcc', 'clang', 'msvc']} diff --git a/cmake/clang_tidy_plugins/CMakeLists.txt b/cmake/clang_tidy_plugins/CMakeLists.txt index aa52bff161d4..f01701656234 100644 --- a/cmake/clang_tidy_plugins/CMakeLists.txt +++ b/cmake/clang_tidy_plugins/CMakeLists.txt @@ -273,3 +273,4 @@ add_subdirectory(modernize_type_traits) add_subdirectory(prefer_cuda_traits) add_subdirectory(simplify_type_traits) add_subdirectory(nodebug_on_aliases) +add_subdirectory(hide_from_abi) diff --git a/cmake/clang_tidy_plugins/avoid_internal_linkage_in_headers/avoid_internal_linkage_in_headers.cpp b/cmake/clang_tidy_plugins/avoid_internal_linkage_in_headers/avoid_internal_linkage_in_headers.cpp index ebc7559c43f1..d1b94036c120 100644 --- a/cmake/clang_tidy_plugins/avoid_internal_linkage_in_headers/avoid_internal_linkage_in_headers.cpp +++ b/cmake/clang_tidy_plugins/avoid_internal_linkage_in_headers/avoid_internal_linkage_in_headers.cpp @@ -6,8 +6,6 @@ #include #include #include -#include -#include #include #include #include @@ -15,9 +13,10 @@ #include #include #include +#include +#include #include #include -#include namespace clang::tidy::cccl { @@ -25,19 +24,6 @@ namespace { using namespace ast_matchers; // NOLINT(google-build-using-namespace) -AST_MATCHER_P(Decl, isExpansionInHeaderFile, const FileExtensionsSet*, extensions) -{ - static_cast(Builder); - static_cast(Finder); - - const auto begin = Node.getBeginLoc(); - const auto& sm = Finder->getASTContext().getSourceManager(); - const auto filename = sm.getFilename(sm.getExpansionLoc(begin)); - // LLVM's header utility rejects extensionless files even when the empty extension is configured. - return utils::isExpansionLocInHeaderFile(begin, sm, *extensions) - || (extensions->contains("") && !filename.empty() && llvm::sys::path::extension(filename).empty()); -} - AST_MATCHER(VarDecl, isFileVarDecl) { static_cast(Builder); @@ -65,89 +51,6 @@ AST_MATCHER(Decl, hasExplicitGlobalAttr) return attr && !attr->isImplicit(); } -#if LLVM_VERSION_MAJOR >= 23 -[[nodiscard]] CharSourceRange findTokenTextInRange( - CharSourceRange Range, - const SourceManager& SM, - const LangOptions& LangOpts, - llvm::function_ref Pred) -{ - return utils::lexer::findTokenTextInRange(std::move(Range), SM, LangOpts, std::move(Pred)); -} -#else -// Word-for-word backport of findTokenTextInRange() -[[nodiscard]] CharSourceRange findTokenTextInRange( - CharSourceRange Range, - const SourceManager& SM, - const LangOptions& LangOpts, - llvm::function_ref Pred) -{ - if (Range.isInvalid()) - { - return {}; - } - - // Normalize to a file-based char range so raw lexing can operate on one - // contiguous buffer and reject unmappable (e.g. macro) ranges. - const CharSourceRange FileRange = Lexer::makeFileCharRange(Range, SM, LangOpts); - - if (FileRange.isInvalid()) - { - return {}; - } - - const auto [BeginFID, BeginOffset] = SM.getDecomposedLoc(FileRange.getBegin()); - const auto [EndFID, EndOffset] = SM.getDecomposedLoc(FileRange.getEnd()); - - if (BeginFID != EndFID || BeginOffset > EndOffset) - { - return {}; - } - - bool Invalid = false; - const StringRef Buffer = SM.getBufferData(BeginFID, &Invalid); - if (Invalid) - { - return {}; - } - - const char* LexStart = Buffer.data() + BeginOffset; - // Re-lex raw tokens in the bounded file buffer while preserving comments so - // callers can match tokens regardless of interleaved comments. - Lexer TheLexer{SM.getLocForStartOfFile(BeginFID), LangOpts, Buffer.begin(), LexStart, Buffer.end()}; - TheLexer.SetCommentRetentionState(true); - - while (true) - { - Token Tok{}; - if (TheLexer.LexFromRawLexer(Tok)) - { - return {}; - } - - if (Tok.is(tok::eof) || Tok.getLocation() == FileRange.getEnd() - || SM.isBeforeInTranslationUnit(FileRange.getEnd(), Tok.getLocation())) - { - return {}; - } - - if (!Pred(Tok)) - { - continue; - } - - Token NextTok{}; - if (TheLexer.LexFromRawLexer(NextTok)) - { - return {}; - } - // Return a char range ending at the next token start so trailing trivia of - // the matched token is included (useful for fix-it removals). - return CharSourceRange::getCharRange(Tok.getLocation(), NextTok.getLocation()); - } -} -#endif - class AvoidInternalLinkageInHeadersCheck final : public ClangTidyCheck { static constexpr StringRef FUNCTION_BIND{"f"}; @@ -171,7 +74,7 @@ class AvoidInternalLinkageInHeadersCheck final : public ClangTidyCheck // misc-anonymous-namespace-in-header already diagnoses anonymous namespaces. const auto header_decl = allOf(unless(isExpansionInSystemHeader()), - isExpansionInHeaderFile(header_extensions_), + util::isExpansionInHeaderFile(header_extensions_), unless(isInAnonymousNamespace())); finder->addMatcher( @@ -250,7 +153,7 @@ class AvoidInternalLinkageInHeadersCheck final : public ClangTidyCheck { const auto& sm = context.getSourceManager(); const auto& lang_opts = context.getLangOpts(); - const auto range = findTokenTextInRange( + const auto range = util::findTokenTextInRange( CharSourceRange::getTokenRange(declaration.getBeginLoc(), declaration.getLocation()), sm, lang_opts, diff --git a/cmake/clang_tidy_plugins/common/lexer.h b/cmake/clang_tidy_plugins/common/lexer.h new file mode 100644 index 000000000000..41cc494012a8 --- /dev/null +++ b/cmake/clang_tidy_plugins/common/lexer.h @@ -0,0 +1,103 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace clang::tidy::cccl::util +{ +#if LLVM_VERSION_MAJOR >= 23 +[[nodiscard]] inline CharSourceRange findTokenTextInRange( + CharSourceRange Range, + const SourceManager& SM, + const LangOptions& LangOpts, + llvm::function_ref Pred) +{ + return utils::lexer::findTokenTextInRange(std::move(Range), SM, LangOpts, std::move(Pred)); +} +#else +// Backport of https://github.com/llvm/llvm-project/blob/main/clang-tools-extra/clang-tidy/utils/LexerUtils.cpp. +[[nodiscard]] inline CharSourceRange findTokenTextInRange( + CharSourceRange Range, + const SourceManager& SM, + const LangOptions& LangOpts, + llvm::function_ref Pred) +{ + if (Range.isInvalid()) + { + return {}; + } + + // Normalize to a file-based char range so raw lexing can operate on one + // contiguous buffer and reject unmappable (e.g. macro) ranges. + const CharSourceRange FileRange = Lexer::makeFileCharRange(Range, SM, LangOpts); + + if (FileRange.isInvalid()) + { + return {}; + } + + const auto [BeginFID, BeginOffset] = SM.getDecomposedLoc(FileRange.getBegin()); + const auto [EndFID, EndOffset] = SM.getDecomposedLoc(FileRange.getEnd()); + + if (BeginFID != EndFID || BeginOffset > EndOffset) + { + return {}; + } + + bool Invalid = false; + const StringRef Buffer = SM.getBufferData(BeginFID, &Invalid); + if (Invalid) + { + return {}; + } + + const char* LexStart = Buffer.data() + BeginOffset; + // Re-lex raw tokens in the bounded file buffer while preserving comments so + // callers can match tokens regardless of interleaved comments. + Lexer TheLexer{SM.getLocForStartOfFile(BeginFID), LangOpts, Buffer.begin(), LexStart, Buffer.end()}; + TheLexer.SetCommentRetentionState(true); + + while (true) + { + Token Tok{}; + if (TheLexer.LexFromRawLexer(Tok)) + { + return {}; + } + + if (Tok.is(tok::eof) || Tok.getLocation() == FileRange.getEnd() + || SM.isBeforeInTranslationUnit(FileRange.getEnd(), Tok.getLocation())) + { + return {}; + } + + if (!Pred(Tok)) + { + continue; + } + + Token NextTok{}; + if (TheLexer.LexFromRawLexer(NextTok)) + { + return {}; + } + // Return a char range ending at the next token start so trailing trivia of + // the matched token is included (useful for fix-it removals). + return CharSourceRange::getCharRange(Tok.getLocation(), NextTok.getLocation()); + } +} +#endif +} // namespace clang::tidy::cccl::util diff --git a/cmake/clang_tidy_plugins/common/matchers.h b/cmake/clang_tidy_plugins/common/matchers.h new file mode 100644 index 000000000000..bd44e680cac3 --- /dev/null +++ b/cmake/clang_tidy_plugins/common/matchers.h @@ -0,0 +1,25 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace clang::tidy::cccl::util +{ +AST_MATCHER_P(Decl, isExpansionInHeaderFile, const FileExtensionsSet*, extensions) // NOLINT +{ + const auto begin = Node.getBeginLoc(); + const auto& sm = Finder->getASTContext().getSourceManager(); + const auto filename = sm.getFilename(sm.getExpansionLoc(begin)); + // LLVM's header utility rejects extensionless files even when the empty extension is configured. + return utils::isExpansionLocInHeaderFile(begin, sm, *extensions) + || (extensions->contains("") && !filename.empty() && llvm::sys::path::extension(filename).empty()); +} +} // namespace clang::tidy::cccl::util diff --git a/cmake/clang_tidy_plugins/hide_from_abi/CMakeLists.txt b/cmake/clang_tidy_plugins/hide_from_abi/CMakeLists.txt new file mode 100644 index 000000000000..228c306cf768 --- /dev/null +++ b/cmake/clang_tidy_plugins/hide_from_abi/CMakeLists.txt @@ -0,0 +1,5 @@ +list(APPEND CMAKE_MESSAGE_CONTEXT "hide_from_abi") + +cccl_create_clang_tidy_plugin(hide_from_abi.cpp) + +add_subdirectory(test) diff --git a/cmake/clang_tidy_plugins/hide_from_abi/hide_from_abi.cpp b/cmake/clang_tidy_plugins/hide_from_abi/hide_from_abi.cpp new file mode 100644 index 000000000000..2e3a129404e1 --- /dev/null +++ b/cmake/clang_tidy_plugins/hide_from_abi/hide_from_abi.cpp @@ -0,0 +1,257 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// Adapted from https://github.com/llvm/llvm-project/blob/main/libcxx/test/tools/clang_tidy_checks/hide_from_abi.cpp. + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace clang::tidy::cccl +{ +namespace +{ +using namespace ast_matchers; // NOLINT(google-build-using-namespace) + +enum class [[clang::flag_enum]] execution_space : std::uint8_t +{ + none = 0, + host = 1 << 1, + device = 1 << 2, + tile = 1 << 3, +}; + +[[nodiscard]] constexpr execution_space operator|(execution_space lhs, execution_space rhs) +{ + return static_cast(llvm::to_underlying(lhs) | llvm::to_underlying(rhs)); +} + +constexpr execution_space& operator|=(execution_space& lhs, execution_space rhs) +{ + lhs = lhs | rhs; + return lhs; +} + +class HideFromABICheck final : public ClangTidyCheck +{ + static constexpr StringRef FUNCTION_BIND{""}; + static constexpr StringRef SPECIAL_FUNCTION_BIND{"s"}; + +public: + HideFromABICheck(StringRef name, ClangTidyContext* context) + : ClangTidyCheck{name, context} + , header_extensions_{std::addressof(context->getHeaderFileExtensions())} + {} + + [[nodiscard]] std::optional getCheckTraversalKind() const override + { + return TK_IgnoreUnlessSpelledInSource; + } + + void registerMatchers(MatchFinder* finder) override + { + const auto is_special_member = cxxMethodDecl( + isDefaulted(), + anyOf(cxxConstructorDecl(), cxxDestructorDecl(), isCopyAssignmentOperator(), isMoveAssignmentOperator())); + + // FunctionDecl includes methods, constructors, destructors, and conversion operators. + finder->addMatcher( + functionDecl( + isDefinition(), + unless(anyOf(isImplicit(), + isDeleted(), + isConsteval(), + hasAttr(attr::Visibility), + cxxMethodDecl(ofClass(cxxRecordDecl(isLambda()))), + isExpansionInSystemHeader())), + util::isExpansionInHeaderFile(header_extensions_), + optionally(is_special_member.bind(SPECIAL_FUNCTION_BIND))) + .bind(FUNCTION_BIND), + this); + } + + void check(const MatchFinder::MatchResult& result) override + { + const auto& nodes = result.Nodes; + const auto* const function = nodes.getNodeAs(FUNCTION_BIND); + const auto& SM = *result.SourceManager; + + if (nodes.getNodeAs(SPECIAL_FUNCTION_BIND)) + { + // Special member functions which are defaulted can't have the usual _CCCL_*_API macros + // because nvcc forbids attaching explicit __host__ __device__ attributes to + // these. Instead they need _CCCL_HIDE_FROM_ABI. + // + // We don't need to do a search because by virtue of being here (the function has no + // visibility attributes) we know that it's missing the macro. + const auto begin = SM.getExpansionLoc(function->getInnerLocStart()); + diag(function->getLocation(), "use _CCCL_HIDE_FROM_ABI to limit ABI visibility") + << FixItHint::CreateInsertion(begin, "_CCCL_HIDE_FROM_ABI "); + return; + } + + handle_regular_functions_(function, SM); + } + +private: + void handle_regular_functions_(const FunctionDecl* fn, const SourceManager& SM) + { + const auto [space, matches] = find_execution_space_(get_annotation_range_(fn, SM), SM); + + if (matches.empty()) + { + // Without an execution-space macro, we cannot choose an API macro for the fix-it. + diag(fn->getLocation(), + "function needs an API macro (one of _CCCL_API, _CCCL_DEVICE_API, or _CCCL_HOST_API) to limit ABI " + "visibility") + << fn->getLocation(); + return; + } + + const auto replacement = get_api_macro_(space); + auto d = diag(matches.front().getBegin(), "use %0 instead to limit ABI visibility") + << replacement << matches.front() + << FixItHint::CreateReplacement( + CharSourceRange::getTokenRange(matches.front().getBegin(), matches.front().getBegin()), replacement); + + for (auto&& m : llvm::drop_begin(matches)) + { + d << m << FixItHint::CreateRemoval(m); + } + } + + [[nodiscard]] static CharSourceRange get_annotation_range_(const FunctionDecl* fn, const SourceManager& SM) + { + // Getting the begin range here is a little fiddly. We are about to textually search the + // macro annotations of the function, but depending on the compilation arguments many of + // those macros define as empty. If we just did the naive thing and used + // `function->getInnerLocStart()`, we would skip over such macro definitions. So instead we + // need to go all the way back to the *end* of either the template parameter list or + // constraint list (if any). So basically: + // + // #define EMPTY_MACRO /* empty */ + // + // fn->getDescribedFunctionTemplate()->getTemplateParameters()->getRAngleLoc() + // v ...->getRequiresClause()->getEndLoc() + // template v + // requires my_requirement + // EMPTY_MACRO inline [[some::other_attribute]] void foo() + // ^ + // fn->getInnerLocStart() + auto begin = fn->getInnerLocStart(); + + if (const auto* const templ = fn->getDescribedFunctionTemplate()) + { + if (const auto* const req = templ->getTemplateParameters()->getRequiresClause()) + { + begin = req->getEndLoc(); + } + else + { + begin = templ->getTemplateParameters()->getRAngleLoc(); + } + } + + return CharSourceRange::getCharRange(SM.getExpansionLoc(begin), SM.getExpansionLoc(fn->getLocation())); + } + + [[nodiscard]] std::pair> + find_execution_space_(CharSourceRange range, const SourceManager& SM) const + { + const auto& lang_opts = getLangOpts(); + auto combined_space = execution_space::none; + auto matches = SmallVector{}; + + for (auto remaining = range;;) + { + const auto match = util::findTokenTextInRange(remaining, SM, lang_opts, [&](const Token& token) { + const auto spelling = Lexer::getSpelling(token, SM, lang_opts); + const auto space = + llvm::StringSwitch{spelling} + .Cases({"_CCCL_DEVICE", "__device__"}, execution_space::device) + .Cases({"_CCCL_HOST", "__host__"}, execution_space::host) + .Cases({"_CCCL_TILE", "__tile__"}, execution_space::tile) + .Case("_CCCL_HOST_DEVICE", execution_space::host | execution_space::device) + .Default(execution_space::none); + + combined_space |= space; + return space != execution_space::none; + }); + + if (match.isInvalid()) + { + break; + } + + matches.push_back(match); + remaining = CharSourceRange::getCharRange(match.getEnd(), range.getEnd()); + } + + return {combined_space, std::move(matches)}; + } + + [[nodiscard]] static StringRef get_api_macro_(execution_space space) + { + switch (space) + { + case execution_space::none: + // The caller rejects functions without recognized execution-space annotations. + llvm::reportFatalUsageError("cccl-hide-from-abi: we should not get exec space none here"); + case execution_space::host: + return "_CCCL_HOST_API"; + case execution_space::device: + return "_CCCL_DEVICE_API"; + case execution_space::tile: + return "_CCCL_TILE_API"; + case execution_space::host | execution_space::device: + return "_CCCL_HOST_DEVICE_API"; + case execution_space::device | execution_space::tile: + // We should really have a _CCCL_TILE_DEVICE_API macro for this + return "_CCCL_TILE _CCCL_DEVICE_API"; + case execution_space::host | execution_space::device | execution_space::tile: + return "_CCCL_API"; + default: + const auto err = "cccl-hide-from-abi: unhandled execution space " + std::to_string(llvm::to_underlying(space)); + + llvm::reportFatalUsageError(StringRef{err}); + } + } + + const FileExtensionsSet* header_extensions_{}; +}; + +class HideFromABICheckModule final : public ClangTidyModule +{ +public: + static constexpr StringRef CHECK_NAME{"cccl-hide-from-abi"}; + + void addCheckFactories(ClangTidyCheckFactories& check_factories) override + { + check_factories.registerCheck(CHECK_NAME); + } +}; + +// NOLINTNEXTLINE(cert-err58-cpp, bugprone-throwing-static-initialization) +ClangTidyModuleRegistry::Add _{ + HideFromABICheckModule::CHECK_NAME, "Checks for missing ABI annotations on function definitions in headers"}; +} // namespace +} // namespace clang::tidy::cccl diff --git a/cmake/clang_tidy_plugins/hide_from_abi/test/CMakeLists.txt b/cmake/clang_tidy_plugins/hide_from_abi/test/CMakeLists.txt new file mode 100644 index 000000000000..3a5a3c598344 --- /dev/null +++ b/cmake/clang_tidy_plugins/hide_from_abi/test/CMakeLists.txt @@ -0,0 +1,21 @@ +list(APPEND CMAKE_MESSAGE_CONTEXT "test") + +file(GLOB test_srcs CONFIGURE_DEPENDS "*.hpp") + +foreach (src IN LISTS test_srcs) + cmake_path(GET src STEM name) + foreach ( + extension + IN + ITEMS "" h hh hpp hxx cuh inl ipp + ) + set(header "${CMAKE_CURRENT_BINARY_DIR}/${name}_${extension}") + if (NOT extension STREQUAL "") + string(APPEND header ".${extension}") + endif() + configure_file("${src}" "${header}" COPYONLY) + # Parse headers directly so the diagnostic line filter includes their declarations. + set_source_files_properties("${header}" PROPERTIES LANGUAGE CXX) + cccl_create_clang_tidy_test(hide_from_abi "${header}") + endforeach() +endforeach() diff --git a/cmake/clang_tidy_plugins/hide_from_abi/test/execution_spaces.hpp b/cmake/clang_tidy_plugins/hide_from_abi/test/execution_spaces.hpp new file mode 100644 index 000000000000..493abdd7551a --- /dev/null +++ b/cmake/clang_tidy_plugins/hide_from_abi/test/execution_spaces.hpp @@ -0,0 +1,97 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// clang-format off +// CLANG_TIDY_CONFIG_BEGIN +// Checks: cccl-hide-from-abi +// CLANG_TIDY_CONFIG_END + +#include + +#undef _CCCL_HOST +#undef _CCCL_DEVICE +#undef _CCCL_TILE +#undef _CCCL_HOST_DEVICE +#undef __host__ +#undef __device__ +#undef __tile__ +// Attribute expansions keep the annotation in the declaration's source range. +#define __host__ __attribute__((annotate("host"))) +#define __device__ __attribute__((annotate("device"))) +#define __tile__ __attribute__((annotate("tile"))) +#define _CCCL_HOST __host__ +#define _CCCL_DEVICE __device__ +#define _CCCL_TILE __tile__ +#define _CCCL_HOST_DEVICE _CCCL_HOST _CCCL_DEVICE + +// CHECK-MESSAGES-NOT: warning: +__host__ __device__ inline void host_device() {} +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_HOST_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +// CHECK-FIXES: DiagnosticName: cccl-hide-from-abi +// CHECK-FIXES: Length: 8 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_DEVICE_API +// CHECK-FIXES: Length: 11 +// CHECK-FIXES-NEXT: ReplacementText: '' + +__device__ __host__ inline void device_host() {} +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_HOST_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +// CHECK-FIXES: DiagnosticName: cccl-hide-from-abi +// CHECK-FIXES: Length: 10 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_DEVICE_API +// CHECK-FIXES: Length: 9 +// CHECK-FIXES-NEXT: ReplacementText: '' + +_CCCL_HOST /* Preserve the comment and specifier. */ inline __device__ void mixed_annotations() {} +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_HOST_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +// CHECK-FIXES: DiagnosticName: cccl-hide-from-abi +// CHECK-FIXES: Length: 10 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_DEVICE_API +// CHECK-FIXES: Length: 11 +// CHECK-FIXES-NEXT: ReplacementText: '' + +__tile__ inline void tile_function() {} +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_TILE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +// CHECK-FIXES: DiagnosticName: cccl-hide-from-abi +// CHECK-FIXES: Length: 8 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_TILE_API + +_CCCL_DEVICE _CCCL_TILE inline void device_tile() {} +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_TILE _CCCL_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +// CHECK-FIXES: DiagnosticName: cccl-hide-from-abi +// CHECK-FIXES: Length: 12 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_TILE _CCCL_DEVICE_API +// CHECK-FIXES: Length: 11 +// CHECK-FIXES-NEXT: ReplacementText: '' + +_CCCL_TILE _CCCL_HOST_DEVICE inline void all_spaces() {} +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +// CHECK-FIXES: DiagnosticName: cccl-hide-from-abi +// CHECK-FIXES: Length: 10 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_API +// CHECK-FIXES: Length: 18 +// CHECK-FIXES-NEXT: ReplacementText: '' + +__host__ __device__ __tile__ inline void separate_spaces() {} +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +// CHECK-FIXES: DiagnosticName: cccl-hide-from-abi +// CHECK-FIXES: Length: 8 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_API +// CHECK-FIXES: Length: 20 +// CHECK-FIXES-NEXT: ReplacementText: '' + +_CCCL_HOST_DEVICE __host__ inline void redundant_host() {} +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_HOST_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +// CHECK-FIXES: DiagnosticName: cccl-hide-from-abi +// CHECK-FIXES: Length: 17 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_DEVICE_API +// CHECK-FIXES: Length: 9 +// CHECK-FIXES-NEXT: ReplacementText: '' +// clang-format on diff --git a/cmake/clang_tidy_plugins/hide_from_abi/test/functions.hpp b/cmake/clang_tidy_plugins/hide_from_abi/test/functions.hpp new file mode 100644 index 000000000000..8c8b9f0bd992 --- /dev/null +++ b/cmake/clang_tidy_plugins/hide_from_abi/test/functions.hpp @@ -0,0 +1,258 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// clang-format off +// CLANG_TIDY_CONFIG_BEGIN +// Checks: cccl-hide-from-abi +// CLANG_TIDY_CONFIG_END + +#include + +#undef _CCCL_HOST +#undef _CCCL_DEVICE +#undef _CCCL_HOST_DEVICE +// Attribute expansions keep the annotation in the declaration's source range. +#define _CCCL_HOST __attribute__((annotate("host"))) +#define _CCCL_DEVICE __attribute__((annotate("device"))) +#define _CCCL_HOST_DEVICE _CCCL_HOST _CCCL_DEVICE + +// CHECK-MESSAGES-NOT: warning: +_CCCL_HOST inline void host_function() {} +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +_CCCL_DEVICE constexpr int device_function() { return 1; } +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +_CCCL_HOST_DEVICE inline void host_device_function() {} +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_HOST_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +struct members { + _CCCL_HOST members() = default; +// CHECK-MESSAGES: :[[@LINE-1]]:14: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + _CCCL_HOST ~members() = default; +// CHECK-MESSAGES: :[[@LINE-1]]:14: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + _CCCL_DEVICE operator bool() const { return true; } +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + _CCCL_HOST_DEVICE members& operator=(const members&) = default; +// CHECK-MESSAGES: :[[@LINE-1]]:30: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + _CCCL_DEVICE void method() {} +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + _CCCL_HOST static void static_method() {} +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + _CCCL_HOST virtual void virtual_method() {} +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + _CCCL_HOST friend void friend_function(members) {} +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + void declared(); +}; +_CCCL_HOST inline void members::declared() {} +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + +inline void unannotated() {} +// CHECK-MESSAGES: :[[@LINE-1]]:13: warning: function needs an API macro (one of _CCCL_API, _CCCL_DEVICE_API, or _CCCL_HOST_API) to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +_CCCL_HOST void declaration_only(); +_CCCL_DEVICE void deleted_function() = delete; +_CCCL_DEVICE inline int variable = 0; +struct implicit_members { int value; }; +inline implicit_members object{}; +inline auto copied_object = object; +inline auto lambda = []() _CCCL_HOST { return 1; }; +inline int lambda_result = lambda(); + +// CHECK-FIXES: DiagnosticName: cccl-hide-from-abi +// CHECK-FIXES: Length: 10 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_API +// CHECK-FIXES: Length: 12 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_DEVICE_API +// CHECK-FIXES: Length: 17 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_DEVICE_API +// CHECK-FIXES: Replacements: [] + +template +_CCCL_HOST_DEVICE T identity(T value) { return value; } +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_HOST_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +inline int instantiated = identity(1); +inline double also_instantiated = identity(1.0); +template long identity(long); +template <> _CCCL_HOST inline char identity(char value) { return value; } +// CHECK-MESSAGES: :[[@LINE-1]]:13: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + +template struct container { + _CCCL_HOST container() = default; +// CHECK-MESSAGES: :[[@LINE-1]]:14: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + _CCCL_DEVICE void method() {} +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +}; +template struct container; +extern template struct container; +inline container implicit_instantiation; + +template <> struct container { + _CCCL_DEVICE void specialized_method() {} +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +}; +template struct container { + _CCCL_DEVICE void partial_method() {} +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +}; +inline container partial_instantiation; + +// CHECK-FIXES: DiagnosticName: cccl-hide-from-abi +// CHECK-FIXES: Length: 17 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_DEVICE_API + +_CCCL_HOST_API inline void host_api() {} +_CCCL_DEVICE_API inline void device_api() {} +_CCCL_API inline void host_device_api() {} +_CCCL_PUBLIC_API inline void public_api() {} +_CCCL_TRIVIAL_API void trivial_api() {} +_CCCL_NODEBUG_API void nodebug_api() {} +_CCCL_HOST __attribute__((visibility("hidden"))) inline void hidden_attribute() {} +_CCCL_HOST __attribute__((visibility("default"))) inline void exported_attribute() {} +__attribute__((visibility("hidden"))) void inherited(); +_CCCL_HOST inline void inherited() {} + +// Exclusion from explicit instantiation alone does not establish visibility. +_CCCL_HOST __attribute__((exclude_from_explicit_instantiation)) inline void excluded() {} +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + +struct __attribute__((visibility("hidden"))) hidden_class { + _CCCL_HOST void unannotated_method() {} +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +}; + +// NOLINTNEXTLINE(cccl-hide-from-abi) +_CCCL_HOST inline void suppressed() {} + +// CHECK-FIXES: DiagnosticName: cccl-hide-from-abi +// CHECK-FIXES: Length: 10 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_API + +_CCCL_HOST [[nodiscard]] constexpr int attributed() { return 0; } +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +// CHECK-FIXES: DiagnosticName: cccl-hide-from-abi +// CHECK-FIXES: Length: 10 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_API + +_CCCL_HOST /* Keep this comment and inline. */ inline int explicit_inline() { return 0; } +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +// CHECK-FIXES: DiagnosticName: cccl-hide-from-abi +// CHECK-FIXES: Length: 10 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_API + +_CCCL_DEVICE _CCCL_FORCEINLINE void forceinline_function() {} +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +// CHECK-FIXES: DiagnosticName: cccl-hide-from-abi +// CHECK-FIXES: Length: 12 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_DEVICE_API + +inline _CCCL_HOST_DEVICE void middle_annotation() {} +// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: use _CCCL_HOST_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +// CHECK-FIXES: DiagnosticName: cccl-hide-from-abi +// CHECK-FIXES: Length: 17 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_DEVICE_API + +#define DEFINE_FUNCTION(name) inline void name() {} +DEFINE_FUNCTION(macro_function) +// CHECK-MESSAGES: :[[@LINE-1]]:17: warning: function needs an API macro (one of _CCCL_API, _CCCL_DEVICE_API, or _CCCL_HOST_API) to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +// CHECK-FIXES: DiagnosticName: cccl-hide-from-abi +// CHECK-FIXES: Replacements: [] + +namespace defaulted_special_members { +// CHECK-MESSAGES-NOT: warning: +struct members { + members() = default; +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + members(const members&) = default; +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + members(members&&) = default; +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + members& operator=(const members&) = default; +// CHECK-MESSAGES: :[[@LINE-1]]:12: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + members& operator=(members&&) = default; +// CHECK-MESSAGES: :[[@LINE-1]]:12: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + ~members() = default; +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +}; + +// CHECK-FIXES: Replacements: +// CHECK-FIXES: Length: 0 +// CHECK-FIXES-NEXT: ReplacementText: '_CCCL_HIDE_FROM_ABI ' +// CHECK-FIXES: Replacements: +// CHECK-FIXES: Length: 0 +// CHECK-FIXES-NEXT: ReplacementText: '_CCCL_HIDE_FROM_ABI ' +// CHECK-FIXES: Replacements: +// CHECK-FIXES: Length: 0 +// CHECK-FIXES-NEXT: ReplacementText: '_CCCL_HIDE_FROM_ABI ' +// CHECK-FIXES: Replacements: +// CHECK-FIXES: Length: 0 +// CHECK-FIXES-NEXT: ReplacementText: '_CCCL_HIDE_FROM_ABI ' +// CHECK-FIXES: Replacements: +// CHECK-FIXES: Length: 0 +// CHECK-FIXES-NEXT: ReplacementText: '_CCCL_HIDE_FROM_ABI ' +// CHECK-FIXES: Replacements: +// CHECK-FIXES: Length: 0 +// CHECK-FIXES-NEXT: ReplacementText: '_CCCL_HIDE_FROM_ABI ' + +struct annotated_members { + _CCCL_HIDE_FROM_ABI annotated_members() = default; + _CCCL_HIDE_FROM_ABI annotated_members(const annotated_members&) = default; + _CCCL_HIDE_FROM_ABI annotated_members(annotated_members&&) = default; + _CCCL_HIDE_FROM_ABI annotated_members& operator=(const annotated_members&) = default; + _CCCL_HIDE_FROM_ABI annotated_members& operator=(annotated_members&&) = default; + _CCCL_HIDE_FROM_ABI ~annotated_members() = default; +}; + +template struct container { + constexpr explicit container() = default; +// CHECK-MESSAGES: :[[@LINE-1]]:22: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +}; +template struct container; +inline container implicit_instantiation; + +// CHECK-FIXES: Replacements: +// CHECK-FIXES: Length: 0 +// CHECK-FIXES-NEXT: ReplacementText: '_CCCL_HIDE_FROM_ABI ' + +struct out_of_line { + out_of_line(); +}; +inline out_of_line::out_of_line() = default; +// CHECK-MESSAGES: :[[@LINE-1]]:21: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +// CHECK-FIXES: Replacements: +// CHECK-FIXES: Length: 0 +// CHECK-FIXES-NEXT: ReplacementText: '_CCCL_HIDE_FROM_ABI ' + +} // namespace defaulted_special_members +// clang-format on From 032bd7b01754f3a7de63f2b0e97c4426d4076223 Mon Sep 17 00:00:00 2001 From: Jacob Faibussowitsch Date: Wed, 4 Nov 2026 07:58:06 -0500 Subject: [PATCH 2/5] cccl-hide-from-abi --- .../stf/include/cccl/c/experimental/stf/stf.h | 4 +- .../stf/test/test_logical_data_with_place.cu | 2 +- c/experimental/stf/test/test_placement.cpp | 2 +- c/experimental/stf/test/test_places.cpp | 2 +- ci/matrix.yaml | 1 - cmake/clang_tidy_plugins/CMakeLists.txt | 17 +- cmake/clang_tidy_plugins/common/matchers.h | 13 +- .../hide_from_abi/hide_from_abi.cpp | 188 ++++-- .../hide_from_abi/test/CMakeLists.txt | 39 +- .../hide_from_abi/test/classes.hpp | 298 +++++++++ .../hide_from_abi/test/execution_spaces.hpp | 10 +- .../hide_from_abi/test/functions.hpp | 141 +--- .../hide_from_abi/test/kernels.cuh | 23 + .../hide_from_abi/test/redeclarations.hpp | 51 ++ .../bench/bitonic_sort/bitonic_common.cuh | 4 +- .../bench/find_bound/find_bound_common.cuh | 10 +- .../bench/histogram/histogram_common.cuh | 2 +- cub/benchmarks/bench/merge/merge_common.cuh | 6 +- .../bench/radix_sort/policy_selector.h | 2 +- cub/benchmarks/bench/reduce/base.cuh | 2 +- .../bench/reduce/warp_reduce_base.cuh | 4 +- .../bench/run_length_encode/base.cuh | 2 +- cub/benchmarks/bench/scan/exclusive/base.cuh | 2 +- cub/benchmarks/bench/scan/warp_scan_base.cuh | 4 +- .../bench/segmented_reduce/base.cuh | 2 +- .../bench/segmented_reduce/variable_base.cuh | 2 +- cub/benchmarks/bench/segmented_scan/base.cuh | 8 +- .../bench/segmented_topk/variable/common.cuh | 4 +- .../variable/indexed_common.cuh | 4 +- .../segmented_topk/variable/keys_common.cuh | 4 +- .../bench/thread_sort/selection/common.cuh | 6 +- .../bench/thread_sort/sort/common.cuh | 19 +- .../transform/applications/P1/bfloat16.h | 80 +-- cub/benchmarks/bench/transform/babelstream.h | 8 +- cub/benchmarks/bench/transform/common.h | 11 +- cub/cub/agent/agent_adjacent_difference.cuh | 10 +- cub/cub/agent/agent_batch_memcpy.cuh | 42 +- cub/cub/agent/agent_batched_topk_cluster.cuh | 114 ++-- cub/cub/agent/agent_find.cuh | 8 +- cub/cub/agent/agent_for.cuh | 2 +- cub/cub/agent/agent_histogram.cuh | 36 +- cub/cub/agent/agent_merge.cuh | 4 +- cub/cub/agent/agent_merge_sort.cuh | 18 +- cub/cub/agent/agent_radix_sort_downsweep.cuh | 20 +- cub/cub/agent/agent_radix_sort_histogram.cuh | 14 +- cub/cub/agent/agent_radix_sort_onesweep.cuh | 50 +- cub/cub/agent/agent_radix_sort_upsweep.cuh | 22 +- cub/cub/agent/agent_reduce.cuh | 20 +- cub/cub/agent/agent_reduce_by_key.cuh | 18 +- cub/cub/agent/agent_rle.cuh | 22 +- cub/cub/agent/agent_scan.cuh | 20 +- cub/cub/agent/agent_scan_by_key.cuh | 22 +- cub/cub/agent/agent_segmented_radix_sort.cuh | 6 +- cub/cub/agent/agent_select_if.cuh | 40 +- cub/cub/agent/agent_sub_warp_merge_sort.cuh | 18 +- cub/cub/agent/agent_three_way_partition.cuh | 34 +- cub/cub/agent/agent_topk.cuh | 22 +- cub/cub/agent/agent_unique_by_key.cuh | 16 +- cub/cub/agent/single_pass_scan_operators.cuh | 175 ++--- cub/cub/block/block_adjacent_difference.cuh | 28 +- cub/cub/block/block_discontinuity.cuh | 34 +- cub/cub/block/block_exchange.cuh | 62 +- cub/cub/block/block_histogram.cuh | 12 +- cub/cub/block/block_load.cuh | 46 +- cub/cub/block/block_load_to_shared.cuh | 4 +- cub/cub/block/block_merge_sort.cuh | 59 +- cub/cub/block/block_radix_rank.cuh | 66 +- cub/cub/block/block_radix_sort.cuh | 68 +- cub/cub/block/block_raking_layout.cuh | 4 +- cub/cub/block/block_reduce.cuh | 22 +- cub/cub/block/block_run_length_decode.cuh | 20 +- cub/cub/block/block_scan.cuh | 72 +- cub/cub/block/block_shuffle.cuh | 18 +- cub/cub/block/block_store.cuh | 28 +- cub/cub/block/radix_rank_sort_operations.cuh | 84 +-- .../block_histogram_atomic.cuh | 4 +- .../specializations/block_histogram_sort.cuh | 8 +- .../specializations/block_reduce_raking.cuh | 10 +- .../block_reduce_raking_commutative_only.cuh | 6 +- .../block_reduce_warp_reductions.cuh | 11 +- .../specializations/block_scan_raking.cuh | 33 +- .../specializations/block_scan_warp_scans.cuh | 29 +- cub/cub/detail/array_utils.cuh | 4 +- cub/cub/detail/binary_search_helpers.cuh | 17 +- cub/cub/detail/choose_offset.cuh | 2 +- cub/cub/detail/cub_vector.cuh | 176 ++--- cub/cub/detail/device_double_buffer.cuh | 8 +- cub/cub/detail/double_buffer.cuh | 12 +- cub/cub/detail/future_value.cuh | 4 +- cub/cub/detail/input_value.cuh | 8 +- cub/cub/detail/key_value_pair.cuh | 10 +- cub/cub/detail/null_type.cuh | 10 +- cub/cub/detail/numeric_traits.cuh | 45 +- cub/cub/detail/prefetch.cuh | 2 +- cub/cub/detail/rfa.cuh | 78 +-- cub/cub/detail/segmented_params.cuh | 35 +- cub/cub/detail/segmented_scan_helpers.cuh | 66 +- cub/cub/detail/strong_load.cuh | 22 +- cub/cub/detail/strong_store.cuh | 32 +- cub/cub/detail/temporary_storage.cuh | 30 +- cub/cub/detail/topk_helpers.cuh | 32 +- cub/cub/detail/uninitialized.cuh | 2 +- cub/cub/detail/uninitialized_copy.cuh | 4 +- cub/cub/detail/unsafe_bitcast.cuh | 2 +- cub/cub/detail/warpspeed/sync_handler.cuh | 2 +- cub/cub/device/device_for.cuh | 4 +- cub/cub/device/device_histogram.cuh | 2 +- cub/cub/device/device_merge_sort.cuh | 2 +- cub/cub/device/device_radix_sort.cuh | 2 +- .../device/device_segmented_radix_sort.cuh | 2 +- cub/cub/device/device_segmented_sort.cuh | 2 +- cub/cub/device/device_transform.cuh | 2 +- .../device/dispatch/dispatch_batched_topk.cuh | 6 +- .../device/dispatch/dispatch_copy_mdspan.cuh | 2 +- .../dispatch/dispatch_reduce_by_key.cuh | 18 +- .../dispatch_reduce_deterministic.cuh | 8 +- cub/cub/device/dispatch/dispatch_rle.cuh | 14 +- cub/cub/device/dispatch/dispatch_scan.cuh | 2 +- .../dispatch/dispatch_segmented_reduce.cuh | 4 +- .../device/dispatch/dispatch_select_if.cuh | 32 +- .../dispatch/dispatch_streaming_reduce.cuh | 18 +- .../dispatch/kernels/kernel_batched_topk.cuh | 4 +- .../dispatch/kernels/kernel_histogram.cuh | 26 +- .../device/dispatch/kernels/kernel_reduce.cuh | 14 +- .../device/dispatch/kernels/kernel_scan.cuh | 2 +- .../kernels/kernel_segmented_reduce.cuh | 4 +- .../kernels/kernel_segmented_scan.cuh | 14 +- .../kernels/kernel_segmented_sort.cuh | 8 +- .../kernels/kernel_three_way_partition.cuh | 14 +- .../dispatch/kernels/kernel_transform.cuh | 43 +- cub/cub/device/dispatch/tuning/common.cuh | 6 +- .../tuning/structural_inplace_vector.cuh | 2 +- .../tuning/tuning_adjacent_difference.cuh | 2 +- .../dispatch/tuning/tuning_batch_memcpy.cuh | 10 +- .../dispatch/tuning/tuning_batched_topk.cuh | 14 +- .../device/dispatch/tuning/tuning_find.cuh | 2 +- .../tuning_find_bound_sorted_values.cuh | 2 +- cub/cub/device/dispatch/tuning/tuning_for.cuh | 2 +- .../dispatch/tuning/tuning_histogram.cuh | 2 +- .../device/dispatch/tuning/tuning_merge.cuh | 2 +- .../dispatch/tuning/tuning_merge_sort.cuh | 2 +- .../dispatch/tuning/tuning_radix_sort.cuh | 26 +- .../device/dispatch/tuning/tuning_reduce.cuh | 14 +- .../dispatch/tuning/tuning_reduce_by_key.cuh | 6 +- .../dispatch/tuning/tuning_rle_encode.cuh | 8 +- .../tuning/tuning_rle_non_trivial_runs.cuh | 6 +- .../device/dispatch/tuning/tuning_scan.cuh | 20 +- .../dispatch/tuning/tuning_scan_by_key.cuh | 16 +- .../tuning/tuning_segmented_radix_sort.cuh | 2 +- .../tuning/tuning_segmented_reduce.cuh | 4 +- .../dispatch/tuning/tuning_segmented_scan.cuh | 4 +- .../dispatch/tuning/tuning_segmented_sort.cuh | 42 +- .../dispatch/tuning/tuning_select_if.cuh | 26 +- .../dispatch/tuning/tuning_set_operations.cuh | 2 +- .../tuning/tuning_three_way_partition.cuh | 16 +- .../device/dispatch/tuning/tuning_topk.cuh | 2 +- .../dispatch/tuning/tuning_transform.cuh | 20 +- .../dispatch/tuning/tuning_unique_by_key.cuh | 10 +- cub/cub/grid/grid_even_share.cuh | 12 +- cub/cub/grid/grid_queue.cuh | 18 +- cub/cub/iterator/arg_index_input_iterator.cuh | 30 +- .../cache_modified_input_iterator.cuh | 31 +- .../cache_modified_output_iterator.cuh | 30 +- cub/cub/iterator/tex_obj_input_iterator.cuh | 34 +- cub/cub/thread/thread_load.cuh | 128 ++-- cub/cub/thread/thread_operators.cuh | 43 +- cub/cub/thread/thread_reduce.cuh | 21 +- cub/cub/thread/thread_scan.cuh | 20 +- cub/cub/thread/thread_search.cuh | 6 +- cub/cub/thread/thread_simd.cuh | 32 +- cub/cub/thread/thread_store.cuh | 140 ++-- cub/cub/util_allocator.cuh | 32 +- cub/cub/util_debug.cuh | 2 +- cub/cub/util_device.cuh | 36 +- cub/cub/util_macro.cuh | 2 +- cub/cub/util_math.cuh | 14 +- cub/cub/util_ptx.cuh | 32 +- cub/cub/util_temporary_storage.cuh | 2 +- cub/cub/util_vsmem.cuh | 14 +- .../specializations/warp_exchange_shfl.cuh | 18 +- .../specializations/warp_exchange_smem.cuh | 11 +- .../warp/specializations/warp_reduce_smem.cuh | 375 +++++++++++ .../warp/specializations/warp_scan_shfl.cuh | 54 +- .../warp/specializations/warp_scan_smem.cuh | 39 +- cub/cub/warp/warp_bitonic_sort.cuh | 18 +- cub/cub/warp/warp_exchange.cuh | 11 +- cub/cub/warp/warp_load.cuh | 57 +- cub/cub/warp/warp_merge_sort.cuh | 6 +- cub/cub/warp/warp_reduce.cuh | 77 +-- cub/cub/warp/warp_reduce_batched.cuh | 2 +- cub/cub/warp/warp_scan.cuh | 54 +- cub/cub/warp/warp_store.cuh | 38 +- .../bench/cuco/common/key_generator.cuh | 26 +- .../experimental/__places/cute_partition.cuh | 160 ++--- .../__places/exec/cuda_context.cuh | 32 +- .../__places/exec/green_context.cuh | 50 +- .../__places/partitions/blocked_partition.cuh | 8 +- .../__places/partitions/cyclic_shape.cuh | 36 +- .../__places/partitions/tiled_partition.cuh | 14 +- .../cuda/experimental/__places/places.cuh | 458 ++++++------- .../__stf/allocators/buddy_allocator.cuh | 42 +- .../experimental/__stf/graph/graph_ctx.cuh | 104 +-- .../__stf/internal/async_resources_handle.cuh | 49 +- .../experimental/__stf/internal/context.cuh | 236 +++---- .../__stf/internal/execution_policy.cuh | 50 +- .../__stf/internal/inner_shape.cuh | 4 +- .../internal/interpreted_execution_policy.cuh | 50 +- .../experimental/__stf/internal/slice.cuh | 65 +- .../__stf/internal/thread_hierarchy.cuh | 48 +- .../__stf/stackable/stackable_ctx.cuh | 264 ++++---- .../experimental/__stf/stream/stream_ctx.cuh | 138 ++-- .../__stf/utility/cartesian_iterator.cuh | 90 +-- .../__stf/utility/cuda_safe_call.cuh | 50 +- .../experimental/__stf/utility/dimensions.cuh | 82 +-- .../__stf/utility/exception_policy.cuh | 622 +++++++++--------- .../experimental/__stf/utility/handle.cuh | 54 +- .../cuda/experimental/__stf/utility/hash.cuh | 8 +- .../experimental/__stf/utility/memory.cuh | 112 ++-- .../__stf/utility/source_location.cuh | 10 +- docs/cub/Doxyfile | 14 +- docs/cudax/Doxyfile | 14 +- docs/libcudacxx/Doxyfile | 14 +- docs/thrust/Doxyfile | 20 +- .../include/cuda/__argument/argument_bounds.h | 2 +- libcudacxx/include/cuda/__container/buffer.h | 18 +- .../__container/uninitialized_async_buffer.h | 13 +- .../include/cuda/__device/all_devices.h | 2 +- libcudacxx/include/cuda/__event/timed_event.h | 8 +- .../cuda/__hierarchy/hierarchy_dimensions.h | 7 +- .../cuda/__iterator/counting_iterator.h | 8 +- .../include/cuda/__launch/configuration.h | 52 +- libcudacxx/include/cuda/__launch/launch.h | 10 +- .../cuda/__memory_pool/device_memory_pool.h | 4 +- .../__memory_pool/shared_device_memory_pool.h | 4 +- .../__memory_pool/shared_memory_pool_base.h | 4 +- .../__memory_pool/shared_pinned_memory_pool.h | 4 +- .../__memory_resource/allocation_alignment.h | 2 +- .../cuda/__memory_resource/any_resource.h | 47 +- .../include/cuda/__memory_resource/resource.h | 4 +- libcudacxx/include/cuda/__nvtx/nvtx.h | 2 +- .../include/cuda/__stream/internal_streams.h | 2 +- .../include/cuda/__stream/launch_transform.h | 2 +- .../__utility/__basic_any/basic_any_base.h | 14 +- .../__utility/__basic_any/basic_any_ptr.h | 2 +- .../__utility/__basic_any/basic_any_ref.h | 4 +- .../__utility/__basic_any/basic_any_value.h | 10 +- .../cuda/__utility/__basic_any/conversions.h | 10 +- .../include/cuda/__utility/__basic_any/iset.h | 2 +- .../cuda/__utility/__basic_any/virtual_ptrs.h | 4 +- .../include/cuda/std/__complex/complex.h | 6 +- .../include/cuda/std/__complex/nvbf16.h | 4 +- .../include/cuda/std/__complex/nvfp16.h | 5 +- .../cuda/std/__expected/bad_expected_access.h | 2 +- .../include/cuda/std/__optional/optional.h | 4 +- .../cuda/std/__optional/optional_base.h | 2 +- .../include/cuda/std/__pstl/cuda/sort.h | 2 +- .../std/__random/bernoulli_distribution.h | 4 +- .../cuda/std/__random/binomial_distribution.h | 4 +- .../cuda/std/__random/cauchy_distribution.h | 6 +- .../std/__random/chi_squared_distribution.h | 8 +- .../std/__random/exponential_distribution.h | 4 +- .../std/__random/extreme_value_distribution.h | 4 +- .../cuda/std/__random/fisher_f_distribution.h | 6 +- .../cuda/std/__random/gamma_distribution.h | 6 +- .../std/__random/geometric_distribution.h | 8 +- .../std/__random/lognormal_distribution.h | 6 +- .../__random/negative_binomial_distribution.h | 6 +- .../cuda/std/__random/normal_distribution.h | 4 +- .../cuda/std/__random/poisson_distribution.h | 4 +- .../std/__random/student_t_distribution.h | 6 +- .../std/__random/uniform_int_distribution.h | 6 +- .../std/__random/uniform_real_distribution.h | 6 +- .../cuda/std/__random/weibull_distribution.h | 6 +- .../include/cuda/std/__tuple_dir/ignore.h | 2 +- .../cuda/std/__tuple_dir/sfinae_helpers.h | 30 +- .../include/cuda/std/__tuple_dir/tuple_leaf.h | 4 +- .../include/cuda/std/__utility/typeid.h | 2 +- .../include/cuda/std/__variant/variant_base.h | 10 +- libcudacxx/include/nv/target | 18 +- .../nvbench_helper/device_side_benchmark.cuh | 12 +- .../nvbench_helper/nvbench_helper.cuh | 101 +-- .../benchmarks/bench/set_operations/base.cuh | 2 +- .../bench/set_operations/by_key.cuh | 2 +- thrust/testing/copy_construct_test.h | 10 +- thrust/testing/unittest/random.h | 12 +- thrust/testing/unittest/special_types.h | 126 ++-- thrust/testing/unittest/unittest.h | 20 +- thrust/thrust/adjacent_difference.h | 8 +- thrust/thrust/allocate_unique.h | 48 +- thrust/thrust/binary_search.h | 58 +- thrust/thrust/complex.h | 160 ++--- thrust/thrust/copy.h | 34 +- thrust/thrust/count.h | 8 +- thrust/thrust/detail/adjacent_difference.inl | 8 +- thrust/thrust/detail/alignment.h | 2 +- .../detail/allocator/allocator_system.h | 2 +- .../detail/allocator/copy_construct_range.h | 10 +- .../thrust/detail/allocator/destroy_range.h | 6 +- .../detail/allocator/fill_construct_range.h | 4 +- .../detail/allocator/no_throw_allocator.h | 8 +- .../detail/allocator/tagged_allocator.h | 18 +- .../detail/allocator/temporary_allocator.h | 10 +- .../detail/allocator/value_initialize_range.h | 4 +- .../detail/allocator_aware_execution_policy.h | 8 +- thrust/thrust/detail/binary_search.inl | 58 +- thrust/thrust/detail/caching_allocator.h | 2 +- thrust/thrust/detail/complex/arithmetic.h | 42 +- thrust/thrust/detail/complex/catrig.h | 50 +- thrust/thrust/detail/complex/catrigf.h | 36 +- thrust/thrust/detail/complex/ccosh.h | 12 +- thrust/thrust/detail/complex/ccoshf.h | 8 +- thrust/thrust/detail/complex/cexp.h | 10 +- thrust/thrust/detail/complex/cexpf.h | 8 +- thrust/thrust/detail/complex/clog.h | 10 +- thrust/thrust/detail/complex/clogf.h | 6 +- thrust/thrust/detail/complex/complex.inl | 54 +- thrust/thrust/detail/complex/cpow.h | 6 +- thrust/thrust/detail/complex/cproj.h | 10 +- thrust/thrust/detail/complex/csinh.h | 12 +- thrust/thrust/detail/complex/csinhf.h | 8 +- thrust/thrust/detail/complex/csqrt.h | 6 +- thrust/thrust/detail/complex/csqrtf.h | 4 +- thrust/thrust/detail/complex/ctanh.h | 12 +- thrust/thrust/detail/complex/ctanhf.h | 8 +- thrust/thrust/detail/complex/math_private.h | 16 +- thrust/thrust/detail/complex/stream.h | 6 +- thrust/thrust/detail/contiguous_storage.h | 73 +- thrust/thrust/detail/contiguous_storage.inl | 55 +- thrust/thrust/detail/copy.h | 12 +- thrust/thrust/detail/copy.inl | 12 +- thrust/thrust/detail/copy_if.h | 4 +- thrust/thrust/detail/copy_if.inl | 8 +- thrust/thrust/detail/count.h | 8 +- thrust/thrust/detail/count.inl | 9 +- thrust/thrust/detail/device_ptr.inl | 4 +- thrust/thrust/detail/equal.inl | 9 +- thrust/thrust/detail/execute_with_allocator.h | 4 +- .../detail/execute_with_allocator_fwd.h | 6 +- thrust/thrust/detail/execution_policy.h | 6 +- thrust/thrust/detail/extrema.inl | 25 +- thrust/thrust/detail/fill.inl | 8 +- thrust/thrust/detail/find.inl | 12 +- thrust/thrust/detail/for_each.inl | 8 +- thrust/thrust/detail/function.h | 2 +- thrust/thrust/detail/functional/actor.h | 8 +- thrust/thrust/detail/functional/operators.h | 62 +- thrust/thrust/detail/gather.inl | 12 +- thrust/thrust/detail/generate.inl | 8 +- thrust/thrust/detail/get_iterator_value.h | 4 +- thrust/thrust/detail/inner_product.inl | 9 +- thrust/thrust/detail/internal_functional.h | 32 +- thrust/thrust/detail/logical.inl | 12 +- thrust/thrust/detail/malloc_and_free.h | 6 +- thrust/thrust/detail/malloc_and_free_fwd.h | 6 +- thrust/thrust/detail/memory_algorithms.h | 22 +- thrust/thrust/detail/merge.inl | 28 +- thrust/thrust/detail/mismatch.inl | 16 +- thrust/thrust/detail/nvtx_policy.h | 2 +- thrust/thrust/detail/overlapped_copy.h | 10 +- thrust/thrust/detail/partition.inl | 42 +- thrust/thrust/detail/pointer.h | 58 +- thrust/thrust/detail/random_bijection.h | 14 +- thrust/thrust/detail/range/head_flags.h | 18 +- thrust/thrust/detail/range/tail_flags.h | 21 +- thrust/thrust/detail/raw_pointer_cast.h | 6 +- thrust/thrust/detail/raw_reference_cast.h | 8 +- thrust/thrust/detail/reduce.inl | 37 +- thrust/thrust/detail/reference.h | 107 +-- thrust/thrust/detail/remove.inl | 26 +- thrust/thrust/detail/replace.inl | 25 +- thrust/thrust/detail/reverse.inl | 13 +- thrust/thrust/detail/scan.inl | 53 +- thrust/thrust/detail/scatter.inl | 23 +- thrust/thrust/detail/seq.h | 4 +- thrust/thrust/detail/sequence.inl | 12 +- thrust/thrust/detail/set_operations.inl | 64 +- thrust/thrust/detail/shuffle.inl | 8 +- thrust/thrust/detail/sort.inl | 71 +- thrust/thrust/detail/static_map.h | 8 +- thrust/thrust/detail/swap_ranges.inl | 4 +- thrust/thrust/detail/tabulate.inl | 4 +- thrust/thrust/detail/temporary_array.h | 32 +- thrust/thrust/detail/temporary_array.inl | 17 +- thrust/thrust/detail/temporary_buffer.h | 10 +- thrust/thrust/detail/transform_reduce.inl | 4 +- thrust/thrust/detail/transform_scan.inl | 12 +- thrust/thrust/detail/trivial_sequence.h | 22 +- .../detail/type_traits/has_nested_type.h | 24 +- thrust/thrust/detail/uninitialized_copy.inl | 8 +- thrust/thrust/detail/uninitialized_fill.inl | 8 +- thrust/thrust/detail/unique.inl | 41 +- thrust/thrust/detail/vector_base.h | 67 +- thrust/thrust/detail/vector_base.inl | 171 ++--- thrust/thrust/device_allocator.h | 18 +- thrust/thrust/device_delete.h | 2 +- thrust/thrust/device_free.h | 2 +- thrust/thrust/device_make_unique.h | 2 +- thrust/thrust/device_malloc.h | 2 +- thrust/thrust/device_malloc_allocator.h | 24 +- thrust/thrust/device_new.h | 6 +- thrust/thrust/device_new_allocator.h | 22 +- thrust/thrust/device_ptr.h | 17 +- thrust/thrust/device_reference.h | 20 +- thrust/thrust/device_vector.h | 54 +- thrust/thrust/equal.h | 9 +- thrust/thrust/extrema.h | 25 +- thrust/thrust/fill.h | 8 +- thrust/thrust/find.h | 12 +- thrust/thrust/for_each.h | 8 +- thrust/thrust/functional.h | 12 +- thrust/thrust/gather.h | 12 +- thrust/thrust/generate.h | 8 +- thrust/thrust/host_vector.h | 55 +- thrust/thrust/inner_product.h | 9 +- thrust/thrust/iterator/constant_iterator.h | 18 +- thrust/thrust/iterator/counting_iterator.h | 28 +- thrust/thrust/iterator/detail/any_assign.h | 2 +- .../thrust/iterator/detail/any_system_tag.h | 2 +- .../detail/iterator_category_to_system.h | 26 +- .../detail/iterator_category_to_traversal.h | 24 +- .../thrust/iterator/detail/minimum_system.h | 4 +- .../thrust/iterator/detail/normal_iterator.h | 6 +- thrust/thrust/iterator/detail/retag.h | 17 +- .../thrust/iterator/detail/tagged_iterator.h | 6 +- .../detail/tuple_of_iterator_references.h | 24 +- thrust/thrust/iterator/discard_iterator.h | 6 +- thrust/thrust/iterator/iterator_adaptor.h | 22 +- thrust/thrust/iterator/iterator_facade.h | 146 ++-- thrust/thrust/iterator/offset_iterator.h | 20 +- thrust/thrust/iterator/permutation_iterator.h | 10 +- thrust/thrust/iterator/retag.h | 4 +- thrust/thrust/iterator/shuffle_iterator.h | 8 +- thrust/thrust/iterator/strided_iterator.h | 18 +- .../iterator/tabulate_output_iterator.h | 12 +- .../transform_input_output_iterator.h | 18 +- thrust/thrust/iterator/transform_iterator.h | 18 +- .../iterator/transform_output_iterator.h | 14 +- thrust/thrust/iterator/zip_iterator.h | 35 +- thrust/thrust/logical.h | 12 +- thrust/thrust/memory.h | 18 +- thrust/thrust/merge.h | 28 +- thrust/thrust/mismatch.h | 16 +- thrust/thrust/mr/allocator.h | 26 +- thrust/thrust/mr/disjoint_pool.h | 44 +- thrust/thrust/mr/disjoint_sync_pool.h | 16 +- thrust/thrust/mr/disjoint_tls_pool.h | 2 +- thrust/thrust/mr/fancy_pointer_resource.h | 8 +- thrust/thrust/mr/memory_resource.h | 28 +- thrust/thrust/mr/new.h | 10 +- thrust/thrust/mr/polymorphic_adaptor.h | 10 +- thrust/thrust/mr/pool.h | 18 +- thrust/thrust/mr/pool_options.h | 2 +- thrust/thrust/mr/sync_pool.h | 16 +- thrust/thrust/mr/tls_pool.h | 2 +- thrust/thrust/partition.h | 42 +- thrust/thrust/per_device_resource.h | 10 +- .../random/detail/discard_block_engine.inl | 31 +- .../detail/linear_congruential_engine.inl | 26 +- .../linear_congruential_engine_discard.h | 6 +- .../detail/linear_feedback_shift_engine.inl | 26 +- thrust/thrust/random/detail/mod.h | 6 +- .../random/detail/normal_distribution.inl | 39 +- .../random/detail/normal_distribution_base.h | 8 +- .../thrust/random/detail/random_core_access.h | 6 +- .../detail/subtract_with_carry_engine.inl | 26 +- .../detail/uniform_int_distribution.inl | 40 +- .../detail/uniform_real_distribution.inl | 37 +- thrust/thrust/random/detail/urng_traits.h | 8 +- .../random/detail/xor_combine_engine.inl | 36 +- .../random/detail/xor_combine_engine_max.h | 8 +- thrust/thrust/random/discard_block_engine.h | 26 +- .../random/linear_congruential_engine.h | 22 +- .../random/linear_feedback_shift_engine.h | 22 +- thrust/thrust/random/normal_distribution.h | 35 +- .../random/subtract_with_carry_engine.h | 22 +- .../thrust/random/uniform_int_distribution.h | 32 +- .../thrust/random/uniform_real_distribution.h | 32 +- thrust/thrust/random/xor_combine_engine.h | 32 +- thrust/thrust/reduce.h | 37 +- thrust/thrust/remove.h | 27 +- thrust/thrust/replace.h | 25 +- thrust/thrust/reverse.h | 13 +- thrust/thrust/scan.h | 52 +- thrust/thrust/scatter.h | 23 +- thrust/thrust/sequence.h | 12 +- thrust/thrust/set_operations.h | 64 +- thrust/thrust/shuffle.h | 8 +- thrust/thrust/sort.h | 71 +- thrust/thrust/swap.h | 4 +- .../system/cpp/detail/execution_policy.h | 2 +- thrust/thrust/system/cpp/detail/memory.inl | 6 +- thrust/thrust/system/cpp/memory.h | 6 +- thrust/thrust/system/cuda/config.h | 2 +- .../system/cuda/detail/adjacent_difference.h | 6 +- .../thrust/system/cuda/detail/assign_value.h | 10 +- thrust/thrust/system/cuda/detail/copy.h | 29 +- thrust/thrust/system/cuda/detail/copy_if.h | 8 +- .../cuda/detail/core/triple_chevron_launch.h | 22 +- thrust/thrust/system/cuda/detail/count.h | 8 +- .../thrust/system/cuda/detail/cross_system.h | 107 +-- thrust/thrust/system/cuda/detail/equal.h | 4 +- thrust/thrust/system/cuda/detail/error.inl | 14 +- .../system/cuda/detail/execution_policy.h | 32 +- thrust/thrust/system/cuda/detail/extrema.h | 6 +- thrust/thrust/system/cuda/detail/fill.h | 4 +- thrust/thrust/system/cuda/detail/find.h | 16 +- thrust/thrust/system/cuda/detail/gather.h | 6 +- thrust/thrust/system/cuda/detail/generate.h | 5 +- thrust/thrust/system/cuda/detail/get_value.h | 4 +- .../thrust/system/cuda/detail/inner_product.h | 4 +- thrust/thrust/system/cuda/detail/iter_swap.h | 4 +- .../system/cuda/detail/malloc_and_free.h | 4 +- thrust/thrust/system/cuda/detail/memory.inl | 6 +- thrust/thrust/system/cuda/detail/merge.h | 18 +- thrust/thrust/system/cuda/detail/mismatch.h | 20 +- .../thrust/system/cuda/detail/parallel_for.h | 2 +- thrust/thrust/system/cuda/detail/partition.h | 20 +- .../system/cuda/detail/per_device_resource.h | 2 +- thrust/thrust/system/cuda/detail/reduce.h | 31 +- .../thrust/system/cuda/detail/reduce_by_key.h | 8 +- thrust/thrust/system/cuda/detail/remove.h | 13 +- thrust/thrust/system/cuda/detail/replace.h | 12 +- thrust/thrust/system/cuda/detail/reverse.h | 10 +- thrust/thrust/system/cuda/detail/scan.h | 24 +- .../thrust/system/cuda/detail/scan_by_key.h | 18 +- thrust/thrust/system/cuda/detail/scatter.h | 6 +- .../system/cuda/detail/set_operations.h | 34 +- thrust/thrust/system/cuda/detail/sort.h | 17 +- .../thrust/system/cuda/detail/swap_ranges.h | 6 +- thrust/thrust/system/cuda/detail/tabulate.h | 3 +- .../system/cuda/detail/temporary_buffer.h | 9 +- thrust/thrust/system/cuda/detail/terminate.h | 2 +- thrust/thrust/system/cuda/detail/transform.h | 4 +- .../system/cuda/detail/transform_reduce.h | 2 +- .../system/cuda/detail/transform_scan.h | 6 +- .../system/cuda/detail/uninitialized_copy.h | 6 +- .../system/cuda/detail/uninitialized_fill.h | 5 +- thrust/thrust/system/cuda/detail/unique.h | 19 +- .../thrust/system/cuda/detail/unique_by_key.h | 12 +- thrust/thrust/system/cuda/detail/util.h | 22 +- thrust/thrust/system/cuda/memory.h | 6 +- thrust/thrust/system/cuda/memory_resource.h | 9 +- thrust/thrust/system/detail/bad_alloc.h | 6 +- .../thrust/system/detail/error_category.inl | 32 +- thrust/thrust/system/detail/error_code.inl | 50 +- .../thrust/system/detail/error_condition.inl | 23 +- .../detail/generic/adjacent_difference.h | 4 +- .../detail/generic/adjacent_difference.inl | 4 +- .../system/detail/generic/binary_search.h | 28 +- .../system/detail/generic/binary_search.inl | 42 +- thrust/thrust/system/detail/generic/copy.h | 4 +- thrust/thrust/system/detail/generic/copy.inl | 4 +- thrust/thrust/system/detail/generic/copy_if.h | 4 +- .../thrust/system/detail/generic/copy_if.inl | 6 +- thrust/thrust/system/detail/generic/count.h | 4 +- thrust/thrust/system/detail/generic/count.inl | 8 +- thrust/thrust/system/detail/generic/equal.h | 4 +- thrust/thrust/system/detail/generic/equal.inl | 4 +- thrust/thrust/system/detail/generic/extrema.h | 12 +- .../thrust/system/detail/generic/extrema.inl | 28 +- thrust/thrust/system/detail/generic/fill.h | 4 +- thrust/thrust/system/detail/generic/find.h | 6 +- thrust/thrust/system/detail/generic/find.inl | 8 +- .../thrust/system/detail/generic/for_each.h | 4 +- thrust/thrust/system/detail/generic/gather.h | 6 +- .../thrust/system/detail/generic/gather.inl | 6 +- .../thrust/system/detail/generic/generate.h | 4 +- .../thrust/system/detail/generic/generate.inl | 6 +- .../system/detail/generic/inner_product.h | 4 +- .../system/detail/generic/inner_product.inl | 4 +- thrust/thrust/system/detail/generic/logical.h | 6 +- thrust/thrust/system/detail/generic/memory.h | 13 +- .../thrust/system/detail/generic/memory.inl | 13 +- thrust/thrust/system/detail/generic/merge.h | 8 +- thrust/thrust/system/detail/generic/merge.inl | 8 +- .../thrust/system/detail/generic/mismatch.h | 8 +- .../thrust/system/detail/generic/mismatch.inl | 8 +- .../thrust/system/detail/generic/partition.h | 20 +- .../system/detail/generic/partition.inl | 20 +- .../detail/generic/per_device_resource.h | 2 +- thrust/thrust/system/detail/generic/reduce.h | 12 +- .../thrust/system/detail/generic/reduce.inl | 13 +- .../system/detail/generic/reduce_by_key.h | 6 +- .../system/detail/generic/reduce_by_key.inl | 10 +- thrust/thrust/system/detail/generic/remove.h | 12 +- .../thrust/system/detail/generic/remove.inl | 12 +- thrust/thrust/system/detail/generic/replace.h | 12 +- .../thrust/system/detail/generic/replace.inl | 22 +- thrust/thrust/system/detail/generic/reverse.h | 4 +- .../thrust/system/detail/generic/reverse.inl | 4 +- .../detail/generic/scalar/binary_search.h | 12 +- .../detail/generic/scalar/binary_search.inl | 13 +- thrust/thrust/system/detail/generic/scan.h | 10 +- .../system/detail/generic/scan_by_key.h | 14 +- .../system/detail/generic/scan_by_key.inl | 18 +- thrust/thrust/system/detail/generic/scatter.h | 6 +- .../thrust/system/detail/generic/scatter.inl | 6 +- .../system/detail/generic/select_system.h | 8 +- .../thrust/system/detail/generic/sequence.h | 4 +- .../system/detail/generic/set_operations.h | 32 +- .../system/detail/generic/set_operations.inl | 32 +- thrust/thrust/system/detail/generic/shuffle.h | 4 +- .../thrust/system/detail/generic/shuffle.inl | 12 +- thrust/thrust/system/detail/generic/sort.h | 24 +- thrust/thrust/system/detail/generic/sort.inl | 24 +- .../system/detail/generic/swap_ranges.h | 2 +- .../system/detail/generic/swap_ranges.inl | 4 +- .../thrust/system/detail/generic/tabulate.h | 2 +- .../thrust/system/detail/generic/tabulate.inl | 2 +- thrust/thrust/system/detail/generic/tag.h | 2 +- .../system/detail/generic/temporary_buffer.h | 8 +- .../detail/generic/temporary_buffer.inl | 9 +- .../thrust/system/detail/generic/transform.h | 40 +- .../system/detail/generic/transform_reduce.h | 2 +- .../detail/generic/transform_reduce.inl | 2 +- .../system/detail/generic/transform_scan.h | 6 +- .../system/detail/generic/transform_scan.inl | 6 +- .../detail/generic/uninitialized_copy.h | 4 +- .../detail/generic/uninitialized_copy.inl | 14 +- .../detail/generic/uninitialized_fill.h | 4 +- .../detail/generic/uninitialized_fill.inl | 12 +- thrust/thrust/system/detail/generic/unique.h | 12 +- .../thrust/system/detail/generic/unique.inl | 12 +- .../system/detail/generic/unique_by_key.h | 8 +- .../system/detail/generic/unique_by_key.inl | 8 +- .../thrust/system/detail/internal/decompose.h | 14 +- .../detail/sequential/adjacent_difference.h | 2 +- .../system/detail/sequential/assign_value.h | 2 +- .../system/detail/sequential/binary_search.h | 6 +- thrust/thrust/system/detail/sequential/copy.h | 12 +- .../system/detail/sequential/copy_backward.h | 2 +- .../thrust/system/detail/sequential/copy_if.h | 2 +- .../detail/sequential/execution_policy.h | 6 +- .../thrust/system/detail/sequential/extrema.h | 6 +- thrust/thrust/system/detail/sequential/find.h | 2 +- .../system/detail/sequential/for_each.h | 4 +- .../system/detail/sequential/general_copy.h | 14 +- .../system/detail/sequential/get_value.h | 2 +- .../system/detail/sequential/insertion_sort.h | 4 +- .../system/detail/sequential/iter_swap.h | 2 +- .../detail/sequential/malloc_and_free.h | 4 +- .../thrust/system/detail/sequential/merge.h | 4 +- .../system/detail/sequential/partition.h | 14 +- .../thrust/system/detail/sequential/reduce.h | 2 +- .../system/detail/sequential/reduce_by_key.h | 2 +- .../thrust/system/detail/sequential/remove.h | 8 +- thrust/thrust/system/detail/sequential/scan.h | 6 +- .../system/detail/sequential/scan_by_key.h | 4 +- .../system/detail/sequential/set_operations.h | 8 +- thrust/thrust/system/detail/sequential/sort.h | 4 +- .../detail/sequential/stable_merge_sort.h | 24 +- .../detail/sequential/stable_primitive_sort.h | 14 +- .../detail/sequential/stable_radix_sort.h | 52 +- .../system/detail/sequential/trivial_copy.h | 2 +- .../thrust/system/detail/sequential/unique.h | 6 +- .../system/detail/sequential/unique_by_key.h | 4 +- thrust/thrust/system/detail/system_error.inl | 16 +- thrust/thrust/system/error_code.h | 97 +-- .../system/omp/detail/execution_policy.h | 6 +- thrust/thrust/system/system_error.h | 2 +- .../system/tbb/detail/adjacent_difference.h | 2 +- thrust/thrust/system/tbb/detail/copy.h | 5 +- thrust/thrust/system/tbb/detail/copy_if.h | 14 +- .../system/tbb/detail/execution_policy.h | 2 +- thrust/thrust/system/tbb/detail/extrema.h | 6 +- thrust/thrust/system/tbb/detail/find.h | 3 +- thrust/thrust/system/tbb/detail/for_each.h | 11 +- thrust/thrust/system/tbb/detail/merge.h | 56 +- thrust/thrust/system/tbb/detail/partition.h | 8 +- thrust/thrust/system/tbb/detail/reduce.h | 10 +- .../thrust/system/tbb/detail/reduce_by_key.h | 26 +- .../system/tbb/detail/reduce_intervals.h | 11 +- thrust/thrust/system/tbb/detail/remove.h | 8 +- thrust/thrust/system/tbb/detail/scan.h | 30 +- thrust/thrust/system/tbb/detail/sort.h | 42 +- thrust/thrust/system/tbb/detail/unique.h | 6 +- .../thrust/system/tbb/detail/unique_by_key.h | 4 +- thrust/thrust/system/tbb/memory.h | 10 +- thrust/thrust/tabulate.h | 4 +- thrust/thrust/transform.h | 40 +- thrust/thrust/transform_reduce.h | 4 +- thrust/thrust/transform_scan.h | 12 +- .../type_traits/unwrap_contiguous_iterator.h | 4 +- thrust/thrust/uninitialized_copy.h | 8 +- thrust/thrust/uninitialized_fill.h | 8 +- thrust/thrust/unique.h | 41 +- thrust/thrust/zip_function.h | 2 +- 687 files changed, 8116 insertions(+), 7154 deletions(-) create mode 100644 cmake/clang_tidy_plugins/hide_from_abi/test/classes.hpp create mode 100644 cmake/clang_tidy_plugins/hide_from_abi/test/kernels.cuh create mode 100644 cmake/clang_tidy_plugins/hide_from_abi/test/redeclarations.hpp create mode 100644 cub/cub/warp/specializations/warp_reduce_smem.cuh diff --git a/c/experimental/stf/include/cccl/c/experimental/stf/stf.h b/c/experimental/stf/include/cccl/c/experimental/stf/stf.h index 2c9d93df5c99..b572e54c8ade 100644 --- a/c/experimental/stf/include/cccl/c/experimental/stf/stf.h +++ b/c/experimental/stf/include/cccl/c/experimental/stf/stf.h @@ -67,6 +67,8 @@ # error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice." #endif // !CCCL_C_EXPERIMENTAL +#include + #include #include #include @@ -1775,7 +1777,7 @@ stf_error_t stf_cuda_kernel_add_desc_cufunc( //! //! \see stf_cuda_kernel_add_desc_cufunc(), stf_cuda_kernel_get_arg() -inline cudaError_t stf_cuda_kernel_add_desc( +_CCCL_HOST_API inline cudaError_t stf_cuda_kernel_add_desc( stf_cuda_kernel_handle k, const void* func, dim3 grid_dim_, diff --git a/c/experimental/stf/test/test_logical_data_with_place.cu b/c/experimental/stf/test/test_logical_data_with_place.cu index 64522a96c8a1..a53bcb18c700 100644 --- a/c/experimental/stf/test/test_logical_data_with_place.cu +++ b/c/experimental/stf/test/test_logical_data_with_place.cu @@ -110,7 +110,7 @@ C2H_TEST("stf_logical_data_with_place - device place (data on current device)", stf_ctx_handle ctx = stf_ctx_create(); REQUIRE(ctx != nullptr); - float* d_raw = nullptr; + float* d_raw = nullptr; // NOLINT(misc-const-correctness) cudaError_t err = cudaMalloc(&d_raw, N * sizeof(float)); REQUIRE(err == cudaSuccess); const std::unique_ptr d_data_owner(d_raw, cudaFree); diff --git a/c/experimental/stf/test/test_placement.cpp b/c/experimental/stf/test/test_placement.cpp index 50f65f3717cd..932968a9b082 100644 --- a/c/experimental/stf/test/test_placement.cpp +++ b/c/experimental/stf/test/test_placement.cpp @@ -197,7 +197,7 @@ C2H_TEST("shaped allocation on composite data places", "[places][placement][allo REQUIRE(dp != nullptr); // A byte count alone cannot carry the tensor geometry: must fail cleanly - void* bad = stf_data_place_allocate(dp, static_cast(n * sizeof(int)), nullptr); + const void* bad = stf_data_place_allocate(dp, static_cast(n * sizeof(int)), nullptr); REQUIRE(bad == nullptr); void* ptr = stf_data_place_allocate_nd(dp, &dims, sizeof(int), nullptr); diff --git a/c/experimental/stf/test/test_places.cpp b/c/experimental/stf/test/test_places.cpp index d44d5c567645..fdd12c8f6d67 100644 --- a/c/experimental/stf/test/test_places.cpp +++ b/c/experimental/stf/test/test_places.cpp @@ -741,7 +741,7 @@ C2H_TEST("data_place_allocate_invalid_returns_null", "[places][allocate]") { stf_data_place_handle inv = stf_data_place_affine(); REQUIRE(inv != nullptr); - void* ptr = stf_data_place_allocate(inv, 64, nullptr); + const void* ptr = stf_data_place_allocate(inv, 64, nullptr); REQUIRE(ptr == nullptr); stf_data_place_destroy(inv); } diff --git a/ci/matrix.yaml b/ci/matrix.yaml index 6ff4837e2e3b..f49b76bd9602 100644 --- a/ci/matrix.yaml +++ b/ci/matrix.yaml @@ -87,7 +87,6 @@ workflows: # # IMPORTANT: Do NOT delete or remove the `override:` key below, even when it is empty. override: - - { jobs: ['build'], project: 'tidy', std: 'min', cxx: ['clang_preview-cuda'], cudacxx: ['clang'], ctk: 'clang_preview-cuda', sm: '75' } pull_request_fork: - {jobs: ['build'], project: ['libcudacxx', 'thrust', 'cudax'], std: 'max', cxx: ['gcc', 'clang', 'msvc']} diff --git a/cmake/clang_tidy_plugins/CMakeLists.txt b/cmake/clang_tidy_plugins/CMakeLists.txt index f01701656234..701dc0efd292 100644 --- a/cmake/clang_tidy_plugins/CMakeLists.txt +++ b/cmake/clang_tidy_plugins/CMakeLists.txt @@ -87,7 +87,8 @@ function(_cccl_parse_clang_tidy_config source_file out_var) [==[{ "Checks": ["-*", "-clang-diagnostic-*"], "CheckOptions": {}, - "ExtraArgs": [], + "ExtraArgs": ["-nocudainc", "-nocudalib"], + "RemovedArgs": ["-G"], "HeaderFileExtensions": ["", "h", "hh", "hpp", "hxx", "cuh", "inl", "ipp"], "SystemHeaders": false }]==] @@ -209,24 +210,24 @@ function(cccl_create_clang_tidy_test plugin source) cmake_path(ABSOLUTE_PATH source NORMALIZE) cmake_path(GET source STEM test_name) - set(target "cccl.test.tidy.${plugin}.${test_name}") - + set(test_target "cccl.test.tidy.${plugin}.${test_name}") + set(compile_target "${test_target}.compile") # We don't actually care about compiling this (and in fact, we never want to!). The # point of adding this here is to get the corresponding entries into the compilation # database so that clang-tidy knows how to interpret the file properly. - add_library("${target}.compile" OBJECT EXCLUDE_FROM_ALL "${source}") + add_library("${compile_target}" OBJECT EXCLUDE_FROM_ALL "${source}") target_link_libraries( - "${target}.compile" + "${compile_target}" PRIVATE cccl.compiler_interface libcudacxx::libcudacxx ) target_compile_options( - "${target}.compile" + "${compile_target}" PRIVATE -Wno-pragma-clang-attribute ) _cccl_parse_clang_tidy_config("${source}" config) add_test( - NAME "${target}" + NAME "${test_target}" COMMAND "${CMAKE_COMMAND}" "-DSOURCE=${source}" "-DOUTPUT=${CMAKE_CURRENT_BINARY_DIR}/${test_name}.out" @@ -235,7 +236,7 @@ function(cccl_create_clang_tidy_test plugin source) "-DFILECHECK=${CCCL_FILECHECK}" -P "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/run_filecheck.cmake" ) - _cccl_test_requires_target("${target}" cccl.tidy.plugins) + _cccl_test_requires_target("${test_target}" cccl.tidy.plugins) endfunction() block() diff --git a/cmake/clang_tidy_plugins/common/matchers.h b/cmake/clang_tidy_plugins/common/matchers.h index bd44e680cac3..7d5682484aeb 100644 --- a/cmake/clang_tidy_plugins/common/matchers.h +++ b/cmake/clang_tidy_plugins/common/matchers.h @@ -16,10 +16,17 @@ namespace clang::tidy::cccl::util AST_MATCHER_P(Decl, isExpansionInHeaderFile, const FileExtensionsSet*, extensions) // NOLINT { const auto begin = Node.getBeginLoc(); - const auto& sm = Finder->getASTContext().getSourceManager(); - const auto filename = sm.getFilename(sm.getExpansionLoc(begin)); + const auto& SM = Finder->getASTContext().getSourceManager(); + const auto filename = SM.getFilename(SM.getExpansionLoc(begin)); // LLVM's header utility rejects extensionless files even when the empty extension is configured. - return utils::isExpansionLocInHeaderFile(begin, sm, *extensions) + return utils::isExpansionLocInHeaderFile(begin, SM, *extensions) || (extensions->contains("") && !filename.empty() && llvm::sys::path::extension(filename).empty()); } + +using file_matching_type = decltype(ast_matchers::isExpansionInFileMatching("")); + +AST_MATCHER_FUNCTION(file_matching_type, isExpansionInTestOrExampleFile) +{ + return ast_matchers::isExpansionInFileMatching(".*/(test|c2h|examples)/.*"); +} } // namespace clang::tidy::cccl::util diff --git a/cmake/clang_tidy_plugins/hide_from_abi/hide_from_abi.cpp b/cmake/clang_tidy_plugins/hide_from_abi/hide_from_abi.cpp index 2e3a129404e1..153b1a6a8467 100644 --- a/cmake/clang_tidy_plugins/hide_from_abi/hide_from_abi.cpp +++ b/cmake/clang_tidy_plugins/hide_from_abi/hide_from_abi.cpp @@ -30,14 +30,16 @@ namespace clang::tidy::cccl { namespace { +constexpr StringRef CHECK_NAME{"cccl-hide-from-abi"}; + using namespace ast_matchers; // NOLINT(google-build-using-namespace) enum class [[clang::flag_enum]] execution_space : std::uint8_t { none = 0, - host = 1 << 1, - device = 1 << 2, - tile = 1 << 3, + host = 1 << 0, + device = 1 << 1, + tile = 1 << 2, }; [[nodiscard]] constexpr execution_space operator|(execution_space lhs, execution_space rhs) @@ -51,17 +53,30 @@ constexpr execution_space& operator|=(execution_space& lhs, execution_space rhs) return lhs; } +AST_MATCHER(CXXRecordDecl, isLocalClass) // NOLINT +{ + return Node.isLocalClass() != nullptr; +} + class HideFromABICheck final : public ClangTidyCheck { + static constexpr StringRef IGNORE_TEST_FILES_OPTION{"IgnoreTestFiles"}; + static constexpr StringRef FUNCTION_BIND{""}; - static constexpr StringRef SPECIAL_FUNCTION_BIND{"s"}; + static constexpr StringRef DEFAULTED_METHOD_BIND{"d"}; public: HideFromABICheck(StringRef name, ClangTidyContext* context) : ClangTidyCheck{name, context} , header_extensions_{std::addressof(context->getHeaderFileExtensions())} + , ignore_test_files_{Options.get(IGNORE_TEST_FILES_OPTION, /*Default=*/true)} {} + void storeOptions(ClangTidyOptions::OptionMap& options) override + { + Options.store(options, IGNORE_TEST_FILES_OPTION, ignore_test_files_); + } + [[nodiscard]] std::optional getCheckTraversalKind() const override { return TK_IgnoreUnlessSpelledInSource; @@ -69,69 +84,132 @@ class HideFromABICheck final : public ClangTidyCheck void registerMatchers(MatchFinder* finder) override { - const auto is_special_member = cxxMethodDecl( - isDefaulted(), - anyOf(cxxConstructorDecl(), cxxDestructorDecl(), isCopyAssignmentOperator(), isMoveAssignmentOperator())); + // Check tests can opt in because their inputs also reside in test directories. + const auto file_filter = ignore_test_files_ ? decl(unless(util::isExpansionInTestOrExampleFile())) : decl(); // FunctionDecl includes methods, constructors, destructors, and conversion operators. finder->addMatcher( functionDecl( + // Functions may be declared any number of times, so we need to make sure we check all + // of them. We could do this by removing the isDefinition() condition below, but I + // found that iterating over redecls() inside check() to be faster since we visit each + // function only once. isDefinition(), unless(anyOf(isImplicit(), isDeleted(), isConsteval(), + // Related to the redeclration problem above, there is another issue, + // which is that visibility attributes on just one declaration sets the + // visibility for all re-decls without those re-decls needing to specify + // them again. Since these are inherited, the hasAttr(Visibility) would + // mean we skip all such functions. + // + // On the one hand we could mandate that all redecls repeat the + // attributes, to be safe against the scenario where you accidentally + // delete the redecl that sets the attribute and now the symbol is + // inadvertently public. + // + // But we are already protected against this; since the symbol is public, + // this check would fire and tell you to set a visibility attribute + // (helpfully flagging all re-decls as well). Conflicting visibility is + // also ok since that's a regular old compiler error. hasAttr(attr::Visibility), - cxxMethodDecl(ofClass(cxxRecordDecl(isLambda()))), + // Local class members have no linkage, including members of nested classes. + cxxMethodDecl(ofClass(cxxRecordDecl(anyOf(isLambda(), isLocalClass())))), isExpansionInSystemHeader())), + file_filter, util::isExpansionInHeaderFile(header_extensions_), - optionally(is_special_member.bind(SPECIAL_FUNCTION_BIND))) + optionally(cxxMethodDecl(isDefaulted()).bind(DEFAULTED_METHOD_BIND))) .bind(FUNCTION_BIND), this); } void check(const MatchFinder::MatchResult& result) override { - const auto& nodes = result.Nodes; - const auto* const function = nodes.getNodeAs(FUNCTION_BIND); - const auto& SM = *result.SourceManager; + const auto& nodes = result.Nodes; + const auto& SM = *result.SourceManager; + const auto is_defaulted = nodes.getNodeAs(DEFAULTED_METHOD_BIND) != nullptr; + const auto* const fn = nodes.getNodeAs(FUNCTION_BIND); - if (nodes.getNodeAs(SPECIAL_FUNCTION_BIND)) + for (const auto* const decl : fn->redecls()) { - // Special member functions which are defaulted can't have the usual _CCCL_*_API macros - // because nvcc forbids attaching explicit __host__ __device__ attributes to - // these. Instead they need _CCCL_HIDE_FROM_ABI. - // - // We don't need to do a search because by virtue of being here (the function has no - // visibility attributes) we know that it's missing the macro. - const auto begin = SM.getExpansionLoc(function->getInnerLocStart()); - diag(function->getLocation(), "use _CCCL_HIDE_FROM_ABI to limit ABI visibility") - << FixItHint::CreateInsertion(begin, "_CCCL_HIDE_FROM_ABI "); - return; + if (is_defaulted) + { + handle_defaulted_method_(decl, SM); + } + else if (decl->hasAttr()) + { + handle_kernel_(decl, SM); + } + else + { + handle_regular_functions_(decl, SM); + } } - - handle_regular_functions_(function, SM); } private: + void handle_defaulted_method_(const FunctionDecl* fn, const SourceManager& SM) + { + // Special member functions which are defaulted can't have the usual _CCCL_*_API macros + // because nvcc forbids attaching explicit __host__ __device__ attributes to + // these. Instead they need _CCCL_HIDE_FROM_ABI. + // + // We don't need to do a search because by virtue of being here (the function has no + // visibility attributes) we know that it's missing the macro. + const auto begin = SM.getExpansionLoc(fn->getInnerLocStart()); + + diag(fn->getLocation(), "use _CCCL_HIDE_FROM_ABI to limit ABI visibility") + << FixItHint::CreateInsertion(begin, "_CCCL_HIDE_FROM_ABI "); + } + + void handle_kernel_(const FunctionDecl* fn, const SourceManager& SM) + { + const auto& lang_opts = getLangOpts(); + const auto loc = unwrap_macro_expansions_(SM, fn->getLocation()); + const auto range = get_annotation_range_(fn, SM); + auto buf = SmallString<128>{}; + const auto token = util::findTokenTextInRange(range, SM, lang_opts, [&](const Token& token) { + buf.reserve(token.getLength()); + + const auto* ptr = buf.data(); + const auto len = Lexer::getSpelling(token, ptr, SM, lang_opts); + + return StringRef{ptr, len} == "__global__"; + }); + + auto d = diag(loc, "use _CCCL_KERNEL_ATTRIBUTES to limit ABI visibility") << loc; + + if (token.isValid()) + { + d << FixItHint::CreateReplacement( + CharSourceRange::getTokenRange(token.getBegin(), token.getBegin()), "_CCCL_KERNEL_ATTRIBUTES"); + } + } + void handle_regular_functions_(const FunctionDecl* fn, const SourceManager& SM) { const auto [space, matches] = find_execution_space_(get_annotation_range_(fn, SM), SM); if (matches.empty()) { + const auto loc = unwrap_macro_expansions_(SM, fn->getLocation()); + // Without an execution-space macro, we cannot choose an API macro for the fix-it. - diag(fn->getLocation(), + diag(loc, "function needs an API macro (one of _CCCL_API, _CCCL_DEVICE_API, or _CCCL_HOST_API) to limit ABI " "visibility") - << fn->getLocation(); + << loc; return; } const auto replacement = get_api_macro_(space); - auto d = diag(matches.front().getBegin(), "use %0 instead to limit ABI visibility") - << replacement << matches.front() - << FixItHint::CreateReplacement( - CharSourceRange::getTokenRange(matches.front().getBegin(), matches.front().getBegin()), replacement); + const auto loc = matches.front().getBegin(); + auto d = // + diag(loc, "use %0 instead to limit ABI visibility") // + << replacement // + << matches.front() // + << FixItHint::CreateReplacement(CharSourceRange::getTokenRange(loc, loc), replacement); for (auto&& m : llvm::drop_begin(matches)) { @@ -139,6 +217,31 @@ class HideFromABICheck final : public ClangTidyCheck } } + [[nodiscard]] static SourceLocation unwrap_macro_expansions_(const SourceManager& SM, SourceLocation loc) + { + // Unwrap any macros, otherwise given: + // + // #define MACRO(name) __device__ name() + // + // MACRO(Foo); + // + // we emit: + // + // error: Use _CCCL_DEVICE_API... + // | MACRO(Foo); + // | ^ + // + // Which is obviously not useful. We want to point to "__device__ name()" instead. The + // other path below doesn't need this treatment because it already searches the textual + // definition pointed to by the function definition and therefore implicitly unwraps + // macros. + while (loc.isMacroID() && SM.isMacroArgExpansion(loc)) + { + loc = SM.getImmediateExpansionRange(loc).getBegin(); + } + return SM.getSpellingLoc(loc); + } + [[nodiscard]] static CharSourceRange get_annotation_range_(const FunctionDecl* fn, const SourceManager& SM) { // Getting the begin range here is a little fiddly. We are about to textually search the @@ -180,13 +283,18 @@ class HideFromABICheck final : public ClangTidyCheck const auto& lang_opts = getLangOpts(); auto combined_space = execution_space::none; auto matches = SmallVector{}; + auto remaining = range; + auto buf = SmallString<128>{}; - for (auto remaining = range;;) + while (true) { const auto match = util::findTokenTextInRange(remaining, SM, lang_opts, [&](const Token& token) { - const auto spelling = Lexer::getSpelling(token, SM, lang_opts); + buf.reserve(token.getLength()); + + const auto* ptr = buf.data(); + const auto len = Lexer::getSpelling(token, ptr, SM, lang_opts); const auto space = - llvm::StringSwitch{spelling} + llvm::StringSwitch{{ptr, len}} .Cases({"_CCCL_DEVICE", "__device__"}, execution_space::device) .Cases({"_CCCL_HOST", "__host__"}, execution_space::host) .Cases({"_CCCL_TILE", "__tile__"}, execution_space::tile) @@ -215,7 +323,7 @@ class HideFromABICheck final : public ClangTidyCheck { case execution_space::none: // The caller rejects functions without recognized execution-space annotations. - llvm::reportFatalUsageError("cccl-hide-from-abi: we should not get exec space none here"); + llvm::reportFatalUsageError(CHECK_NAME + ": we should not get exec space none here"); case execution_space::host: return "_CCCL_HOST_API"; case execution_space::device: @@ -230,20 +338,18 @@ class HideFromABICheck final : public ClangTidyCheck case execution_space::host | execution_space::device | execution_space::tile: return "_CCCL_API"; default: - const auto err = "cccl-hide-from-abi: unhandled execution space " + std::to_string(llvm::to_underlying(space)); - - llvm::reportFatalUsageError(StringRef{err}); + llvm::reportFatalUsageError( + CHECK_NAME + ": unhandled execution space " + std::to_string(llvm::to_underlying(space))); } } const FileExtensionsSet* header_extensions_{}; + bool ignore_test_files_{}; }; class HideFromABICheckModule final : public ClangTidyModule { public: - static constexpr StringRef CHECK_NAME{"cccl-hide-from-abi"}; - void addCheckFactories(ClangTidyCheckFactories& check_factories) override { check_factories.registerCheck(CHECK_NAME); @@ -252,6 +358,6 @@ class HideFromABICheckModule final : public ClangTidyModule // NOLINTNEXTLINE(cert-err58-cpp, bugprone-throwing-static-initialization) ClangTidyModuleRegistry::Add _{ - HideFromABICheckModule::CHECK_NAME, "Checks for missing ABI annotations on function definitions in headers"}; + CHECK_NAME, "Checks for missing ABI annotations on function definitions in headers"}; } // namespace } // namespace clang::tidy::cccl diff --git a/cmake/clang_tidy_plugins/hide_from_abi/test/CMakeLists.txt b/cmake/clang_tidy_plugins/hide_from_abi/test/CMakeLists.txt index 3a5a3c598344..425d7685572e 100644 --- a/cmake/clang_tidy_plugins/hide_from_abi/test/CMakeLists.txt +++ b/cmake/clang_tidy_plugins/hide_from_abi/test/CMakeLists.txt @@ -1,21 +1,28 @@ list(APPEND CMAKE_MESSAGE_CONTEXT "test") file(GLOB test_srcs CONFIGURE_DEPENDS "*.hpp") +file(GLOB cu_test_srcs CONFIGURE_DEPENDS "*.cuh") -foreach (src IN LISTS test_srcs) - cmake_path(GET src STEM name) - foreach ( - extension - IN - ITEMS "" h hh hpp hxx cuh inl ipp - ) - set(header "${CMAKE_CURRENT_BINARY_DIR}/${name}_${extension}") - if (NOT extension STREQUAL "") - string(APPEND header ".${extension}") - endif() - configure_file("${src}" "${header}" COPYONLY) - # Parse headers directly so the diagnostic line filter includes their declarations. - set_source_files_properties("${header}" PROPERTIES LANGUAGE CXX) - cccl_create_clang_tidy_test(hide_from_abi "${header}") +function(cccl_add_abi_tests LANG) + foreach (src IN LISTS ARGN) + cmake_path(GET src STEM name) + foreach ( + extension + IN + ITEMS "" h hh hpp hxx cuh inl ipp + ) + set(header "${CMAKE_CURRENT_BINARY_DIR}/${name}_${extension}") + if (NOT extension STREQUAL "") + string(APPEND header ".${extension}") + endif() + configure_file("${src}" "${header}" COPYONLY) + # Parse headers directly so the diagnostic line filter includes their declarations. + set_source_files_properties("${header}" PROPERTIES LANGUAGE "${LANG}") + cccl_create_clang_tidy_test(hide_from_abi "${header}") + endforeach() endforeach() -endforeach() +endfunction() + +cccl_add_abi_tests(CXX ${test_srcs}) +enable_language(CUDA) +cccl_add_abi_tests(CUDA ${cu_test_srcs}) diff --git a/cmake/clang_tidy_plugins/hide_from_abi/test/classes.hpp b/cmake/clang_tidy_plugins/hide_from_abi/test/classes.hpp new file mode 100644 index 000000000000..837ab8a20f2e --- /dev/null +++ b/cmake/clang_tidy_plugins/hide_from_abi/test/classes.hpp @@ -0,0 +1,298 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// clang-format off +// CLANG_TIDY_CONFIG_BEGIN +// Checks: cccl-hide-from-abi +// CheckOptions: cccl-hide-from-abi.IgnoreTestFiles=false +// CLANG_TIDY_CONFIG_END + +#include + +#undef _CCCL_HOST +#undef _CCCL_DEVICE +#undef _CCCL_HOST_DEVICE +// Attribute expansions keep the annotation in the declaration's source range. +#define _CCCL_HOST __attribute__((annotate("host"))) +#define _CCCL_DEVICE __attribute__((annotate("device"))) +#define _CCCL_HOST_DEVICE _CCCL_HOST _CCCL_DEVICE + +// CHECK-MESSAGES-NOT: warning: +struct members { + _CCCL_HOST members() = default; +// CHECK-MESSAGES: :[[@LINE-1]]:14: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + _CCCL_HOST ~members() = default; +// CHECK-MESSAGES: :[[@LINE-1]]:14: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + _CCCL_DEVICE operator bool() const { return true; } +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + _CCCL_HOST_DEVICE members& operator=(const members&) = default; +// CHECK-MESSAGES: :[[@LINE-1]]:30: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + _CCCL_DEVICE void method() {} +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + _CCCL_HOST static void static_method() {} +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + _CCCL_HOST virtual void virtual_method() {} +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + _CCCL_HOST friend void friend_function(members) {} +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + void declared(); +// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: function needs an API macro (one of _CCCL_API, _CCCL_DEVICE_API, or _CCCL_HOST_API) to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +}; +_CCCL_HOST inline void members::declared() {} +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + +// CHECK-FIXES: Replacements: [] +// CHECK-FIXES: Length: 10 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_API + +struct implicit_members { int value; }; +inline implicit_members object{}; +inline auto copied_object = object; + +template struct container { + _CCCL_HOST container() = default; +// CHECK-MESSAGES: :[[@LINE-1]]:14: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + _CCCL_DEVICE void method() {} +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +}; +template struct container; +extern template struct container; +inline container implicit_instantiation; + +template <> struct container { + _CCCL_DEVICE void specialized_method() {} +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +}; +template struct container { + _CCCL_DEVICE void partial_method() {} +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +}; +inline container partial_instantiation; + +// CHECK-FIXES: DiagnosticName: cccl-hide-from-abi +// CHECK-FIXES: Length: 0 +// CHECK-FIXES-NEXT: ReplacementText: '_CCCL_HIDE_FROM_ABI ' +// CHECK-FIXES: Length: 12 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_DEVICE_API +// CHECK-FIXES: Length: 12 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_DEVICE_API +// CHECK-FIXES: Length: 12 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_DEVICE_API + +struct __attribute__((visibility("hidden"))) hidden_class { + _CCCL_HOST void unannotated_method() {} +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +}; + +// CHECK-FIXES: Length: 10 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_API + +namespace defaulted_special_members { +// CHECK-MESSAGES-NOT: warning: +struct members { + members() = default; +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + members(const members&) = default; +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + members(members&&) = default; +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + members& operator=(const members&) = default; +// CHECK-MESSAGES: :[[@LINE-1]]:12: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + members& operator=(members&&) = default; +// CHECK-MESSAGES: :[[@LINE-1]]:12: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + ~members() = default; +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +}; + +// CHECK-FIXES: Replacements: +// CHECK-FIXES: Length: 0 +// CHECK-FIXES-NEXT: ReplacementText: '_CCCL_HIDE_FROM_ABI ' +// CHECK-FIXES: Replacements: +// CHECK-FIXES: Length: 0 +// CHECK-FIXES-NEXT: ReplacementText: '_CCCL_HIDE_FROM_ABI ' +// CHECK-FIXES: Replacements: +// CHECK-FIXES: Length: 0 +// CHECK-FIXES-NEXT: ReplacementText: '_CCCL_HIDE_FROM_ABI ' +// CHECK-FIXES: Replacements: +// CHECK-FIXES: Length: 0 +// CHECK-FIXES-NEXT: ReplacementText: '_CCCL_HIDE_FROM_ABI ' +// CHECK-FIXES: Replacements: +// CHECK-FIXES: Length: 0 +// CHECK-FIXES-NEXT: ReplacementText: '_CCCL_HIDE_FROM_ABI ' +// CHECK-FIXES: Replacements: +// CHECK-FIXES: Length: 0 +// CHECK-FIXES-NEXT: ReplacementText: '_CCCL_HIDE_FROM_ABI ' + +struct annotated_members { + _CCCL_HIDE_FROM_ABI annotated_members() = default; + _CCCL_HIDE_FROM_ABI annotated_members(const annotated_members&) = default; + _CCCL_HIDE_FROM_ABI annotated_members(annotated_members&&) = default; + _CCCL_HIDE_FROM_ABI annotated_members& operator=(const annotated_members&) = default; + _CCCL_HIDE_FROM_ABI annotated_members& operator=(annotated_members&&) = default; + _CCCL_HIDE_FROM_ABI ~annotated_members() = default; +}; + +template struct container { + constexpr explicit container() = default; +// CHECK-MESSAGES: :[[@LINE-1]]:22: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +}; +template struct container; +inline container implicit_instantiation; + +// CHECK-FIXES: Replacements: +// CHECK-FIXES: Length: 0 +// CHECK-FIXES-NEXT: ReplacementText: '_CCCL_HIDE_FROM_ABI ' + +struct out_of_line { + out_of_line(); +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +// CHECK-FIXES: Replacements: +// CHECK-FIXES: Length: 0 +// CHECK-FIXES-NEXT: ReplacementText: '_CCCL_HIDE_FROM_ABI ' +}; +inline out_of_line::out_of_line() = default; +// CHECK-MESSAGES: :[[@LINE-1]]:21: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +// CHECK-FIXES: Replacements: +// CHECK-FIXES: Length: 0 +// CHECK-FIXES-NEXT: ReplacementText: '_CCCL_HIDE_FROM_ABI ' + +} // namespace defaulted_special_members + +namespace member_redeclarations { +struct members; +struct members; +struct members { + _CCCL_HOST members(); +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + _CCCL_HOST ~members(); +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + _CCCL_HOST void method(); +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + __attribute__((visibility("hidden"))) void hidden_method(); +}; +_CCCL_HOST inline members::members() {} +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +_CCCL_HOST inline members::~members() {} +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +_CCCL_HOST inline void members::method() {} +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +_CCCL_HOST inline void members::hidden_method() {} + +// CHECK-FIXES: Length: 10 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_API +// CHECK-FIXES: Length: 10 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_API +// CHECK-FIXES: Length: 10 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_API +// CHECK-FIXES: Length: 10 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_API +// CHECK-FIXES: Length: 10 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_API +// CHECK-FIXES: Length: 10 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_API + +template struct container; +template struct container { + template + _CCCL_HOST_DEVICE U method(U value); +// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HOST_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +}; +template +template +_CCCL_HOST_DEVICE U container::method(U value) { return value; } +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_HOST_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +inline auto instantiated = container{}.method(1); +inline auto also_instantiated = container{}.method(1.0); + +// CHECK-FIXES: Length: 17 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_DEVICE_API +// CHECK-FIXES: Length: 17 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_DEVICE_API +} // namespace member_redeclarations + +// The enclosing function still needs an annotation; its local members do not. +// CHECK-MESSAGES-NOT: warning: +inline void enclosing_function() { +// CHECK-MESSAGES: :[[@LINE-1]]:13: warning: function needs an API macro (one of _CCCL_API, _CCCL_DEVICE_API, or _CCCL_HOST_API) to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + struct local { + local() = default; + local(const local&) = default; + local(local&&) = default; + local& operator=(const local&) = default; + local& operator=(local&&) = default; + ~local() = default; + explicit local(int) {} + _CCCL_HOST void method() {} + static void static_method() {} + operator bool() const { return true; } + struct nested { + nested() = default; + void method() {} + }; + }; +} + +template +_CCCL_HOST_API inline auto enclosing_template(T value) { + struct local { + T value; + explicit local(T value) : value(value) {} + ~local() {} + T get() const { return value; } + }; + return local{value}; +} +inline auto instantiated_local = enclosing_template(1); + +inline auto enclosing_lambda = [] { + struct local { + local() = default; + explicit local(int) {} + ~local() {} + void method() {} + struct nested { + nested() = default; + void method() {} + }; + }; + return local{1}; +}; +inline auto lambda_local = enclosing_lambda(); + +// CHECK-FIXES: DiagnosticName: cccl-hide-from-abi +// CHECK-FIXES: Replacements: [] +// CHECK-FIXES-NOT: DiagnosticName: +// clang-format on diff --git a/cmake/clang_tidy_plugins/hide_from_abi/test/execution_spaces.hpp b/cmake/clang_tidy_plugins/hide_from_abi/test/execution_spaces.hpp index 493abdd7551a..a7cad43207d5 100644 --- a/cmake/clang_tidy_plugins/hide_from_abi/test/execution_spaces.hpp +++ b/cmake/clang_tidy_plugins/hide_from_abi/test/execution_spaces.hpp @@ -4,17 +4,9 @@ // clang-format off // CLANG_TIDY_CONFIG_BEGIN // Checks: cccl-hide-from-abi +// CheckOptions: cccl-hide-from-abi.IgnoreTestFiles=false // CLANG_TIDY_CONFIG_END -#include - -#undef _CCCL_HOST -#undef _CCCL_DEVICE -#undef _CCCL_TILE -#undef _CCCL_HOST_DEVICE -#undef __host__ -#undef __device__ -#undef __tile__ // Attribute expansions keep the annotation in the declaration's source range. #define __host__ __attribute__((annotate("host"))) #define __device__ __attribute__((annotate("device"))) diff --git a/cmake/clang_tidy_plugins/hide_from_abi/test/functions.hpp b/cmake/clang_tidy_plugins/hide_from_abi/test/functions.hpp index 8c8b9f0bd992..4735fe47138f 100644 --- a/cmake/clang_tidy_plugins/hide_from_abi/test/functions.hpp +++ b/cmake/clang_tidy_plugins/hide_from_abi/test/functions.hpp @@ -4,6 +4,7 @@ // clang-format off // CLANG_TIDY_CONFIG_BEGIN // Checks: cccl-hide-from-abi +// CheckOptions: cccl-hide-from-abi.IgnoreTestFiles=false // CLANG_TIDY_CONFIG_END #include @@ -26,46 +27,12 @@ _CCCL_DEVICE constexpr int device_function() { return 1; } _CCCL_HOST_DEVICE inline void host_device_function() {} // CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_HOST_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] // CHECK-MESSAGES-NOT: warning: -struct members { - _CCCL_HOST members() = default; -// CHECK-MESSAGES: :[[@LINE-1]]:14: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] -// CHECK-MESSAGES-NOT: warning: - _CCCL_HOST ~members() = default; -// CHECK-MESSAGES: :[[@LINE-1]]:14: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] -// CHECK-MESSAGES-NOT: warning: - _CCCL_DEVICE operator bool() const { return true; } -// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] -// CHECK-MESSAGES-NOT: warning: - _CCCL_HOST_DEVICE members& operator=(const members&) = default; -// CHECK-MESSAGES: :[[@LINE-1]]:30: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] -// CHECK-MESSAGES-NOT: warning: - _CCCL_DEVICE void method() {} -// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] -// CHECK-MESSAGES-NOT: warning: - _CCCL_HOST static void static_method() {} -// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] -// CHECK-MESSAGES-NOT: warning: - _CCCL_HOST virtual void virtual_method() {} -// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] -// CHECK-MESSAGES-NOT: warning: - _CCCL_HOST friend void friend_function(members) {} -// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] -// CHECK-MESSAGES-NOT: warning: - void declared(); -}; -_CCCL_HOST inline void members::declared() {} -// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] -// CHECK-MESSAGES-NOT: warning: - inline void unannotated() {} // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: function needs an API macro (one of _CCCL_API, _CCCL_DEVICE_API, or _CCCL_HOST_API) to limit ABI visibility [cccl-hide-from-abi] // CHECK-MESSAGES-NOT: warning: _CCCL_HOST void declaration_only(); _CCCL_DEVICE void deleted_function() = delete; _CCCL_DEVICE inline int variable = 0; -struct implicit_members { int value; }; -inline implicit_members object{}; -inline auto copied_object = object; inline auto lambda = []() _CCCL_HOST { return 1; }; inline int lambda_result = lambda(); @@ -89,30 +56,6 @@ template <> _CCCL_HOST inline char identity(char value) { return value; } // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] // CHECK-MESSAGES-NOT: warning: -template struct container { - _CCCL_HOST container() = default; -// CHECK-MESSAGES: :[[@LINE-1]]:14: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] -// CHECK-MESSAGES-NOT: warning: - _CCCL_DEVICE void method() {} -// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] -// CHECK-MESSAGES-NOT: warning: -}; -template struct container; -extern template struct container; -inline container implicit_instantiation; - -template <> struct container { - _CCCL_DEVICE void specialized_method() {} -// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] -// CHECK-MESSAGES-NOT: warning: -}; -template struct container { - _CCCL_DEVICE void partial_method() {} -// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] -// CHECK-MESSAGES-NOT: warning: -}; -inline container partial_instantiation; - // CHECK-FIXES: DiagnosticName: cccl-hide-from-abi // CHECK-FIXES: Length: 17 // CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_DEVICE_API @@ -133,12 +76,6 @@ _CCCL_HOST __attribute__((exclude_from_explicit_instantiation)) inline void excl // CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] // CHECK-MESSAGES-NOT: warning: -struct __attribute__((visibility("hidden"))) hidden_class { - _CCCL_HOST void unannotated_method() {} -// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HOST_API instead to limit ABI visibility [cccl-hide-from-abi] -// CHECK-MESSAGES-NOT: warning: -}; - // NOLINTNEXTLINE(cccl-hide-from-abi) _CCCL_HOST inline void suppressed() {} @@ -176,83 +113,9 @@ inline _CCCL_HOST_DEVICE void middle_annotation() {} #define DEFINE_FUNCTION(name) inline void name() {} DEFINE_FUNCTION(macro_function) -// CHECK-MESSAGES: :[[@LINE-1]]:17: warning: function needs an API macro (one of _CCCL_API, _CCCL_DEVICE_API, or _CCCL_HOST_API) to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES: :[[@LINE-2]]:43: warning: function needs an API macro (one of _CCCL_API, _CCCL_DEVICE_API, or _CCCL_HOST_API) to limit ABI visibility [cccl-hide-from-abi] // CHECK-MESSAGES-NOT: warning: // CHECK-FIXES: DiagnosticName: cccl-hide-from-abi // CHECK-FIXES: Replacements: [] -namespace defaulted_special_members { -// CHECK-MESSAGES-NOT: warning: -struct members { - members() = default; -// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] -// CHECK-MESSAGES-NOT: warning: - members(const members&) = default; -// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] -// CHECK-MESSAGES-NOT: warning: - members(members&&) = default; -// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] -// CHECK-MESSAGES-NOT: warning: - members& operator=(const members&) = default; -// CHECK-MESSAGES: :[[@LINE-1]]:12: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] -// CHECK-MESSAGES-NOT: warning: - members& operator=(members&&) = default; -// CHECK-MESSAGES: :[[@LINE-1]]:12: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] -// CHECK-MESSAGES-NOT: warning: - ~members() = default; -// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] -// CHECK-MESSAGES-NOT: warning: -}; - -// CHECK-FIXES: Replacements: -// CHECK-FIXES: Length: 0 -// CHECK-FIXES-NEXT: ReplacementText: '_CCCL_HIDE_FROM_ABI ' -// CHECK-FIXES: Replacements: -// CHECK-FIXES: Length: 0 -// CHECK-FIXES-NEXT: ReplacementText: '_CCCL_HIDE_FROM_ABI ' -// CHECK-FIXES: Replacements: -// CHECK-FIXES: Length: 0 -// CHECK-FIXES-NEXT: ReplacementText: '_CCCL_HIDE_FROM_ABI ' -// CHECK-FIXES: Replacements: -// CHECK-FIXES: Length: 0 -// CHECK-FIXES-NEXT: ReplacementText: '_CCCL_HIDE_FROM_ABI ' -// CHECK-FIXES: Replacements: -// CHECK-FIXES: Length: 0 -// CHECK-FIXES-NEXT: ReplacementText: '_CCCL_HIDE_FROM_ABI ' -// CHECK-FIXES: Replacements: -// CHECK-FIXES: Length: 0 -// CHECK-FIXES-NEXT: ReplacementText: '_CCCL_HIDE_FROM_ABI ' - -struct annotated_members { - _CCCL_HIDE_FROM_ABI annotated_members() = default; - _CCCL_HIDE_FROM_ABI annotated_members(const annotated_members&) = default; - _CCCL_HIDE_FROM_ABI annotated_members(annotated_members&&) = default; - _CCCL_HIDE_FROM_ABI annotated_members& operator=(const annotated_members&) = default; - _CCCL_HIDE_FROM_ABI annotated_members& operator=(annotated_members&&) = default; - _CCCL_HIDE_FROM_ABI ~annotated_members() = default; -}; - -template struct container { - constexpr explicit container() = default; -// CHECK-MESSAGES: :[[@LINE-1]]:22: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] -// CHECK-MESSAGES-NOT: warning: -}; -template struct container; -inline container implicit_instantiation; - -// CHECK-FIXES: Replacements: -// CHECK-FIXES: Length: 0 -// CHECK-FIXES-NEXT: ReplacementText: '_CCCL_HIDE_FROM_ABI ' - -struct out_of_line { - out_of_line(); -}; -inline out_of_line::out_of_line() = default; -// CHECK-MESSAGES: :[[@LINE-1]]:21: warning: use _CCCL_HIDE_FROM_ABI to limit ABI visibility [cccl-hide-from-abi] -// CHECK-MESSAGES-NOT: warning: -// CHECK-FIXES: Replacements: -// CHECK-FIXES: Length: 0 -// CHECK-FIXES-NEXT: ReplacementText: '_CCCL_HIDE_FROM_ABI ' - -} // namespace defaulted_special_members // clang-format on diff --git a/cmake/clang_tidy_plugins/hide_from_abi/test/kernels.cuh b/cmake/clang_tidy_plugins/hide_from_abi/test/kernels.cuh new file mode 100644 index 000000000000..9d08f581cc57 --- /dev/null +++ b/cmake/clang_tidy_plugins/hide_from_abi/test/kernels.cuh @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// clang-format off +// CLANG_TIDY_CONFIG_BEGIN +// Checks: cccl-hide-from-abi +// CheckOptions: cccl-hide-from-abi.IgnoreTestFiles=false +// CLANG_TIDY_CONFIG_END + +#include + +#define __global__ __attribute__((global)) + +// CHECK-MESSAGES-NOT: warning: +__global__ void kernel() {} +// CHECK-MESSAGES: :[[@LINE-1]]:17: warning: use _CCCL_KERNEL_ATTRIBUTES to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +// CHECK-FIXES: DiagnosticName: cccl-hide-from-abi +// CHECK-FIXES: Length: 10 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_KERNEL_ATTRIBUTES + +// CHECK-MESSAGES-NOT: warning: +_CCCL_KERNEL_ATTRIBUTES void annotated_kernel() {} diff --git a/cmake/clang_tidy_plugins/hide_from_abi/test/redeclarations.hpp b/cmake/clang_tidy_plugins/hide_from_abi/test/redeclarations.hpp new file mode 100644 index 000000000000..e16820344951 --- /dev/null +++ b/cmake/clang_tidy_plugins/hide_from_abi/test/redeclarations.hpp @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// clang-format off +// CLANG_TIDY_CONFIG_BEGIN +// Checks: cccl-hide-from-abi +// CheckOptions: cccl-hide-from-abi.IgnoreTestFiles=false +// CLANG_TIDY_CONFIG_END + +#define _CCCL_HOST __attribute__((annotate("host"))) +#define _CCCL_DEVICE __attribute__((annotate("device"))) +#define _CCCL_HOST_DEVICE _CCCL_HOST _CCCL_DEVICE + +// CHECK-MESSAGES-NOT: warning: +_CCCL_HOST_DEVICE inline void redeclared_function(); +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_HOST_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +_CCCL_HOST_DEVICE inline void redeclared_function(); +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_HOST_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +_CCCL_HOST_DEVICE inline void redeclared_function() {} +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_HOST_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + +// CHECK-FIXES: Length: 17 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_DEVICE_API +// CHECK-FIXES: Length: 17 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_DEVICE_API +// CHECK-FIXES: Length: 17 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_DEVICE_API + +template +_CCCL_HOST_DEVICE T redeclared(T value); +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_HOST_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +template +_CCCL_HOST_DEVICE T redeclared(T value); +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_HOST_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: +template +_CCCL_HOST_DEVICE T redeclared(T value) { return value; } +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use _CCCL_HOST_DEVICE_API instead to limit ABI visibility [cccl-hide-from-abi] +// CHECK-MESSAGES-NOT: warning: + +// CHECK-FIXES: Length: 17 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_DEVICE_API +// CHECK-FIXES: Length: 17 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_DEVICE_API +// CHECK-FIXES: Length: 17 +// CHECK-FIXES-NEXT: ReplacementText: _CCCL_HOST_DEVICE_API +// clang-format on diff --git a/cub/benchmarks/bench/bitonic_sort/bitonic_common.cuh b/cub/benchmarks/bench/bitonic_sort/bitonic_common.cuh index 385e87b7bc69..58e06bce00f5 100644 --- a/cub/benchmarks/bench/bitonic_sort/bitonic_common.cuh +++ b/cub/benchmarks/bench/bitonic_sort/bitonic_common.cuh @@ -46,7 +46,7 @@ NVBENCH_DECLARE_ENUM_TYPE_STRINGS( }) template -void run_bench(nvbench::state& state) +_CCCL_HOST_API void run_bench(nvbench::state& state) { constexpr int items_per_thread = Len / warp_threads; const auto kernel = benchmark_kernel; @@ -77,7 +77,7 @@ void run_bench(nvbench::state& state) struct CustomLess { template - __device__ bool operator()(const T& lhs, const T& rhs) const + _CCCL_DEVICE_API bool operator()(const T& lhs, const T& rhs) const { return lhs < rhs; } diff --git a/cub/benchmarks/bench/find_bound/find_bound_common.cuh b/cub/benchmarks/bench/find_bound/find_bound_common.cuh index 7aa4b1c851ab..baa34f22d6c3 100644 --- a/cub/benchmarks/bench/find_bound/find_bound_common.cuh +++ b/cub/benchmarks/bench/find_bound/find_bound_common.cuh @@ -22,7 +22,7 @@ struct bounds_bench_data std::size_t elements{}; std::size_t needles{}; - explicit bounds_bench_data(nvbench::state& state) + _CCCL_HOST_API explicit bounds_bench_data(nvbench::state& state) { elements = static_cast(state.get_int64("Elements{io}")); const auto needles_ratio = static_cast(state.get_int64("NeedlesRatio")); @@ -35,23 +35,23 @@ struct bounds_bench_data data.begin() + static_cast::difference_type>(elements)); } - void sort_needles() + _CCCL_HOST_API void sort_needles() { thrust::sort(data.begin() + static_cast::difference_type>(elements), data.end()); } - T* range_ptr() + _CCCL_HOST_API T* range_ptr() { return thrust::raw_pointer_cast(data.data()); } - T* values_ptr() + _CCCL_HOST_API T* values_ptr() { return thrust::raw_pointer_cast( data.data() + static_cast::difference_type>(elements)); } - std::ptrdiff_t* output_ptr() + _CCCL_HOST_API std::ptrdiff_t* output_ptr() { return thrust::raw_pointer_cast(result.data()); } diff --git a/cub/benchmarks/bench/histogram/histogram_common.cuh b/cub/benchmarks/bench/histogram/histogram_common.cuh index c87565264042..bcef9e20c946 100644 --- a/cub/benchmarks/bench/histogram/histogram_common.cuh +++ b/cub/benchmarks/bench/histogram/histogram_common.cuh @@ -59,7 +59,7 @@ struct bench_policy_selector #endif // !TUNE_BASE template -SampleT get_upper_level(OffsetT bins, OffsetT elements) +_CCCL_HOST_API SampleT get_upper_level(OffsetT bins, OffsetT elements) { if constexpr (cuda::std::is_integral_v) { diff --git a/cub/benchmarks/bench/merge/merge_common.cuh b/cub/benchmarks/bench/merge/merge_common.cuh index 146aaa3bfe8f..49f2fb6cb646 100644 --- a/cub/benchmarks/bench/merge/merge_common.cuh +++ b/cub/benchmarks/bench/merge/merge_common.cuh @@ -47,7 +47,7 @@ struct select_if_less_than_t bool negate; uint8_t threshold; - __device__ __forceinline__ bool operator()(uint8_t val) const + _CCCL_DEVICE_API __forceinline__ bool operator()(uint8_t val) const { return negate ? !(val < threshold) : val < threshold; } @@ -59,7 +59,7 @@ struct write_pivot_point_t OffsetT threshold; OffsetT* pivot_point; - __device__ void operator()(OffsetT output_index, OffsetT input_index) const + _CCCL_DEVICE_API void operator()(OffsetT output_index, OffsetT input_index) const { if (output_index == threshold) { @@ -69,7 +69,7 @@ struct write_pivot_point_t }; template -std::pair, thrust::device_vector> +_CCCL_HOST_API std::pair, thrust::device_vector> generate_lhs_rhs(std::size_t num_items_lhs, std::size_t num_items_rhs, bit_entropy entropy) { using offset_t = std::size_t; diff --git a/cub/benchmarks/bench/radix_sort/policy_selector.h b/cub/benchmarks/bench/radix_sort/policy_selector.h index 671bc20f2e93..634d19ebfe9f 100644 --- a/cub/benchmarks/bench/radix_sort/policy_selector.h +++ b/cub/benchmarks/bench/radix_sort/policy_selector.h @@ -126,7 +126,7 @@ constexpr bool fits_in_default_shared_memory() } #else // TUNE_BASE template -constexpr bool fits_in_default_shared_memory() +_CCCL_HOST_API constexpr bool fits_in_default_shared_memory() { return true; } diff --git a/cub/benchmarks/bench/reduce/base.cuh b/cub/benchmarks/bench/reduce/base.cuh index fc3c061d0652..76cdedacb294 100644 --- a/cub/benchmarks/bench/reduce/base.cuh +++ b/cub/benchmarks/bench/reduce/base.cuh @@ -23,7 +23,7 @@ struct policy_selector #endif // !TUNE_BASE template -void reduce(nvbench::state& state, nvbench::type_list) +_CCCL_HOST_API void reduce(nvbench::state& state, nvbench::type_list) { using init_value_t = T; diff --git a/cub/benchmarks/bench/reduce/warp_reduce_base.cuh b/cub/benchmarks/bench/reduce/warp_reduce_base.cuh index 653799f8d600..812accc9da71 100644 --- a/cub/benchmarks/bench/reduce/warp_reduce_base.cuh +++ b/cub/benchmarks/bench/reduce/warp_reduce_base.cuh @@ -12,7 +12,7 @@ struct benchmark_op_t { template - __device__ __forceinline__ T operator()(T thread_data) const + _CCCL_DEVICE_API __forceinline__ T operator()(T thread_data) const { using WarpReduce = cub::WarpReduce; using TempStorage = typename WarpReduce::TempStorage; @@ -23,7 +23,7 @@ struct benchmark_op_t }; template -void warp_reduce(nvbench::state& state, nvbench::type_list) +_CCCL_HOST_API void warp_reduce(nvbench::state& state, nvbench::type_list) { constexpr int block_size = 256; constexpr int unroll_factor = 128; // compromise between compile time and noise diff --git a/cub/benchmarks/bench/run_length_encode/base.cuh b/cub/benchmarks/bench/run_length_encode/base.cuh index 6da4012456e1..7edf3644f1c5 100644 --- a/cub/benchmarks/bench/run_length_encode/base.cuh +++ b/cub/benchmarks/bench/run_length_encode/base.cuh @@ -11,7 +11,7 @@ //! @tparam RunLengthT Offset type large enough to represent the longest run in the sequence template -void rle(nvbench::state& state, nvbench::type_list) +_CCCL_HOST_API void rle(nvbench::state& state, nvbench::type_list) { // Offset type large enough to represent any offset into the input sequence and the total number of runs using offset_t = cub::detail::choose_signed_offset_t; diff --git a/cub/benchmarks/bench/scan/exclusive/base.cuh b/cub/benchmarks/bench/scan/exclusive/base.cuh index b01f25a49932..1a5e10903055 100644 --- a/cub/benchmarks/bench/scan/exclusive/base.cuh +++ b/cub/benchmarks/bench/scan/exclusive/base.cuh @@ -12,7 +12,7 @@ #include "../policy_selector.h" template -void basic(nvbench::state& state, nvbench::type_list) +_CCCL_HOST_API void basic(nvbench::state& state, nvbench::type_list) try { using init_value_t = T; diff --git a/cub/benchmarks/bench/scan/warp_scan_base.cuh b/cub/benchmarks/bench/scan/warp_scan_base.cuh index d3e48334a685..35a41c944b5d 100644 --- a/cub/benchmarks/bench/scan/warp_scan_base.cuh +++ b/cub/benchmarks/bench/scan/warp_scan_base.cuh @@ -12,7 +12,7 @@ struct benchmark_op_t { template - __device__ __forceinline__ T operator()(T thread_data) const + _CCCL_DEVICE_API __forceinline__ T operator()(T thread_data) const { using WarpScan = cub::WarpScan; using TempStorage = typename WarpScan::TempStorage; @@ -25,7 +25,7 @@ struct benchmark_op_t }; template -void warp_scan(nvbench::state& state, nvbench::type_list) +_CCCL_HOST_API void warp_scan(nvbench::state& state, nvbench::type_list) { constexpr int block_size = 256; constexpr int unroll_factor = 128; // compromise between compile time and noise diff --git a/cub/benchmarks/bench/segmented_reduce/base.cuh b/cub/benchmarks/bench/segmented_reduce/base.cuh index a8dc75e5393e..015800aa7b82 100644 --- a/cub/benchmarks/bench/segmented_reduce/base.cuh +++ b/cub/benchmarks/bench/segmented_reduce/base.cuh @@ -41,7 +41,7 @@ struct policy_selector #endif // !TUNE_BASE template -void fixed_size_segmented_reduce(nvbench::state& state, nvbench::type_list) +_CCCL_HOST_API void fixed_size_segmented_reduce(nvbench::state& state, nvbench::type_list) { static constexpr bool is_argmin = std::is_same_v; diff --git a/cub/benchmarks/bench/segmented_reduce/variable_base.cuh b/cub/benchmarks/bench/segmented_reduce/variable_base.cuh index cda9a599393c..292bd440faac 100644 --- a/cub/benchmarks/bench/segmented_reduce/variable_base.cuh +++ b/cub/benchmarks/bench/segmented_reduce/variable_base.cuh @@ -23,7 +23,7 @@ using some_offset_types = nvbench::type_list; #endif template -void variable_segmented_reduce(nvbench::state& state, nvbench::type_list) +_CCCL_HOST_API void variable_segmented_reduce(nvbench::state& state, nvbench::type_list) { static constexpr bool is_argmin = std::is_same_v; static constexpr bool is_argmax = std::is_same_v; diff --git a/cub/benchmarks/bench/segmented_scan/base.cuh b/cub/benchmarks/bench/segmented_scan/base.cuh index cf554bc8c5ea..8e4a12206f33 100644 --- a/cub/benchmarks/bench/segmented_scan/base.cuh +++ b/cub/benchmarks/bench/segmented_scan/base.cuh @@ -51,7 +51,7 @@ struct to_offsets_functor OffsetT segment_size; OffsetT wobble; - __host__ __device__ __forceinline__ OffsetT operator()(size_t i) const + _CCCL_HOST_DEVICE_API __forceinline__ OffsetT operator()(size_t i) const { const auto fixed_size_value = static_cast(i) * segment_size; const auto correction = ((i & 1) ? wobble : OffsetT{0}); @@ -61,7 +61,7 @@ struct to_offsets_functor }; template -void bench_impl(nvbench::state& state, nvbench::type_list) +_CCCL_HOST_API void bench_impl(nvbench::state& state, nvbench::type_list) { #if !TUNE_BASE using policy_t = policy_selector_t; @@ -115,13 +115,13 @@ void bench_impl(nvbench::state& state, nvbench::type_list) } template -void fixed_segment_size_bench(nvbench::state& state, nvbench::type_list tl) +_CCCL_HOST_API void fixed_segment_size_bench(nvbench::state& state, nvbench::type_list tl) { return bench_impl<0, T, OffsetT>(state, tl); } template -void varying_segment_size_bench(nvbench::state& state, nvbench::type_list tl) +_CCCL_HOST_API void varying_segment_size_bench(nvbench::state& state, nvbench::type_list tl) { return bench_impl<1, T, OffsetT>(state, tl); } diff --git a/cub/benchmarks/bench/segmented_topk/variable/common.cuh b/cub/benchmarks/bench/segmented_topk/variable/common.cuh index c40382965f20..432a50507426 100644 --- a/cub/benchmarks/bench/segmented_topk/variable/common.cuh +++ b/cub/benchmarks/bench/segmented_topk/variable/common.cuh @@ -33,7 +33,7 @@ enum class pattern_kind : int pivot_tie }; -[[nodiscard]] inline pattern_kind string_to_pattern(const std::string& pattern) +[[nodiscard]] _CCCL_HOST_API inline pattern_kind string_to_pattern(const std::string& pattern) { if (pattern == "random") { @@ -59,7 +59,7 @@ enum class pattern_kind : int } template -[[nodiscard]] thrust::device_vector +[[nodiscard]] _CCCL_HOST_API thrust::device_vector gen_data(int num_segments, pattern_kind pattern, const segment_size_t* d_seg_sizes) { const auto num_keys = static_cast(num_segments) * static_cast(MaxSegmentSize); diff --git a/cub/benchmarks/bench/segmented_topk/variable/indexed_common.cuh b/cub/benchmarks/bench/segmented_topk/variable/indexed_common.cuh index 31716ae0d3d4..9764c74d7b76 100644 --- a/cub/benchmarks/bench/segmented_topk/variable/indexed_common.cuh +++ b/cub/benchmarks/bench/segmented_topk/variable/indexed_common.cuh @@ -111,7 +111,7 @@ inline constexpr auto selected_tie_break = template struct topk_backend_selector { - [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability cc) const + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(cuda::compute_capability cc) const -> cub::detail::batched_topk::topk_policy { // The baseline backend cannot honor a deterministic result set / concrete tie-break request. The `sizeof(KeyT) == @@ -221,7 +221,7 @@ _CCCL_HOST_API cudaError_t batched_topk_indexed( // produced by a counting iterator that restarts at 0 for every segment, so indices are not (pre-)materialized in global // memory template -void decode_style_variable_topk_indexed( +_CCCL_HOST_API void decode_style_variable_topk_indexed( nvbench::state& state, nvbench::type_list, nvbench::enum_type>) { if constexpr (K > MaxSegmentSize) diff --git a/cub/benchmarks/bench/segmented_topk/variable/keys_common.cuh b/cub/benchmarks/bench/segmented_topk/variable/keys_common.cuh index b2cd3fdf5b06..87c5739d72e0 100644 --- a/cub/benchmarks/bench/segmented_topk/variable/keys_common.cuh +++ b/cub/benchmarks/bench/segmented_topk/variable/keys_common.cuh @@ -125,7 +125,7 @@ static_assert(selected_backend != topk_backend::device template struct topk_backend_selector { - [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability cc) const + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(cuda::compute_capability cc) const -> cub::detail::batched_topk::topk_policy { // The baseline backend cannot honor a deterministic result set / concrete tie-break request. The `sizeof(KeyT) == @@ -261,7 +261,7 @@ _CCCL_HOST_API cudaError_t batched_topk_keys( } template -void decode_style_variable_topk_keys( +_CCCL_HOST_API void decode_style_variable_topk_keys( nvbench::state& state, nvbench::type_list, nvbench::enum_type>) { if constexpr (K > MaxSegmentSize) diff --git a/cub/benchmarks/bench/thread_sort/selection/common.cuh b/cub/benchmarks/bench/thread_sort/selection/common.cuh index ff82850c4d0e..66c812480531 100644 --- a/cub/benchmarks/bench/thread_sort/selection/common.cuh +++ b/cub/benchmarks/bench/thread_sort/selection/common.cuh @@ -27,7 +27,7 @@ template struct benchmark_op_t { template - __device__ __forceinline__ void operator()(KeyT (&keys)[ItemsPerThread]) const + _CCCL_DEVICE_API __forceinline__ void operator()(KeyT (&keys)[ItemsPerThread]) const { cub::NullType values[ItemsPerThread]; cub::UnstablePairwiseSort(keys, values, cuda::std::less<>{}); @@ -35,7 +35,7 @@ struct benchmark_op_t }; template -__global__ void selection_benchmark_kernel(int num_iterations, ActionT action) +_CCCL_KERNEL_ATTRIBUTES void selection_benchmark_kernel(int num_iterations, ActionT action) { constexpr int warp_threads = 32; __shared__ volatile KeyT output[BlockSize]; @@ -71,7 +71,7 @@ __global__ void selection_benchmark_kernel(int num_iterations, ActionT action) } template -void run_benchmark(nvbench::state& state) +_CCCL_HOST_API void run_benchmark(nvbench::state& state) { constexpr int block_size = 256; constexpr int num_iterations = 128; diff --git a/cub/benchmarks/bench/thread_sort/sort/common.cuh b/cub/benchmarks/bench/thread_sort/sort/common.cuh index 9b35ffb8227f..412d2bfe1a82 100644 --- a/cub/benchmarks/bench/thread_sort/sort/common.cuh +++ b/cub/benchmarks/bench/thread_sort/sort/common.cuh @@ -41,7 +41,7 @@ template struct benchmark_op_t { template - __device__ __forceinline__ void operator()(KeyT (&keys)[ItemsPerThread]) const + _CCCL_DEVICE_API __forceinline__ void operator()(KeyT (&keys)[ItemsPerThread]) const { cub::NullType values[ItemsPerThread]; sort(keys, values); @@ -49,7 +49,7 @@ struct benchmark_op_t } template - __device__ __forceinline__ void operator()(KeyT (&keys)[ItemsPerThread], ValueT (&values)[ItemsPerThread]) const + _CCCL_DEVICE_API __forceinline__ void operator()(KeyT (&keys)[ItemsPerThread], ValueT (&values)[ItemsPerThread]) const { sort(keys, values); avoid_code_elimitation(keys); @@ -58,7 +58,7 @@ struct benchmark_op_t private: template - __device__ __forceinline__ static void avoid_code_elimitation(T (&values)[ItemsPerThread]) + _CCCL_DEVICE_API __forceinline__ static void avoid_code_elimitation(T (&values)[ItemsPerThread]) { _CCCL_PRAGMA_UNROLL_FULL() for (int i = 0; i < ItemsPerThread; ++i) @@ -68,7 +68,7 @@ private: } template - __device__ __forceinline__ static void sort(KeyT (&keys)[ItemsPerThread], ValueT (&values)[ItemsPerThread]) + _CCCL_DEVICE_API __forceinline__ static void sort(KeyT (&keys)[ItemsPerThread], ValueT (&values)[ItemsPerThread]) { if constexpr (Algorithm == sort_algorithm::stable) { @@ -82,7 +82,7 @@ private: }; template -void run_benchmark(nvbench::state& state) +_CCCL_HOST_API void run_benchmark(nvbench::state& state) { constexpr int block_size = 256; constexpr int num_iterations = 128; @@ -114,16 +114,17 @@ using item_counts = nvbench::enum_type_list<3, 4, 7, 8, 15, 16 /*, 31, 32, 63, 6 // key-only sorting benchmarks template -void keys(nvbench::state& state, - nvbench::type_list, KeyT, nvbench::enum_type>) +_CCCL_HOST_API void keys(nvbench::state& state, + nvbench::type_list, KeyT, nvbench::enum_type>) { run_benchmark(state); } // key-value sorting benchmarks template -void pairs(nvbench::state& state, - nvbench::type_list, KeyT, ValueT, nvbench::enum_type>) +_CCCL_HOST_API void +pairs(nvbench::state& state, + nvbench::type_list, KeyT, ValueT, nvbench::enum_type>) { run_benchmark(state); } diff --git a/cub/benchmarks/bench/transform/applications/P1/bfloat16.h b/cub/benchmarks/bench/transform/applications/P1/bfloat16.h index 61ca10c7b5b6..c611d0db6d89 100644 --- a/cub/benchmarks/bench/transform/applications/P1/bfloat16.h +++ b/cub/benchmarks/bench/transform/applications/P1/bfloat16.h @@ -18,7 +18,7 @@ namespace bf16_detail { -inline __host__ __device__ float f32_from_bits(uint16_t src) +inline _CCCL_HOST_DEVICE_API float f32_from_bits(uint16_t src) { float res = 0; uint32_t tmp = src; @@ -27,7 +27,7 @@ inline __host__ __device__ float f32_from_bits(uint16_t src) return res; } -inline __host__ __device__ uint16_t round_to_nearest_even(float src) +inline _CCCL_HOST_DEVICE_API uint16_t round_to_nearest_even(float src) { if (std::isnan(src)) { @@ -45,16 +45,16 @@ struct alignas(2) BFloat16 { uint16_t x; - BFloat16() = default; + _CCCL_HIDE_FROM_ABI BFloat16() = default; struct from_bits_t {}; - static constexpr __host__ __device__ from_bits_t from_bits() + static constexpr _CCCL_HOST_DEVICE_API from_bits_t from_bits() { return from_bits_t(); } - constexpr __host__ __device__ BFloat16(unsigned short bits, from_bits_t) + constexpr _CCCL_HOST_DEVICE_API BFloat16(unsigned short bits, from_bits_t) : x(bits) {} @@ -65,7 +65,7 @@ struct alignas(2) BFloat16 explicit inline __host__ __device__ operator __nv_bfloat16() const; }; -inline __host__ __device__ BFloat16::BFloat16(float value) +inline _CCCL_HOST_DEVICE_API BFloat16::BFloat16(float value) { NV_IF_ELSE_TARGET(NV_PROVIDES_SM_80, ({ @@ -75,155 +75,155 @@ inline __host__ __device__ BFloat16::BFloat16(float value) ({ x = bf16_detail::round_to_nearest_even(value); })); } -inline __host__ __device__ BFloat16::operator float() const +inline _CCCL_HOST_DEVICE_API BFloat16::operator float() const { return __bfloat162float(*reinterpret_cast(&x)); } -inline __host__ __device__ BFloat16::BFloat16(const __nv_bfloat16& value) +inline _CCCL_HOST_DEVICE_API BFloat16::BFloat16(const __nv_bfloat16& value) : x(*reinterpret_cast(&value)) {} -inline __host__ __device__ BFloat16::operator __nv_bfloat16() const +inline _CCCL_HOST_DEVICE_API BFloat16::operator __nv_bfloat16() const { return *reinterpret_cast(&x); } // Arithmetic — BFloat16 x BFloat16 → BFloat16 -inline __host__ __device__ BFloat16 operator+(const BFloat16& a, const BFloat16& b) +inline _CCCL_HOST_DEVICE_API BFloat16 operator+(const BFloat16& a, const BFloat16& b) { return static_cast(a) + static_cast(b); } -inline __host__ __device__ BFloat16 operator-(const BFloat16& a, const BFloat16& b) +inline _CCCL_HOST_DEVICE_API BFloat16 operator-(const BFloat16& a, const BFloat16& b) { return static_cast(a) - static_cast(b); } -inline __host__ __device__ BFloat16 operator*(const BFloat16& a, const BFloat16& b) +inline _CCCL_HOST_DEVICE_API BFloat16 operator*(const BFloat16& a, const BFloat16& b) { return static_cast(a) * static_cast(b); } -inline __host__ __device__ BFloat16 operator/(const BFloat16& a, const BFloat16& b) +inline _CCCL_HOST_DEVICE_API BFloat16 operator/(const BFloat16& a, const BFloat16& b) { return static_cast(a) / static_cast(b); } -inline __host__ __device__ BFloat16 operator-(const BFloat16& a) +inline _CCCL_HOST_DEVICE_API BFloat16 operator-(const BFloat16& a) { return -static_cast(a); } // Compound assignment — BFloat16 -inline __host__ __device__ BFloat16& operator+=(BFloat16& a, const BFloat16& b) +inline _CCCL_HOST_DEVICE_API BFloat16& operator+=(BFloat16& a, const BFloat16& b) { a = a + b; return a; } -inline __host__ __device__ BFloat16& operator-=(BFloat16& a, const BFloat16& b) +inline _CCCL_HOST_DEVICE_API BFloat16& operator-=(BFloat16& a, const BFloat16& b) { a = a - b; return a; } -inline __host__ __device__ BFloat16& operator*=(BFloat16& a, const BFloat16& b) +inline _CCCL_HOST_DEVICE_API BFloat16& operator*=(BFloat16& a, const BFloat16& b) { a = a * b; return a; } -inline __host__ __device__ BFloat16& operator/=(BFloat16& a, const BFloat16& b) +inline _CCCL_HOST_DEVICE_API BFloat16& operator/=(BFloat16& a, const BFloat16& b) { a = a / b; return a; } // Arithmetic — BFloat16 x float → float -inline __host__ __device__ float operator+(BFloat16 a, float b) +inline _CCCL_HOST_DEVICE_API float operator+(BFloat16 a, float b) { return static_cast(a) + b; } -inline __host__ __device__ float operator-(BFloat16 a, float b) +inline _CCCL_HOST_DEVICE_API float operator-(BFloat16 a, float b) { return static_cast(a) - b; } -inline __host__ __device__ float operator*(BFloat16 a, float b) +inline _CCCL_HOST_DEVICE_API float operator*(BFloat16 a, float b) { return static_cast(a) * b; } -inline __host__ __device__ float operator/(BFloat16 a, float b) +inline _CCCL_HOST_DEVICE_API float operator/(BFloat16 a, float b) { return static_cast(a) / b; } -inline __host__ __device__ float operator+(float a, BFloat16 b) +inline _CCCL_HOST_DEVICE_API float operator+(float a, BFloat16 b) { return a + static_cast(b); } -inline __host__ __device__ float operator-(float a, BFloat16 b) +inline _CCCL_HOST_DEVICE_API float operator-(float a, BFloat16 b) { return a - static_cast(b); } -inline __host__ __device__ float operator*(float a, BFloat16 b) +inline _CCCL_HOST_DEVICE_API float operator*(float a, BFloat16 b) { return a * static_cast(b); } -inline __host__ __device__ float operator/(float a, BFloat16 b) +inline _CCCL_HOST_DEVICE_API float operator/(float a, BFloat16 b) { return a / static_cast(b); } // Compound assignment — float x BFloat16 → float -inline __host__ __device__ float& operator+=(float& a, const BFloat16& b) +inline _CCCL_HOST_DEVICE_API float& operator+=(float& a, const BFloat16& b) { return a += static_cast(b); } -inline __host__ __device__ float& operator-=(float& a, const BFloat16& b) +inline _CCCL_HOST_DEVICE_API float& operator-=(float& a, const BFloat16& b) { return a -= static_cast(b); } -inline __host__ __device__ float& operator*=(float& a, const BFloat16& b) +inline _CCCL_HOST_DEVICE_API float& operator*=(float& a, const BFloat16& b) { return a *= static_cast(b); } -inline __host__ __device__ float& operator/=(float& a, const BFloat16& b) +inline _CCCL_HOST_DEVICE_API float& operator/=(float& a, const BFloat16& b) { return a /= static_cast(b); } // Arithmetic — BFloat16 x int → BFloat16 -inline __host__ __device__ BFloat16 operator+(BFloat16 a, int b) +inline _CCCL_HOST_DEVICE_API BFloat16 operator+(BFloat16 a, int b) { return a + static_cast(static_cast(b)); } -inline __host__ __device__ BFloat16 operator-(BFloat16 a, int b) +inline _CCCL_HOST_DEVICE_API BFloat16 operator-(BFloat16 a, int b) { return a - static_cast(static_cast(b)); } -inline __host__ __device__ BFloat16 operator*(BFloat16 a, int b) +inline _CCCL_HOST_DEVICE_API BFloat16 operator*(BFloat16 a, int b) { return a * static_cast(static_cast(b)); } -inline __host__ __device__ BFloat16 operator/(BFloat16 a, int b) +inline _CCCL_HOST_DEVICE_API BFloat16 operator/(BFloat16 a, int b) { return a / static_cast(static_cast(b)); } -inline __host__ __device__ BFloat16 operator+(int a, BFloat16 b) +inline _CCCL_HOST_DEVICE_API BFloat16 operator+(int a, BFloat16 b) { return static_cast(static_cast(a)) + b; } -inline __host__ __device__ BFloat16 operator-(int a, BFloat16 b) +inline _CCCL_HOST_DEVICE_API BFloat16 operator-(int a, BFloat16 b) { return static_cast(static_cast(a)) - b; } -inline __host__ __device__ BFloat16 operator*(int a, BFloat16 b) +inline _CCCL_HOST_DEVICE_API BFloat16 operator*(int a, BFloat16 b) { return static_cast(static_cast(a)) * b; } -inline __host__ __device__ BFloat16 operator/(int a, BFloat16 b) +inline _CCCL_HOST_DEVICE_API BFloat16 operator/(int a, BFloat16 b) { return static_cast(static_cast(a)) / b; } // Comparison — for std::min/std::max -inline __host__ __device__ bool operator>(BFloat16& lhs, BFloat16& rhs) +inline _CCCL_HOST_DEVICE_API bool operator>(BFloat16& lhs, BFloat16& rhs) { return float(lhs) > float(rhs); } -inline __host__ __device__ bool operator<(BFloat16& lhs, BFloat16& rhs) +inline _CCCL_HOST_DEVICE_API bool operator<(BFloat16& lhs, BFloat16& rhs) { return float(lhs) < float(rhs); } diff --git a/cub/benchmarks/bench/transform/babelstream.h b/cub/benchmarks/bench/transform/babelstream.h index 74da8e612914..9762f2801fed 100644 --- a/cub/benchmarks/bench/transform/babelstream.h +++ b/cub/benchmarks/bench/transform/babelstream.h @@ -37,7 +37,7 @@ inline constexpr auto startScalar = -2; // BabelStream: 0.4 static_assert(startA == (startA + startB + (startScalar * startC)), "nstream must have a consistent workload"); template -void mul(nvbench::state& state, nvbench::type_list) +_CCCL_HOST_API void mul(nvbench::state& state, nvbench::type_list) try { const auto n = state.get_int64("Elements{io}"); @@ -68,7 +68,7 @@ NVBENCH_BENCH_TYPES(mul, NVBENCH_TYPE_AXES(element_types)) .add_int64_power_of_two_axis("Elements{io}", array_size_powers); template -void add(nvbench::state& state, nvbench::type_list) +_CCCL_HOST_API void add(nvbench::state& state, nvbench::type_list) try { const auto n = state.get_int64("Elements{io}"); @@ -102,7 +102,7 @@ NVBENCH_BENCH_TYPES(add, NVBENCH_TYPE_AXES(element_types)) .add_int64_power_of_two_axis("Elements{io}", array_size_powers); template -void triad(nvbench::state& state, nvbench::type_list) +_CCCL_HOST_API void triad(nvbench::state& state, nvbench::type_list) try { const auto n = state.get_int64("Elements{io}"); @@ -137,7 +137,7 @@ NVBENCH_BENCH_TYPES(triad, NVBENCH_TYPE_AXES(element_types)) .add_int64_power_of_two_axis("Elements{io}", array_size_powers); template -void nstream(nvbench::state& state, nvbench::type_list) +_CCCL_HOST_API void nstream(nvbench::state& state, nvbench::type_list) try { const auto n = state.get_int64("Elements{io}"); diff --git a/cub/benchmarks/bench/transform/common.h b/cub/benchmarks/bench/transform/common.h index 0a06137e0679..e56d68d92143 100644 --- a/cub/benchmarks/bench/transform/common.h +++ b/cub/benchmarks/bench/transform/common.h @@ -75,11 +75,12 @@ struct policy_selector #endif // !TUNE_BASE template -void bench_transform(nvbench::state& state, - cuda::std::tuple inputs, - RandomAccessIteratorOut output, - ::cuda::std::int64_t num_items, - TransformOp transform_op) +_CCCL_HOST_API void bench_transform( + nvbench::state& state, + cuda::std::tuple inputs, + RandomAccessIteratorOut output, + ::cuda::std::int64_t num_items, + TransformOp transform_op) { state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](const nvbench::launch& launch) { cub::DeviceTransform::Transform( diff --git a/cub/cub/agent/agent_adjacent_difference.cuh b/cub/cub/agent/agent_adjacent_difference.cuh index 3b9232ef77eb..029cb4a84e5c 100644 --- a/cub/cub/agent/agent_adjacent_difference.cuh +++ b/cub/cub/agent/agent_adjacent_difference.cuh @@ -102,7 +102,7 @@ struct AgentDifference DifferenceOpT difference_op; OffsetT num_items; - _CCCL_DEVICE _CCCL_FORCEINLINE AgentDifference( + _CCCL_DEVICE_API _CCCL_FORCEINLINE AgentDifference( TempStorage& temp_storage, InputIteratorT input_it, InputT* first_tile_previous, @@ -119,7 +119,7 @@ struct AgentDifference {} template - _CCCL_DEVICE _CCCL_FORCEINLINE void consume_tile_impl(int num_remaining, int tile_idx, OffsetT tile_base) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void consume_tile_impl(int num_remaining, int tile_idx, OffsetT tile_base) { InputT input[ITEMS_PER_THREAD]; OutputT output[ITEMS_PER_THREAD]; @@ -198,7 +198,7 @@ struct AgentDifference } template - _CCCL_DEVICE _CCCL_FORCEINLINE void consume_tile(int num_remaining, int tile_idx, OffsetT tile_base) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void consume_tile(int num_remaining, int tile_idx, OffsetT tile_base) { if (tile_idx == 0) { @@ -210,7 +210,7 @@ struct AgentDifference } } - _CCCL_DEVICE _CCCL_FORCEINLINE void Process(int tile_idx, OffsetT tile_base) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Process(int tile_idx, OffsetT tile_base) { OffsetT num_remaining = num_items - tile_base; @@ -230,7 +230,7 @@ struct AgentDifferenceInit { static constexpr int BLOCK_THREADS = 128; - static _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API static _CCCL_FORCEINLINE void Process(int tile_idx, InputIteratorT first, InputT* result, OffsetT num_tiles, int items_per_tile) { OffsetT tile_base = static_cast(tile_idx) * items_per_tile; diff --git a/cub/cub/agent/agent_batch_memcpy.cuh b/cub/cub/agent/agent_batch_memcpy.cuh index 88c76d513fe1..509e1f362a3c 100644 --- a/cub/cub/agent/agent_batch_memcpy.cuh +++ b/cub/cub/agent/agent_batch_memcpy.cuh @@ -93,7 +93,7 @@ using AgentBatchMemcpyPolicy namespace detail::batch_memcpy { template -_CCCL_FORCEINLINE _CCCL_DEVICE void +_CCCL_FORCEINLINE _CCCL_DEVICE_API void LoadVectorAndFunnelShiftR(uint32_t const* aligned_ptr, uint32_t bit_shift, uint4& data_out) { data_out = {aligned_ptr[0], aligned_ptr[1], aligned_ptr[2], aligned_ptr[3]}; @@ -109,7 +109,7 @@ LoadVectorAndFunnelShiftR(uint32_t const* aligned_ptr, uint32_t bit_shift, uint4 } template -_CCCL_FORCEINLINE _CCCL_DEVICE void +_CCCL_FORCEINLINE _CCCL_DEVICE_API void LoadVectorAndFunnelShiftR(uint32_t const* aligned_ptr, uint32_t bit_shift, uint2& data_out) { data_out = {aligned_ptr[0], aligned_ptr[1]}; @@ -123,7 +123,7 @@ LoadVectorAndFunnelShiftR(uint32_t const* aligned_ptr, uint32_t bit_shift, uint2 } template -_CCCL_FORCEINLINE _CCCL_DEVICE void +_CCCL_FORCEINLINE _CCCL_DEVICE_API void LoadVectorAndFunnelShiftR(uint32_t const* aligned_ptr, uint32_t bit_shift, uint32_t& data_out) { data_out = aligned_ptr[0]; @@ -147,7 +147,7 @@ LoadVectorAndFunnelShiftR(uint32_t const* aligned_ptr, uint32_t bit_shift, uint3 * @param data_out The vector type that stores the data loaded from \p ptr */ template -_CCCL_FORCEINLINE _CCCL_DEVICE void LoadVector(const char* ptr, VectorT& data_out) +_CCCL_FORCEINLINE _CCCL_DEVICE_API void LoadVector(const char* ptr, VectorT& data_out) { const uint32_t offset = reinterpret_cast(ptr) % 4U; const uint32_t* aligned_ptr = reinterpret_cast(ptr - offset); @@ -197,7 +197,7 @@ struct PointerRange * @return The byte range that can safely be copied using vectorized stores of type VectorT */ template -_CCCL_DEVICE _CCCL_FORCEINLINE PointerRange +_CCCL_DEVICE_API _CCCL_FORCEINLINE PointerRange GetAlignedPtrs(const void* in_begin, void* out_begin, ByteOffsetT num_bytes) { // Data type size used for vectorized stores @@ -271,7 +271,7 @@ GetAlignedPtrs(const void* in_begin, void* out_begin, ByteOffsetT num_bytes) * @param src Pointer to the memory location to copy from */ template -_CCCL_DEVICE _CCCL_FORCEINLINE void +_CCCL_DEVICE_API _CCCL_FORCEINLINE void vectorized_copy(int32_t thread_rank, void* dest, ByteOffsetT num_bytes, const void* src) { char* out_ptr = static_cast(dest); @@ -327,7 +327,7 @@ vectorized_copy(int32_t thread_rank, void* dest, ByteOffsetT num_bytes, const vo } template -_CCCL_DEVICE _CCCL_FORCEINLINE void +_CCCL_DEVICE_API _CCCL_FORCEINLINE void copy_items(InputBufferT input_buffer, OutputBufferT output_buffer, OffsetT num_items, OffsetT offset = 0) { if constexpr (IsMemcpy) @@ -350,7 +350,7 @@ copy_items(InputBufferT input_buffer, OutputBufferT output_buffer, OffsetT num_i } template -_CCCL_DEVICE _CCCL_FORCEINLINE AliasT read_item(InputIt buffer_src, OffsetT offset) +_CCCL_DEVICE_API _CCCL_FORCEINLINE AliasT read_item(InputIt buffer_src, OffsetT offset) { if constexpr (IsMemcpy) { @@ -363,7 +363,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE AliasT read_item(InputIt buffer_src, OffsetT offs } template -_CCCL_DEVICE _CCCL_FORCEINLINE void write_item(OutputIt buffer_dst, OffsetT offset, AliasT value) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void write_item(OutputIt buffer_dst, OffsetT offset, AliasT value) { if constexpr (IsMemcpy) { @@ -434,7 +434,7 @@ class bit_packed_counter //------------------------------------------------------------------------------ public: - _CCCL_DEVICE _CCCL_FORCEINLINE uint32_t get(uint32_t index) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE uint32_t get(uint32_t index) const { const uint32_t target_offset = index * BITS_PER_ITEM; uint32_t val = 0; @@ -453,7 +453,7 @@ public: return val; } - _CCCL_DEVICE _CCCL_FORCEINLINE void add(uint32_t index, uint32_t value) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void add(uint32_t index, uint32_t value) { const uint32_t target_offset = index * BITS_PER_ITEM; @@ -470,7 +470,7 @@ public: } } - _CCCL_DEVICE bit_packed_counter operator+(const bit_packed_counter& rhs) const + _CCCL_DEVICE_API bit_packed_counter operator+(const bit_packed_counter& rhs) const { bit_packed_counter result; @@ -708,7 +708,7 @@ private: /** * @brief Loads this tile's buffers' sizes, without any guards (i.e., out-of-bounds checks) */ - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void LoadBufferSizesFullTile(BufferSizeIteratorT tile_buffer_sizes_it, BufferSizeT (&buffer_sizes)[BUFFERS_PER_THREAD]) { BufferLoadT(temp_storage.load_storage).Load(tile_buffer_sizes_it, buffer_sizes); @@ -717,7 +717,7 @@ private: /** * @brief Loads this tile's buffers' sizes, making sure to read at most \p num_valid items. */ - _CCCL_DEVICE _CCCL_FORCEINLINE void LoadBufferSizesPartialTile( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void LoadBufferSizesPartialTile( BufferSizeIteratorT tile_buffer_sizes_it, BufferSizeT (&buffer_sizes)[BUFFERS_PER_THREAD], BufferOffsetT num_valid) { // Out-of-bounds buffer items are initialized to '0', so those buffers will simply be ignored @@ -731,7 +731,7 @@ private: * @brief Computes the histogram over the number of buffers belonging to each of the three * size-classes (TLEV, WLEV, BLEV). */ - _CCCL_DEVICE _CCCL_FORCEINLINE VectorizedSizeClassCounterT + _CCCL_DEVICE_API _CCCL_FORCEINLINE VectorizedSizeClassCounterT GetBufferSizeClassHistogram(const BufferSizeT (&buffer_sizes)[BUFFERS_PER_THREAD]) { VectorizedSizeClassCounterT vectorized_counters{}; @@ -755,7 +755,7 @@ private: /** * @brief Scatters the buffers into the respective buffer's size-class partition. */ - _CCCL_DEVICE _CCCL_FORCEINLINE void PartitionBuffersBySize( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void PartitionBuffersBySize( const BufferSizeT (&buffer_sizes)[BUFFERS_PER_THREAD], VectorizedSizeClassCounterT& vectorized_offsets, BufferTuple (&buffers_by_size_class)[BUFFERS_PER_BLOCK]) @@ -786,7 +786,7 @@ private: * @brief Read in all the buffers that require block-level collaboration and put them to a queue * that will get picked up in a separate, subsequent kernel. */ - _CCCL_DEVICE _CCCL_FORCEINLINE void EnqueueBLEVBuffers( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void EnqueueBLEVBuffers( BufferTuple* buffers_by_size_class, InputBufferIt tile_buffer_srcs, OutputBufferIt tile_buffer_dsts, @@ -856,7 +856,7 @@ private: * @brief Read in all the buffers of this tile that require warp-level collaboration and copy * their bytes to the corresponding destination buffer */ - _CCCL_DEVICE _CCCL_FORCEINLINE void BatchMemcpyWLEVBuffers( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void BatchMemcpyWLEVBuffers( BufferTuple* buffers_by_size_class, InputBufferIt tile_buffer_srcs, OutputBufferIt tile_buffer_dsts, @@ -878,7 +878,7 @@ private: * @brief Read in all the buffers of this tile that require thread-level collaboration and copy * their bytes to the corresponding destination buffer */ - _CCCL_DEVICE _CCCL_FORCEINLINE void BatchMemcpyTLEVBuffers( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void BatchMemcpyTLEVBuffers( BufferTuple* buffers_by_size_class, InputBufferIt tile_buffer_srcs, OutputBufferIt tile_buffer_dsts, @@ -1007,7 +1007,7 @@ private: //----------------------------------------------------------------------------- public: - _CCCL_DEVICE _CCCL_FORCEINLINE void ConsumeTile(BufferOffsetT tile_id) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ConsumeTile(BufferOffsetT tile_id) { // Offset into this tile's buffers BufferOffsetT buffer_offset = tile_id * BUFFERS_PER_BLOCK; @@ -1125,7 +1125,7 @@ public: //----------------------------------------------------------------------------- // CONSTRUCTOR //----------------------------------------------------------------------------- - _CCCL_DEVICE _CCCL_FORCEINLINE AgentBatchMemcpy( + _CCCL_DEVICE_API _CCCL_FORCEINLINE AgentBatchMemcpy( TempStorage& temp_storage, InputBufferIt input_buffer_it, OutputBufferIt output_buffer_it, diff --git a/cub/cub/agent/agent_batched_topk_cluster.cuh b/cub/cub/agent/agent_batched_topk_cluster.cuh index 9878eb68b63d..cc5a495802db 100644 --- a/cub/cub/agent/agent_batched_topk_cluster.cuh +++ b/cub/cub/agent/agent_batched_topk_cluster.cuh @@ -110,7 +110,7 @@ struct smem_block_tile_layout static constexpr int base_padding_bytes = slot_alignment; static_assert(ChunkBytes % slot_alignment == 0, "ChunkBytes must be a multiple of the load and key alignment"); - [[nodiscard]] _CCCL_HOST_DEVICE static constexpr ::cuda::std::uint32_t + [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr ::cuda::std::uint32_t max_block_resident_items(int num_dynamic_smem_bytes) noexcept { const int num_usable_bytes = (::cuda::std::max) (0, num_dynamic_smem_bytes - base_padding_bytes); @@ -120,14 +120,14 @@ struct smem_block_tile_layout // Number of chunk-sized slots spanned by `num_items` keys (the segment's chunk count). Encapsulates the // `max_chunk_items` granularity so callers size segments/clusters without inlining chunk arithmetic. - [[nodiscard]] _CCCL_HOST_DEVICE static constexpr ::cuda::std::uint64_t + [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr ::cuda::std::uint64_t num_chunks_from_num_items(::cuda::std::uint64_t num_items) noexcept { return ::cuda::ceil_div(num_items, max_chunk_items); } // Dynamic-SMEM byte count for `chunks` chunk-sized slots plus the base-alignment padding. - [[nodiscard]] _CCCL_HOST_DEVICE static constexpr int + [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr int min_smem_bytes_from_num_chunks(::cuda::std::uint64_t num_chunks) noexcept { _CCCL_ASSERT(base_padding_bytes + (num_chunks * static_cast<::cuda::std::uint64_t>(chunk_bytes)) @@ -138,7 +138,7 @@ struct smem_block_tile_layout // Smallest dynamic-SMEM byte count whose per-CTA capacity (`max_block_resident_items`) reaches `items` keys // (rounding up to whole chunks). Inverse of `max_block_resident_items`. - [[nodiscard]] _CCCL_HOST_DEVICE static constexpr int + [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr int min_smem_bytes_from_num_items(::cuda::std::uint64_t num_items) noexcept { return min_smem_bytes_from_num_chunks(num_chunks_from_num_items(num_items)); @@ -146,7 +146,7 @@ struct smem_block_tile_layout }; // Whether a segment takes the cluster-barrier-free single-CTA path. -[[nodiscard]] _CCCL_HOST_DEVICE constexpr bool is_single_cta_eligible( +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool is_single_cta_eligible( ::cuda::std::uint32_t segment_size, ::cuda::std::uint32_t max_block_resident_items, int single_block_max_seg_size) noexcept @@ -159,7 +159,7 @@ struct smem_block_tile_layout // `min_chunks_per_block` chunks, clamped to `[1, max_cluster_blocks]`. Identical arithmetic on both sides; only // the cap differs (the live cluster size on the device, max launchable blocks on the host). `min_chunks_per_block` is // `static_assert`ed positive, so the divide is well-defined. 32-bit: a segment holds at most 2^21 chunks. -[[nodiscard]] _CCCL_HOST_DEVICE constexpr ::cuda::std::uint32_t compute_num_logical_cluster_blocks( +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::uint32_t compute_num_logical_cluster_blocks( ::cuda::std::uint32_t num_chunks, int min_chunks_per_block, ::cuda::std::uint32_t max_cluster_blocks) noexcept { const auto num_blocks = num_chunks / static_cast<::cuda::std::uint32_t>(min_chunks_per_block); @@ -303,7 +303,7 @@ struct agent_batched_topk_cluster // Clamp a per-thread unroll down to the segment's bounded round count (only when `should_clamp_items_to_segment`); // larger/unbounded segments keep the full tuning width, and the guard keeps the rounds arithmetic in `int`. - [[nodiscard]] static _CCCL_CONSTEVAL int clamp_unroll(int rounds, int tuning) + [[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL int clamp_unroll(int rounds, int tuning) { return should_clamp_items_to_segment ? ::cuda::std::clamp(rounds, 1, tuning) : tuning; } @@ -433,7 +433,7 @@ struct agent_batched_topk_cluster // (its trailing `< num_load_align_items` items). The block-load path reports the aligned bulk `round_down(raw, // num_load_align_items)` in `num_items` (that suffix is peeled into `edge_keys`); the generic path reports the full // raw count. - [[nodiscard]] static _CCCL_DEVICE _CCCL_FORCEINLINE chunk_desc + [[nodiscard]] static _CCCL_DEVICE_API _CCCL_FORCEINLINE chunk_desc get_chunk(offset_t chunk_idx, offset_t segment_size, offset_t num_cluster_head_items) { const offset_t offset = num_cluster_head_items + (chunk_idx * offset_t{max_chunk_items}); @@ -462,7 +462,7 @@ struct agent_batched_topk_cluster offset_t stride; // distance between consecutive owned chunks (`num_logical_cluster_blocks` strided, `1` blocked) offset_t num_chunks; // number of chunks owned by this rank - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE offset_t global_index(offset_t local_chunk_idx) const + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE offset_t global_index(offset_t local_chunk_idx) const { _CCCL_ASSERT(local_chunk_idx < num_chunks, "global_index: rank-local chunk index out of range"); return first + (local_chunk_idx * stride); @@ -483,7 +483,7 @@ struct agent_batched_topk_cluster // with an arbitrary end) would let a boundary chunk be loaded with a single aligned-bulk + in-place edge call and // remove these hand-rolled copies entirely. template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void stage_and_consume_edge(key_t* dst, const key_t* src, int num_keys, Apply&& apply) const { _CCCL_ASSERT(num_keys >= 0 && num_keys <= num_load_align_items, "a boundary edge must fit in its edge_keys slot"); @@ -504,7 +504,7 @@ struct agent_batched_topk_cluster // clamped (floor) items per thread, so the sub-tile remainder is handled by a single fused block-stride loop bounded // by the count. template - _CCCL_DEVICE _CCCL_FORCEINLINE void for_each_chunk_key_impl(const key_t* keys, int num_keys, Apply&& apply) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE void for_each_chunk_key_impl(const key_t* keys, int num_keys, Apply&& apply) const { constexpr int tile = Unroll * policy.threads_per_block; const int full_tiles = ::cuda::round_down(num_keys, tile); @@ -534,7 +534,7 @@ struct agent_batched_topk_cluster } template - _CCCL_DEVICE _CCCL_FORCEINLINE void for_each_chunk_key(const key_t* chunk_keys, int num_keys, KeyOp&& key_op) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE void for_each_chunk_key(const key_t* chunk_keys, int num_keys, KeyOp&& key_op) const { for_each_chunk_key_impl(chunk_keys, num_keys, [&](const key_t& key, int) { key_op(key); @@ -552,7 +552,7 @@ struct agent_batched_topk_cluster // init distinct stages in parallel via a block-stride loop. The caller's block `__syncthreads()` orders these // generic-proxy inits before the first `cp.async.bulk` issue, so no `fence.proxy.async` is needed (cf. // `BlockLoadToShared`, which front-loads init + `__syncthreads()` likewise). - _CCCL_DEVICE _CCCL_FORCEINLINE void init_load_barriers() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void init_load_barriers() { _CCCL_PRAGMA_NOUNROLL() for (int stage = tid; stage < policy.pipeline_stages; stage += policy.threads_per_block) @@ -565,7 +565,7 @@ struct agent_batched_topk_cluster // which also arrives with the transaction byte count (an empty copy arrives with zero so the phase still completes). // The stage mbarrier must already be initialized (see `init_load_barriers`). Each call must be paired, in issue order // per stage, with a matching `wait_stage(stage)`. - _CCCL_DEVICE _CCCL_FORCEINLINE void issue_bulk_copy(int stage, char* dst, ::cuda::std::span src) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void issue_bulk_copy(int stage, char* dst, ::cuda::std::span src) { _CCCL_ASSERT(stage >= 0 && stage < policy.pipeline_stages, "pipeline stage index out of range"); // Only the block leader (see `is_block_leader`) drives the mbarrier, for a uniform branch and better mbarrier @@ -612,7 +612,7 @@ struct agent_batched_topk_cluster } // Wait for stage `stage`'s copy to land (all threads spin on its current parity), then flip that stage's parity bit. - _CCCL_DEVICE _CCCL_FORCEINLINE void wait_stage(int stage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void wait_stage(int stage) { _CCCL_ASSERT(stage >= 0 && stage < policy.pipeline_stages, "pipeline stage index out of range"); const ::cuda::std::uint32_t parity = (load_phase >> stage) & 1u; @@ -746,7 +746,7 @@ struct agent_batched_topk_cluster } private: - _CCCL_DEVICE _CCCL_FORCEINLINE void reset_hist() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void reset_hist() { // The `num_buckets / policy.threads_per_block` full strided rounds are in range for every thread. The `< // policy.threads_per_block` leftover is compiled out when `num_buckets` is a multiple of @@ -774,7 +774,7 @@ private: // (endianness-independent). Every read follows a cluster/block barrier that orders the leader's publish, so the weak // load is enough; `volatile` keeps it from being hoisted or CSE'd across passes (`result` changes each pass). template - _CCCL_DEVICE _CCCL_FORCEINLINE FieldT leader_state_load(::cuda::std::size_t byte_offset) + _CCCL_DEVICE_API _CCCL_FORCEINLINE FieldT leader_state_load(::cuda::std::size_t byte_offset) { const ::cuda::std::uint32_t addr = layout.leader_state32 + static_cast<::cuda::std::uint32_t>(byte_offset); if constexpr (sizeof(FieldT) == sizeof(::cuda::std::uint64_t)) @@ -799,7 +799,7 @@ private: // Two independent 32-bit reductions rather than one 64-bit one: `red.add.u64` on shared memory is emulated with a // CAS spin loop, whereas `red.add.u32` is a native shared-memory atomic. Drives the cross-CTA selected/tied // prefix scan (see `prime_placement_counters`). - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void add_remote_prefix(unsigned target_rank, offset_t num_local_selected_for_scan, offset_t num_local_tied_for_scan) { const ::cuda::std::uint32_t own_selected = @@ -830,7 +830,7 @@ private: // single (thread, slot) pair that owns the k-th bucket writes the per-pass // state. The caller must guarantee the leader block has finished its DSMEM // merge before invoking this. - _CCCL_DEVICE _CCCL_FORCEINLINE void leader_identify_kth_bucket() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void leader_identify_kth_bucket() { // Capture `state.size.leftover_k` before the scan: the owning thread overwrites it in the loop below, so any later // read would race with that write. @@ -893,7 +893,7 @@ private: // Adopt the streaming-slot window (`num_local_stream_slots`, sized by `compute_segment_layout`) and reset the // ping-pong/priming cursor for this segment. `run` may then flip `stream_is_forward` for the deterministic filter's // entry parity. - _CCCL_DEVICE _CCCL_FORCEINLINE void init_overflow_stream() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void init_overflow_stream() { stream_slot_base = static_cast(layout.num_local_resident_slots); // Use the whole reserved streaming region as pipeline stages for maximal ping-pong reuse; the `1` floor covers the @@ -923,7 +923,7 @@ private: // span from `block_keys_in_bulk_ptr`; empty when it has none. `get_chunk` already reports the aligned bulk in // `num_items` (the global-last chunk's unaligned suffix is peeled into `edge_keys`). Shared source computation for // both the resident and overflow-stream bulk-copy issues. - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE ::cuda::std::span + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE ::cuda::std::span chunk_bulk_src(offset_t window_base, offset_t window_chunk_idx) const { const auto chunk = get_chunk(layout.local_partition.global_index(window_base + window_chunk_idx), @@ -941,7 +941,7 @@ private: // Shared-memory view of the chunk currently resident in `stage`'s slot: the slot index is a pure function of `stage` // and the length is recomputed from `overflow_idx` (no spillable per-stage array). Returns the aligned bulk only (the // always-peeled tail suffix is excluded). - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE smem_keys_t stage_span(int stage, offset_t overflow_idx) const + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE smem_keys_t stage_span(int stage, offset_t overflow_idx) const { const auto chunk = get_chunk(layout.local_partition.global_index(layout.overflow_base + overflow_idx), layout.segment_size, @@ -958,7 +958,7 @@ private: // top-k fully placed -- polled before the prefetch so we never launch a copy we would only drain again; the // prefetches already in flight are drained after `run_overflow_pass`'s consume loops. template - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE bool + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE bool consume_overflow_visit(offset_t overflow_visit_idx, int& stage, BlockApply&& block_apply, Continue&& should_continue) { const offset_t overflow_idx = @@ -1027,7 +1027,7 @@ private: // break can leave prefetches in flight, so the pass drains the remaining stages before returning (a full pass ends // with an empty `stream_inflight_mask`, so the drain is a no-op). template - _CCCL_DEVICE _CCCL_FORCEINLINE void run_overflow_pass( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void run_overflow_pass( BlockApply&& block_apply, GenericApply&& generic_apply, OverlapWork&& overlap_work, Continue&& should_continue) { // Every pass begins drained: `load_and_histogram_first_pass` fully drains the first wave before returning, and each @@ -1129,7 +1129,7 @@ private: // unrolls the generic (gmem fallback) consume loop; callers pass their clamped items-per-thread. See // `run_overflow_pass` for the overlap semantics of `overlap_work`. template - _CCCL_DEVICE _CCCL_FORCEINLINE void consume_overflow_keys(KeyConsumer&& consume_key, OverlapWork&& overlap_work) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void consume_overflow_keys(KeyConsumer&& consume_key, OverlapWork&& overlap_work) { // The histogram poll is barrier-free, so `consume_overflow_visit` keeps its own slot-reuse barrier. run_overflow_pass( @@ -1163,17 +1163,17 @@ private: // CTA's prior writes and signals arrival; `wait` acquires all CTAs' writes once every CTA has arrived. Both are // `.aligned` since every thread reaches them under a uniform branch. Issuing independent, block-local work between // them hides the cluster-arrival latency. - _CCCL_DEVICE _CCCL_FORCEINLINE static void cluster_arrive() + _CCCL_DEVICE_API _CCCL_FORCEINLINE static void cluster_arrive() { asm volatile("barrier.cluster.arrive.release.aligned;" : : : "memory"); } - _CCCL_DEVICE _CCCL_FORCEINLINE static void cluster_wait() + _CCCL_DEVICE_API _CCCL_FORCEINLINE static void cluster_wait() { asm volatile("barrier.cluster.wait.acquire.aligned;" : : : "memory"); } // Cluster-wide barrier: arrive immediately followed by wait, no work window exploited. - _CCCL_DEVICE _CCCL_FORCEINLINE static void cluster_barrier() + _CCCL_DEVICE_API _CCCL_FORCEINLINE static void cluster_barrier() { cluster_arrive(); cluster_wait(); @@ -1183,7 +1183,7 @@ private: // it and the cluster-scoped barrier is unnecessary. `is_single_cta` is the agent's logical-cluster member (see // `init_logical_cluster`): 1 when a small segment collapsed onto rank 0, not the raw cluster size. It is // per-segment uniform across the surviving block(s), so the branch is reached uniformly. - _CCCL_DEVICE _CCCL_FORCEINLINE static void cluster_or_block_sync(bool is_single_cta) + _CCCL_DEVICE_API _CCCL_FORCEINLINE static void cluster_or_block_sync(bool is_single_cta) { if (is_single_cta) { @@ -1199,7 +1199,7 @@ private: // work: `arrive`, then the work, then `wait`. A single-CTA "cluster" has no arrival latency to hide and no cross-CTA // state, so it takes the whole `__syncthreads()` at `arrive` and its `wait` is a no-op. In the multi-CTA case the // work between the two must touch only block-local state (a cross-CTA read before `wait` may miss a peer's writes). - _CCCL_DEVICE _CCCL_FORCEINLINE static void cluster_or_block_arrive(bool is_single_cta) + _CCCL_DEVICE_API _CCCL_FORCEINLINE static void cluster_or_block_arrive(bool is_single_cta) { if (is_single_cta) { @@ -1210,7 +1210,7 @@ private: cluster_arrive(); } } - _CCCL_DEVICE _CCCL_FORCEINLINE static void cluster_or_block_wait(bool is_single_cta) + _CCCL_DEVICE_API _CCCL_FORCEINLINE static void cluster_or_block_wait(bool is_single_cta) { if (!is_single_cta) { @@ -1235,7 +1235,7 @@ private: // only `selected_prefix`); `is_single_cta` yields both zero (selected stays 0, tie is just // `num_cluster_selected`). The successor pushes are lane-parallel (each thread owns a strided slice); all threads // see CTA-uniform counts, so the guard and the barrier stay uniform. - _CCCL_DEVICE _CCCL_FORCEINLINE cross_cta_prefixes prime_placement_counters( + _CCCL_DEVICE_API _CCCL_FORCEINLINE cross_cta_prefixes prime_placement_counters( offset_t num_local_selected_for_scan, offset_t num_local_tied_for_scan, offset_t num_cluster_selected) { if (is_single_cta) @@ -1350,7 +1350,7 @@ private: // key's segment-local index `seg_idx` from gmem and store it at the same slot. Compiled out (and never dereferences // the null value iterators) in keys-only builds; `segment_id` is loop-invariant, so the per-segment iterators hoist // out of the writes. - _CCCL_DEVICE _CCCL_FORCEINLINE void final_filter_write_value(out_offset_t pos, offset_t seg_idx) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void final_filter_write_value(out_offset_t pos, offset_t seg_idx) { if constexpr (!is_keys_only) { @@ -1376,7 +1376,7 @@ private: // past the boundary (phase B). `classify_tile` and `place_tile` must use the same `Blocked` so `keys[i]`/`flags[i]` // denote the same element; `emit_indexed` is intrinsically blocked. template - _CCCL_DEVICE _CCCL_FORCEINLINE int tile_item_pos(int tile_base, int i) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE int tile_item_pos(int tile_base, int i) const { if constexpr (Blocked) { @@ -1394,7 +1394,7 @@ private: // always accepts selected keys; pairs builds also copy the key's value from `seg_idx`. (Deterministic index-ordered // tie resolution goes through `emit_indexed` instead.) template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void place_one(KeyOutIt block_keys_out, bool is_tied, const key_t& key, offset_t seg_idx) { offset_t* const counter = is_tied ? &temp_storage.tie_offset_counter : &temp_storage.selected_offset_counter; @@ -1413,7 +1413,7 @@ private: // post-sync; every lane must reach it. Polled only at critical points -- between regions and after each consumed // overflow chunk -- never per tile. template - _CCCL_DEVICE _CCCL_FORCEINLINE bool final_filter_should_stop(const det_filter_state& state) + _CCCL_DEVICE_API _CCCL_FORCEINLINE bool final_filter_should_stop(const det_filter_state& state) { // Counters run absolute (primed with the region base), so completion compares against each region's end. Every // region is visited at most once, so neither counter can pass its end. @@ -1442,7 +1442,7 @@ private: // candidates here and leaves them to `emit_indexed`; the lazy path passes `true` and its crossing tile is later // overwritten by `emit_indexed`. template - _CCCL_DEVICE _CCCL_FORCEINLINE void place_tile( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void place_tile( State& state, const key_t (&keys)[ItemsPerThread], const offset_t (&flags)[ItemsPerThread], @@ -1471,7 +1471,7 @@ private: // Returns the tile's candidate total. Used where the K-boundary falls in this tile (terminal tile, or the lazy // path's crossing tile). template - _CCCL_DEVICE _CCCL_FORCEINLINE offset_t emit_indexed( + _CCCL_DEVICE_API _CCCL_FORCEINLINE offset_t emit_indexed( State& state, const key_t (&keys)[ItemsPerThread], const offset_t (&flags)[ItemsPerThread], @@ -1516,7 +1516,7 @@ private: // without touching `identify_op` again. `Blocked` picks the thread->element arrangement (`tile_item_pos`); the source // is `smem_src` (folded in-region position) or gmem `block_keys_in`. Out-of-range lanes stay `flag_none`. template - _CCCL_DEVICE _CCCL_FORCEINLINE void classify_tile( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void classify_tile( State& state, [[maybe_unused]] const key_t* smem_src, offset_t seg_base, @@ -1568,7 +1568,7 @@ private: // blocked only while ties remain (phase A, `emit_indexed`'s scan needs it) and switches once to striped for the // remaining strictly-selected/rejected keys (phase B). template - _CCCL_DEVICE _CCCL_FORCEINLINE void process_tiles( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void process_tiles( State& state, [[maybe_unused]] const key_t* smem_src, offset_t seg_base, @@ -1689,7 +1689,7 @@ private: // deterministic mode): ascending walks the low-index window forward, descending the high-index window // (`resident_base`) in reverse. One `process_tiles` call per span handles both, folding the index at compile time. template - _CCCL_DEVICE _CCCL_FORCEINLINE void process_resident(det_filter_state& state) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void process_resident(det_filter_state& state) { if constexpr (use_block_load_to_shared) { @@ -1739,7 +1739,7 @@ private: // `stream_is_primed` carries in as `true` and this pass reuses the resident turn-around chunks with no re-prime (the // generic fallback re-reads gmem each pass regardless). template - _CCCL_DEVICE _CCCL_FORCEINLINE void process_overflow(det_filter_state& state) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void process_overflow(det_filter_state& state) { // Guards the straddling CTA (the only rank needing scan order): `run` preselected the direction so it enters at // `stream_is_forward == !is_tie_reversed` after the full `num_passes`. The escape disjuncts are the cases where @@ -1796,7 +1796,7 @@ private: // fewer than `num_load_align_items` items (it fits in its `edge_keys` slot), so it runs `process_tiles` at unroll // factor 1 (non-unrolled) instead of the wider tie-break unroll the potentially large resident/overflow regions use. template - _CCCL_DEVICE _CCCL_FORCEINLINE void process_edge( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void process_edge( [[maybe_unused]] det_filter_state& state, [[maybe_unused]] const key_t* keys, [[maybe_unused]] offset_t seg_base, @@ -1815,7 +1815,7 @@ private: // (base 0). In global-index order it is the leading region (ascending) / trailing region (descending); a non-head // rank or empty prefix is a no-op. template - _CCCL_DEVICE _CCCL_FORCEINLINE void process_head_edge(det_filter_state& state) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void process_head_edge(det_filter_state& state) { process_edge( state, @@ -1829,7 +1829,7 @@ private: // (base `segment_size - num_local_tail_edge_items`). In global-index order it is the trailing region (ascending) / // leading region (descending); an aligned or non-owned tail is a no-op. template - _CCCL_DEVICE _CCCL_FORCEINLINE void process_tail_edge(det_filter_state& state) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void process_tail_edge(det_filter_state& state) { process_edge( state, @@ -1846,7 +1846,7 @@ private: // top-k is placed; `is_residency_reversed` keeps the first-visited (high-index) chunks resident in the descending // order so this holds. template - _CCCL_DEVICE _CCCL_FORCEINLINE void run_filter(det_filter_state& state) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void run_filter(det_filter_state& state) { const auto step = [&](auto&& region) { // No explicit barrier: `final_filter_should_stop` self-syncs, isolating its read from `region`'s placement @@ -1886,7 +1886,7 @@ private: // reading the keys already staged in `edge_keys`. Used by the histogram passes; both final filters consume the edges // through `process_tiles` instead (as separate regions for the deterministic filter, via `nondet_consume_resident`). template - _CCCL_DEVICE _CCCL_FORCEINLINE void consume_boundary_edges( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void consume_boundary_edges( [[maybe_unused]] int num_head_edge_keys, [[maybe_unused]] int num_tail_edge_keys, [[maybe_unused]] Apply&& apply) { if constexpr (use_block_load_to_shared) @@ -1928,7 +1928,7 @@ private: // disjoint from the streaming slots, so this never races the overflow apply. Each chunk's `chunk.offset` is passed as // the region's segment base (used for the pair value fetch; elided in keys-only). template - _CCCL_DEVICE _CCCL_FORCEINLINE void nondet_consume_resident(nondet_filter_state& state) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void nondet_consume_resident(nondet_filter_state& state) { if constexpr (use_block_load_to_shared) { @@ -2005,7 +2005,7 @@ private: // `overlap_work`) and place into block-local SMEM atomics. `run_overflow_pass` breaks the stream once this CTA's // contribution is fully placed (all its selected keys in the front, and its tie counter reached the region end `k`). template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void write_nondeterministic_topk(IdentifyOp identify_op, KeyOutIt block_keys_out, smem_keys_t resident_keys) { // Leader's remaining tie count after the passes, pulled while every block is still coupled to the pass loop's final @@ -2099,7 +2099,7 @@ private: // selected/tie bases and lets it detect whether all/none/some of its candidates win. This member computes the // `det_filter_state` inputs and hands them to `run_filter`, which drives the per-region sweeps. template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void write_deterministic_topk(IdentifyOp identify_op, KeyOutIt block_keys_out, smem_keys_t resident_keys) { // Cache the leader's final `size` pair now, while every block is still tightly coupled to the pass loop's final @@ -2237,7 +2237,7 @@ private: // visibility of the zeroed histogram comes later, from the deferred initial cluster wait in `run_radix_passes` (this // whole load runs in that arrive->wait window). template - _CCCL_DEVICE _CCCL_FORCEINLINE void load_and_histogram_first_pass(smem_keys_t& resident_keys) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void load_and_histogram_first_pass(smem_keys_t& resident_keys) { using extract_bin_op_t = detail::topk::extract_bin_op_t; @@ -2710,7 +2710,7 @@ private: } // Fill the `layout` member for this block's `segment_id`/`segment_size`. Called once at the top of `run`. - _CCCL_DEVICE _CCCL_FORCEINLINE void compute_segment_layout() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void compute_segment_layout() { layout.block_keys_in = d_key_segments_it[segment_id]; layout.segment_size = static_cast(segment_size); @@ -2916,7 +2916,7 @@ private: // is pinned down (or early stop fires). Folds `kth_key_bits_local` up digit by digit and returns the number of passes // that actually ran (`num_executed_passes`), which the final filter uses to size its identify operator. template - _CCCL_DEVICE _CCCL_FORCEINLINE int run_radix_passes(smem_keys_t resident_keys, key_prefix_t& kth_key_bits_local) + _CCCL_DEVICE_API _CCCL_FORCEINLINE int run_radix_passes(smem_keys_t resident_keys, key_prefix_t& kth_key_bits_local) { using extract_bin_op_t = detail::topk::extract_bin_op_t; @@ -3134,7 +3134,7 @@ private: } template - _CCCL_DEVICE _CCCL_FORCEINLINE void run() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void run() { using identify_candidates_op_t = detail::topk:: identify_candidates_op_t; @@ -3225,7 +3225,7 @@ private: // values (pairs only) are staged together so both arrays' loads issue before either array's stores, keeping up to // `2 * copy_items` loads in flight. A per-array copy would instead order the value loads after the key stores (the // compiler cannot reorder across the separate loops) and expose their latency on small segments. - _CCCL_DEVICE _CCCL_FORCEINLINE void copy_segment_select_all() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void copy_segment_select_all() { constexpr int copy_items = copy_items_per_thread_clamped; const offset_t num_cluster_items = static_cast(segment_size); @@ -3306,7 +3306,7 @@ private: // path; the cluster's other CTAs are redundant and this returns false so they exit, freeing their SM slots. The // decision is per-segment uniform across the block, so a redundant CTA returns whole. Compiled out for host-exact // sizes, which the dispatch already sized to exact cluster blocks (returns true for all). - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE bool init_logical_cluster() + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE bool init_logical_cluster() { if constexpr (enable_runtime_single_cta) { @@ -3325,7 +3325,7 @@ private: return true; } - _CCCL_DEVICE _CCCL_FORCEINLINE void process_impl() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void process_impl() { // Raw launched cluster rank/size from the PTX special registers (replaces cooperative_groups' `this_cluster()`), // cached into the members. The radix path runs on the *logical* cluster (`layout.num_logical_cluster_blocks`, set diff --git a/cub/cub/agent/agent_find.cuh b/cub/cub/agent/agent_find.cuh index 80ac916a855b..c031592d3ce1 100644 --- a/cub/cub/agent/agent_find.cuh +++ b/cub/cub/agent/agent_find.cuh @@ -72,7 +72,7 @@ struct agent_t OffsetT num_items; template - _CCCL_DEVICE _CCCL_FORCEINLINE bool is_aligned_and_full_tile(OffsetT tile_offset) + _CCCL_DEVICE_API _CCCL_FORCEINLINE bool is_aligned_and_full_tile(OffsetT tile_offset) { if constexpr (CanVectorize) { @@ -93,7 +93,7 @@ struct agent_t } } - _CCCL_DEVICE _CCCL_FORCEINLINE bool ConsumeTile(OffsetT tile_offset, ::cuda::std::true_type /*CAN_VECTORIZE*/) + _CCCL_DEVICE_API _CCCL_FORCEINLINE bool ConsumeTile(OffsetT tile_offset, ::cuda::std::true_type /*CAN_VECTORIZE*/) { using InputT = typename ::cuda::std::iterator_traits::value_type; @@ -123,7 +123,7 @@ struct agent_t return false; } - _CCCL_DEVICE _CCCL_FORCEINLINE bool ConsumeTile(OffsetT tile_offset, ::cuda::std::false_type /*CAN_VECTORIZE*/) + _CCCL_DEVICE_API _CCCL_FORCEINLINE bool ConsumeTile(OffsetT tile_offset, ::cuda::std::false_type /*CAN_VECTORIZE*/) { for (int i = 0; i < ItemsPerThread; ++i) { @@ -143,7 +143,7 @@ struct agent_t return false; } - _CCCL_DEVICE _CCCL_FORCEINLINE void Process() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Process() { if (threadIdx.x == 0) { diff --git a/cub/cub/agent/agent_for.cuh b/cub/cub/agent/agent_for.cuh index dea073957ee6..d4a107a6cd2e 100644 --- a/cub/cub/agent/agent_for.cuh +++ b/cub/cub/agent/agent_for.cuh @@ -33,7 +33,7 @@ struct agent_block_striped_t OpT op; template - _CCCL_DEVICE _CCCL_FORCEINLINE void consume_tile(int items_in_tile, int threads_per_block) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void consume_tile(int items_in_tile, int threads_per_block) { _CCCL_PRAGMA_UNROLL_FULL() for (int item = 0; item < items_per_thread; item++) diff --git a/cub/cub/agent/agent_histogram.cuh b/cub/cub/agent/agent_histogram.cuh index 44c1f71acf09..da8b61ff0402 100644 --- a/cub/cub/agent/agent_histogram.cuh +++ b/cub/cub/agent/agent_histogram.cuh @@ -68,7 +68,7 @@ namespace detail } } // namespace detail -inline ::std::ostream& operator<<(::std::ostream& os, BlockHistogramMemoryPreference mempref) +_CCCL_HOST_API inline ::std::ostream& operator<<(::std::ostream& os, BlockHistogramMemoryPreference mempref) { return os << _CUB::detail::to_string(mempref); } @@ -155,14 +155,14 @@ namespace detail::histogram { // Return a native item pointer (specialized for CacheModifiedInputIterator types) template -_CCCL_DEVICE _CCCL_FORCEINLINE auto NativePointer(CacheModifiedInputIterator itr) +_CCCL_DEVICE_API _CCCL_FORCEINLINE auto NativePointer(CacheModifiedInputIterator itr) { return itr.ptr; } // Return a native item pointer (specialized for other types) template -_CCCL_DEVICE _CCCL_FORCEINLINE auto NativePointer(IteratorT itr) +_CCCL_DEVICE_API _CCCL_FORCEINLINE auto NativePointer(IteratorT itr) { return nullptr; } @@ -276,7 +276,7 @@ struct AgentHistogram bool prefer_smem; // for privatized counterss template - _CCCL_DEVICE _CCCL_FORCEINLINE void ZeroBinCounters(TwoDimSubscriptableCounterT& privatized_histograms) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ZeroBinCounters(TwoDimSubscriptableCounterT& privatized_histograms) { _CCCL_PRAGMA_UNROLL_FULL() for (int ch = 0; ch < NumActiveChannels; ++ch) @@ -294,7 +294,7 @@ struct AgentHistogram // Update final output histograms from privatized histograms template - _CCCL_DEVICE _CCCL_FORCEINLINE void StoreOutput(TwoDimSubscriptableCounterT& privatized_histograms) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void StoreOutput(TwoDimSubscriptableCounterT& privatized_histograms) { // Barrier to make sure all threads are done updating counters __syncthreads(); @@ -321,7 +321,7 @@ struct AgentHistogram // Accumulate items. Specialized for RLE compression. template - _CCCL_DEVICE _CCCL_FORCEINLINE void AccumulateItems( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void AccumulateItems( SampleT samples[items_per_thread][NumChannels], bool is_valid[items_per_thread], TwoDimSubscriptableCounterT& privatized_histograms, @@ -371,7 +371,7 @@ struct AgentHistogram // Accumulate items. Specialized for individual accumulation of each item. template - _CCCL_DEVICE _CCCL_FORCEINLINE void AccumulateItems( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void AccumulateItems( SampleT samples[items_per_thread][NumChannels], bool is_valid[items_per_thread], TwoDimSubscriptableCounterT& privatized_histograms, @@ -396,7 +396,7 @@ struct AgentHistogram } // Load full, aligned tile using item iterator - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void LoadFullAlignedTile(OffsetT block_offset, SampleT (&samples)[items_per_thread][NumChannels]) { // The vectorized load can only load whole tiles if the samples per thread are a multiple of the vector size @@ -417,7 +417,7 @@ struct AgentHistogram } template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void LoadTile(OffsetT block_offset, int valid_samples, SampleT (&samples)[items_per_thread][NumChannels]) { if constexpr (IsFullTile) @@ -457,7 +457,7 @@ struct AgentHistogram } template - _CCCL_DEVICE _CCCL_FORCEINLINE void MarkValid(bool (&is_valid)[items_per_thread], int valid_samples) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void MarkValid(bool (&is_valid)[items_per_thread], int valid_samples) { _CCCL_PRAGMA_UNROLL_FULL() for (int item = 0; item < items_per_thread; ++item) @@ -481,7 +481,7 @@ struct AgentHistogram //! @tparam IsFullTile //! Whether the tile is full template - _CCCL_DEVICE _CCCL_FORCEINLINE void ConsumeTile(OffsetT block_offset, int valid_samples) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ConsumeTile(OffsetT block_offset, int valid_samples) { SampleT samples[items_per_thread][NumChannels]; bool is_valid[items_per_thread]; @@ -513,7 +513,7 @@ struct AgentHistogram //! @param tiles_per_row //! Number of image tiles per row template - _CCCL_DEVICE _CCCL_FORCEINLINE void ConsumeTiles( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ConsumeTiles( OffsetT num_row_items, OffsetT num_rows, OffsetT row_stride_samples, @@ -561,7 +561,7 @@ struct AgentHistogram // The shared memory of BLOCK_LOAD_TRANSPOSE is reused when loading the next tile. The work-stealing ConsumeTiles // already synchronizes between tiles. - _CCCL_DEVICE _CCCL_FORCEINLINE void SyncBeforeNextTile() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SyncBeforeNextTile() { if constexpr (AgentHistogramPolicyT::LOAD_ALGORITHM == BLOCK_LOAD_TRANSPOSE) { @@ -580,7 +580,7 @@ struct AgentHistogram //! @param row_stride_samples //! The number of samples between starts of consecutive rows in the region of interest template - _CCCL_DEVICE _CCCL_FORCEINLINE void ConsumeTiles( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ConsumeTiles( OffsetT num_row_items, OffsetT num_rows, OffsetT row_stride_samples, int, GridQueue, ::cuda::std::false_type) { for (int row = static_cast(blockIdx.y); row < num_rows; row += static_cast(gridDim.y)) @@ -638,7 +638,7 @@ struct AgentHistogram //! //! @param privatized_decode_op //! The transform operator for determining privatized counter indices from samples, one for each channel - _CCCL_DEVICE _CCCL_FORCEINLINE AgentHistogram( + _CCCL_DEVICE_API _CCCL_FORCEINLINE AgentHistogram( TempStorage& temp_storage, SampleIteratorT d_samples, const int* num_output_bins, @@ -687,7 +687,7 @@ struct AgentHistogram //! //! @param tile_queue //! Queue descriptor for assigning tiles of work to thread blocks - _CCCL_DEVICE _CCCL_FORCEINLINE void ConsumeTiles( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ConsumeTiles( OffsetT num_row_items, OffsetT num_rows, OffsetT row_stride_samples, int tiles_per_row, GridQueue tile_queue) { // Check whether all row starting offsets are vec-aligned (in single-channel) or item-aligned (in multi-channel) @@ -724,7 +724,7 @@ struct AgentHistogram } //! Initialize privatized bin counters. Specialized for privatized shared-memory counters - _CCCL_DEVICE _CCCL_FORCEINLINE void InitBinCounters() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InitBinCounters() { if (prefer_smem) { @@ -737,7 +737,7 @@ struct AgentHistogram } //! Store privatized histogram to device-accessible memory. Specialized for privatized shared-memory counters - _CCCL_DEVICE _CCCL_FORCEINLINE void StoreOutput() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void StoreOutput() { if (prefer_smem) { diff --git a/cub/cub/agent/agent_merge.cuh b/cub/cub/agent/agent_merge.cuh index f9d6c0039d8c..ad2ad956c9c9 100644 --- a/cub/cub/agent/agent_merge.cuh +++ b/cub/cub/agent/agent_merge.cuh @@ -149,7 +149,7 @@ struct agent_t Offset* key1_beg_offsets; template - _CCCL_DEVICE _CCCL_FORCEINLINE void consume_tile(Offset tile_idx, Offset tile_base, int num_remaining) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void consume_tile(Offset tile_idx, Offset tile_base, int num_remaining) { const Offset diag0 = items_per_tile * tile_idx; Offset diag1 = diag0 + items_per_tile; @@ -354,7 +354,7 @@ struct agent_t } } - _CCCL_DEVICE _CCCL_FORCEINLINE void operator()() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void operator()() { const Offset tile_idx = blockIdx.x; const Offset tile_base = tile_idx * items_per_tile; diff --git a/cub/cub/agent/agent_merge_sort.cuh b/cub/cub/agent/agent_merge_sort.cuh index fede61ceace3..cb0e4b7d2c42 100644 --- a/cub/cub/agent/agent_merge_sort.cuh +++ b/cub/cub/agent/agent_merge_sort.cuh @@ -103,7 +103,7 @@ struct AgentBlockSort ValueT* items_out_raw; CompareOpT compare_op; - _CCCL_DEVICE _CCCL_FORCEINLINE AgentBlockSort( + _CCCL_DEVICE_API _CCCL_FORCEINLINE AgentBlockSort( bool ping_, TempStorage& storage_, KeysLoadIt keys_in_, @@ -126,7 +126,7 @@ struct AgentBlockSort , compare_op(compare_op_) {} - _CCCL_DEVICE _CCCL_FORCEINLINE void Process() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Process() { const auto tile_idx = static_cast(blockIdx.x); const auto num_tiles = static_cast(gridDim.x); @@ -144,7 +144,7 @@ struct AgentBlockSort } template - _CCCL_DEVICE _CCCL_FORCEINLINE void consume_tile(OffsetT tile_base, int num_remaining) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void consume_tile(OffsetT tile_base, int num_remaining) { ValueT items_local[ITEMS_PER_THREAD]; @@ -272,7 +272,7 @@ struct AgentPartition int items_per_tile; OffsetT num_partitions; - _CCCL_DEVICE _CCCL_FORCEINLINE void Process() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Process() { const OffsetT merged_tiles_number = target_merged_tiles_number / 2; @@ -338,7 +338,7 @@ struct AgentPartition * stores the result in output[item]. */ template -_CCCL_DEVICE _CCCL_FORCEINLINE void +_CCCL_DEVICE_API _CCCL_FORCEINLINE void gmem_to_reg(T (&output)[ItemsPerThread], It1 input1, It2 input2, int count1, int count2) { if constexpr (IsFullTile) @@ -367,7 +367,7 @@ gmem_to_reg(T (&output)[ItemsPerThread], It1 input1, It2 input2, int count1, int /// \brief Stores data in a coalesced fashion in[item] -> out[BlockThreads * item + tid] template -_CCCL_DEVICE _CCCL_FORCEINLINE void reg_to_shared(It output, T (&input)[ItemsPerThread]) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void reg_to_shared(It output, T (&input)[ItemsPerThread]) { _CCCL_PRAGMA_UNROLL_FULL() for (int item = 0; item < ItemsPerThread; ++item) @@ -467,7 +467,7 @@ struct AgentMerge //--------------------------------------------------------------------- template - _CCCL_DEVICE _CCCL_FORCEINLINE void consume_tile(int tid, OffsetT tile_idx, OffsetT tile_base, int count) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void consume_tile(int tid, OffsetT tile_idx, OffsetT tile_base, int count) { _CCCL_PDL_GRID_DEPENDENCY_SYNC(); @@ -657,7 +657,7 @@ struct AgentMerge } } - _CCCL_DEVICE _CCCL_FORCEINLINE AgentMerge( + _CCCL_DEVICE_API _CCCL_FORCEINLINE AgentMerge( bool ping_, TempStorage& storage_, KeysLoadPingIt keys_in_ping_, @@ -688,7 +688,7 @@ struct AgentMerge , target_merged_tiles_number(target_merged_tiles_number_) {} - _CCCL_DEVICE _CCCL_FORCEINLINE void Process() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Process() { const int tile_idx = static_cast(blockIdx.x); const int num_tiles = static_cast(gridDim.x); diff --git a/cub/cub/agent/agent_radix_sort_downsweep.cuh b/cub/cub/agent/agent_radix_sort_downsweep.cuh index 5280e2308677..68281d6d0cc1 100644 --- a/cub/cub/agent/agent_radix_sort_downsweep.cuh +++ b/cub/cub/agent/agent_radix_sort_downsweep.cuh @@ -269,7 +269,7 @@ struct AgentRadixSortDownsweep // Utility methods //--------------------------------------------------------------------- - _CCCL_DEVICE _CCCL_FORCEINLINE digit_extractor_t digit_extractor() + _CCCL_DEVICE_API _CCCL_FORCEINLINE digit_extractor_t digit_extractor() { return traits::template digit_extractor(current_bit, num_bits, decomposer); } @@ -278,7 +278,7 @@ struct AgentRadixSortDownsweep * Scatter ranked keys through shared memory, then to device-accessible memory */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterKeys( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterKeys( bit_ordered_type (&twiddled_keys)[ITEMS_PER_THREAD], OffsetT (&relative_bin_offsets)[ITEMS_PER_THREAD], int (&ranks)[ITEMS_PER_THREAD], @@ -314,7 +314,7 @@ struct AgentRadixSortDownsweep * Scatter ranked values through shared memory, then to device-accessible memory */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterValues( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterValues( ValueT (&values)[ITEMS_PER_THREAD], OffsetT (&relative_bin_offsets)[ITEMS_PER_THREAD], int (&ranks)[ITEMS_PER_THREAD], @@ -350,7 +350,7 @@ struct AgentRadixSortDownsweep * Truck along associated values */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void GatherScatterValues( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void GatherScatterValues( OffsetT (&relative_bin_offsets)[ITEMS_PER_THREAD], int (&ranks)[ITEMS_PER_THREAD], OffsetT block_offset, @@ -395,7 +395,7 @@ struct AgentRadixSortDownsweep * Process tile */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void ProcessTile(OffsetT block_offset, OffsetT valid_items = TILE_ITEMS) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ProcessTile(OffsetT block_offset, OffsetT valid_items = TILE_ITEMS) { bit_ordered_type keys[ITEMS_PER_THREAD]; int ranks[ITEMS_PER_THREAD]; @@ -521,7 +521,7 @@ struct AgentRadixSortDownsweep * Copy tiles within the range of input */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void Copy(InputIteratorT d_in, T* d_out, OffsetT block_offset, OffsetT block_end) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Copy(InputIteratorT d_in, T* d_out, OffsetT block_offset, OffsetT block_end) { // Simply copy the input while (block_end - block_offset >= TILE_ITEMS) @@ -552,7 +552,7 @@ struct AgentRadixSortDownsweep * Copy tiles within the range of input (specialized for NullType) */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Copy(InputIteratorT /*d_in*/, NullType* /*d_out*/, OffsetT /*block_offset*/, OffsetT /*block_end*/) {} @@ -563,7 +563,7 @@ struct AgentRadixSortDownsweep /** * Constructor */ - _CCCL_DEVICE _CCCL_FORCEINLINE AgentRadixSortDownsweep( + _CCCL_DEVICE_API _CCCL_FORCEINLINE AgentRadixSortDownsweep( TempStorage& temp_storage, OffsetT (&bin_offset)[BINS_TRACKED_PER_THREAD], OffsetT num_items, @@ -603,7 +603,7 @@ struct AgentRadixSortDownsweep /** * Constructor */ - _CCCL_DEVICE _CCCL_FORCEINLINE AgentRadixSortDownsweep( + _CCCL_DEVICE_API _CCCL_FORCEINLINE AgentRadixSortDownsweep( TempStorage& temp_storage, OffsetT num_items, OffsetT* d_spine, @@ -652,7 +652,7 @@ struct AgentRadixSortDownsweep /** * Distribute keys from a segment of input tiles. */ - _CCCL_DEVICE _CCCL_FORCEINLINE void ProcessRegion(OffsetT block_offset, OffsetT block_end) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ProcessRegion(OffsetT block_offset, OffsetT block_end) { if (short_circuit) { diff --git a/cub/cub/agent/agent_radix_sort_histogram.cuh b/cub/cub/agent/agent_radix_sort_histogram.cuh index c291e377204d..0ada29097c0f 100644 --- a/cub/cub/agent/agent_radix_sort_histogram.cuh +++ b/cub/cub/agent/agent_radix_sort_histogram.cuh @@ -143,7 +143,7 @@ struct AgentRadixSortHistogram DecomposerT decomposer; - _CCCL_DEVICE _CCCL_FORCEINLINE AgentRadixSortHistogram( + _CCCL_DEVICE_API _CCCL_FORCEINLINE AgentRadixSortHistogram( TempStorage& temp_storage, OffsetT* d_bins_out, const KeyT* d_keys_in, @@ -161,7 +161,7 @@ struct AgentRadixSortHistogram , decomposer(decomposer) {} - _CCCL_DEVICE _CCCL_FORCEINLINE void Init() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Init() { // Initialize bins to 0. _CCCL_PRAGMA_UNROLL_FULL() @@ -180,7 +180,7 @@ struct AgentRadixSortHistogram __syncthreads(); } - _CCCL_DEVICE _CCCL_FORCEINLINE void LoadTileKeys(OffsetT tile_offset, bit_ordered_type (&keys)[ITEMS_PER_THREAD]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void LoadTileKeys(OffsetT tile_offset, bit_ordered_type (&keys)[ITEMS_PER_THREAD]) { // tile_offset < num_items always, hence the line below works const bool full_tile = num_items - tile_offset >= TILE_ITEMS; @@ -201,7 +201,7 @@ struct AgentRadixSortHistogram } } - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void AccumulateSharedHistograms(OffsetT tile_offset, bit_ordered_type (&keys)[ITEMS_PER_THREAD]) { const int part = ::cuda::ptx::get_sreg_laneid() % NUM_PARTS; @@ -222,7 +222,7 @@ struct AgentRadixSortHistogram } } - _CCCL_DEVICE _CCCL_FORCEINLINE void AccumulateGlobalHistograms() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void AccumulateGlobalHistograms() { _CCCL_PRAGMA_UNROLL_FULL() for (int bin = static_cast(threadIdx.x); bin < RADIX_DIGITS; bin += BLOCK_THREADS) @@ -244,7 +244,7 @@ struct AgentRadixSortHistogram } } - _CCCL_DEVICE _CCCL_FORCEINLINE void Process() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Process() { _CCCL_PDL_TRIGGER_NEXT_LAUNCH(); // Within a portion, avoid overflowing (u)int32 counters. @@ -277,7 +277,7 @@ struct AgentRadixSortHistogram } } - _CCCL_DEVICE _CCCL_FORCEINLINE digit_extractor_t digit_extractor(int current_bit, int num_bits) + _CCCL_DEVICE_API _CCCL_FORCEINLINE digit_extractor_t digit_extractor(int current_bit, int num_bits) { return traits::template digit_extractor(current_bit, num_bits, decomposer); } diff --git a/cub/cub/agent/agent_radix_sort_onesweep.cuh b/cub/cub/agent/agent_radix_sort_onesweep.cuh index 88dfca9fdb79..2bda4ef9e6ec 100644 --- a/cub/cub/agent/agent_radix_sort_onesweep.cuh +++ b/cub/cub/agent/agent_radix_sort_onesweep.cuh @@ -80,7 +80,7 @@ namespace detail #endif // _CCCL_HOSTED() #if _CCCL_HOSTED() && !defined(_CCCL_DOXYGEN_INVOKED) -inline ::std::ostream& operator<<(::std::ostream& os, RadixSortStoreAlgorithm algo) +_CCCL_HOST_API inline ::std::ostream& operator<<(::std::ostream& os, RadixSortStoreAlgorithm algo) { return os << _CUB::detail::to_string(algo); } @@ -260,23 +260,23 @@ struct AgentRadixSortOnesweep PortionOffsetT block_idx; bool full_block; - _CCCL_DEVICE _CCCL_FORCEINLINE digit_extractor_t digit_extractor() + _CCCL_DEVICE_API _CCCL_FORCEINLINE digit_extractor_t digit_extractor() { return traits::template digit_extractor(current_bit, num_bits, decomposer); } // helper methods - _CCCL_DEVICE _CCCL_FORCEINLINE uint32_t Digit(bit_ordered_type key) + _CCCL_DEVICE_API _CCCL_FORCEINLINE uint32_t Digit(bit_ordered_type key) { return digit_extractor().Digit(key); } - _CCCL_DEVICE _CCCL_FORCEINLINE int ThreadBin(int u) + _CCCL_DEVICE_API _CCCL_FORCEINLINE int ThreadBin(int u) { return (threadIdx.x * BINS_PER_THREAD) + u; } - _CCCL_DEVICE _CCCL_FORCEINLINE void LookbackPartial(int (&bins)[BINS_PER_THREAD]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void LookbackPartial(int (&bins)[BINS_PER_THREAD]) { _CCCL_PRAGMA_UNROLL_FULL() for (int u = 0; u < BINS_PER_THREAD; ++u) @@ -300,13 +300,13 @@ struct AgentRadixSortOnesweep int (&bins)[BINS_PER_THREAD]; bit_ordered_type (&keys)[ITEMS_PER_THREAD]; static constexpr bool EMPTY = false; - _CCCL_DEVICE _CCCL_FORCEINLINE + _CCCL_DEVICE_API _CCCL_FORCEINLINE CountsCallback(AgentT& agent, int (&bins)[BINS_PER_THREAD], bit_ordered_type (&keys)[ITEMS_PER_THREAD]) : agent(agent) , bins(bins) , keys(keys) {} - _CCCL_DEVICE _CCCL_FORCEINLINE void operator()(int (&other_bins)[BINS_PER_THREAD]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void operator()(int (&other_bins)[BINS_PER_THREAD]) { _CCCL_PRAGMA_UNROLL_FULL() for (int u = 0; u < BINS_PER_THREAD; ++u) @@ -322,7 +322,7 @@ struct AgentRadixSortOnesweep } }; - _CCCL_DEVICE _CCCL_FORCEINLINE void LookbackGlobal(int (&bins)[BINS_PER_THREAD]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void LookbackGlobal(int (&bins)[BINS_PER_THREAD]) { _CCCL_PRAGMA_UNROLL_FULL() for (int u = 0; u < BINS_PER_THREAD; ++u) @@ -360,7 +360,7 @@ struct AgentRadixSortOnesweep _CCCL_PDL_TRIGGER_NEXT_LAUNCH(); } - _CCCL_DEVICE _CCCL_FORCEINLINE void LoadKeys(OffsetT tile_offset, bit_ordered_type (&keys)[ITEMS_PER_THREAD]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void LoadKeys(OffsetT tile_offset, bit_ordered_type (&keys)[ITEMS_PER_THREAD]) { if (full_block) { @@ -379,7 +379,7 @@ struct AgentRadixSortOnesweep } } - _CCCL_DEVICE _CCCL_FORCEINLINE void LoadValues(OffsetT tile_offset, ValueT (&values)[ITEMS_PER_THREAD]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void LoadValues(OffsetT tile_offset, ValueT (&values)[ITEMS_PER_THREAD]) { if (full_block) { @@ -403,7 +403,7 @@ struct AgentRadixSortOnesweep * d_keys_out corresponding to their digit. The values (if also sorting * values) assigned to the current thread block are similarly copied from * d_values_in to d_values_out. */ - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void TryShortCircuit(bit_ordered_type (&keys)[ITEMS_PER_THREAD], int (&bins)[BINS_PER_THREAD]) { // check if any bin can be short-circuited @@ -426,7 +426,7 @@ struct AgentRadixSortOnesweep ShortCircuitCopy(keys, bins); } - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ShortCircuitCopy(bit_ordered_type (&keys)[ITEMS_PER_THREAD], int (&bins)[BINS_PER_THREAD]) { // short-circuit handling; note that global look-back is still required @@ -486,7 +486,7 @@ struct AgentRadixSortOnesweep _CUB::ThreadExit(); } - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterKeysShared(bit_ordered_type (&keys)[ITEMS_PER_THREAD], int (&ranks)[ITEMS_PER_THREAD]) { // write to shared memory @@ -497,7 +497,7 @@ struct AgentRadixSortOnesweep } } - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterValuesShared(ValueT (&values)[ITEMS_PER_THREAD], int (&ranks)[ITEMS_PER_THREAD]) { // write to shared memory @@ -508,7 +508,7 @@ struct AgentRadixSortOnesweep } } - _CCCL_DEVICE _CCCL_FORCEINLINE void LoadBinsToOffsetsGlobal(int (&offsets)[BINS_PER_THREAD]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void LoadBinsToOffsetsGlobal(int (&offsets)[BINS_PER_THREAD]) { // global offset - global part _CCCL_PRAGMA_UNROLL_FULL() @@ -522,7 +522,7 @@ struct AgentRadixSortOnesweep } } - _CCCL_DEVICE _CCCL_FORCEINLINE void UpdateBinsGlobal(int (&bins)[BINS_PER_THREAD], int (&offsets)[BINS_PER_THREAD]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void UpdateBinsGlobal(int (&bins)[BINS_PER_THREAD], int (&offsets)[BINS_PER_THREAD]) { const bool last_block = (block_idx + 1) * TILE_ITEMS >= num_items; if (d_bins_out != nullptr && last_block) @@ -540,7 +540,7 @@ struct AgentRadixSortOnesweep } template - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterKeysGlobalDirect() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterKeysGlobalDirect() { const int tile_items = FullTile ? TILE_ITEMS : num_items - block_idx * TILE_ITEMS; @@ -559,7 +559,7 @@ struct AgentRadixSortOnesweep } template - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterValuesGlobalDirect(int (&digits)[ITEMS_PER_THREAD]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterValuesGlobalDirect(int (&digits)[ITEMS_PER_THREAD]) { const int tile_items = FullTile ? TILE_ITEMS : num_items - block_idx * TILE_ITEMS; @@ -577,7 +577,7 @@ struct AgentRadixSortOnesweep } } - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterKeysGlobalAligned() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterKeysGlobalAligned() { // this only works with full tiles constexpr int ITEMS_PER_WARP = TILE_ITEMS / BLOCK_WARPS; @@ -618,7 +618,7 @@ struct AgentRadixSortOnesweep } } - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterKeysGlobal() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterKeysGlobal() { // write block data to global memory if (full_block) @@ -638,7 +638,7 @@ struct AgentRadixSortOnesweep } } - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterValuesGlobal(int (&digits)[ITEMS_PER_THREAD]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterValuesGlobal(int (&digits)[ITEMS_PER_THREAD]) { // write block data to global memory if (full_block) @@ -651,7 +651,7 @@ struct AgentRadixSortOnesweep } } - _CCCL_DEVICE _CCCL_FORCEINLINE void ComputeKeyDigits(int (&digits)[ITEMS_PER_THREAD]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ComputeKeyDigits(int (&digits)[ITEMS_PER_THREAD]) { _CCCL_PRAGMA_UNROLL_FULL() for (int u = 0; u < ITEMS_PER_THREAD; ++u) @@ -661,7 +661,7 @@ struct AgentRadixSortOnesweep } } - _CCCL_DEVICE _CCCL_FORCEINLINE void GatherScatterValues(int (&ranks)[ITEMS_PER_THREAD]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void GatherScatterValues(int (&ranks)[ITEMS_PER_THREAD]) { // compute digits corresponding to the keys int digits[ITEMS_PER_THREAD]; @@ -679,7 +679,7 @@ struct AgentRadixSortOnesweep ScatterValuesGlobal(digits); } - _CCCL_DEVICE _CCCL_FORCEINLINE void Process() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Process() { // load keys // if warp1 < warp2, all elements of warp1 occur before those of warp2 @@ -714,7 +714,7 @@ struct AgentRadixSortOnesweep } } - _CCCL_DEVICE _CCCL_FORCEINLINE // + _CCCL_DEVICE_API _CCCL_FORCEINLINE // AgentRadixSortOnesweep( TempStorage& temp_storage, AtomicOffsetT* d_lookback, diff --git a/cub/cub/agent/agent_radix_sort_upsweep.cuh b/cub/cub/agent/agent_radix_sort_upsweep.cuh index 394bcc456618..19c979eda517 100644 --- a/cub/cub/agent/agent_radix_sort_upsweep.cuh +++ b/cub/cub/agent/agent_radix_sort_upsweep.cuh @@ -219,7 +219,7 @@ struct AgentRadixSortUpsweep //--------------------------------------------------------------------- // Utility methods //--------------------------------------------------------------------- - _CCCL_DEVICE _CCCL_FORCEINLINE digit_extractor_t digit_extractor() + _CCCL_DEVICE_API _CCCL_FORCEINLINE digit_extractor_t digit_extractor() { return traits::template digit_extractor(current_bit, num_bits, decomposer); } @@ -227,7 +227,7 @@ struct AgentRadixSortUpsweep /** * Decode a key and increment corresponding smem digit counter */ - _CCCL_DEVICE _CCCL_FORCEINLINE void Bucket(bit_ordered_type key) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Bucket(bit_ordered_type key) { // Perform transform op // `Digit()` takes a mutable reference for the decomposer overload. @@ -251,7 +251,7 @@ struct AgentRadixSortUpsweep /** * Reset composite counters */ - _CCCL_DEVICE _CCCL_FORCEINLINE void ResetDigitCounters() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ResetDigitCounters() { _CCCL_PRAGMA_UNROLL_FULL() for (int LANE = 0; LANE < COUNTER_LANES; LANE++) @@ -263,7 +263,7 @@ struct AgentRadixSortUpsweep /** * Reset the unpacked counters in each thread */ - _CCCL_DEVICE _CCCL_FORCEINLINE void ResetUnpackedCounters() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ResetUnpackedCounters() { _CCCL_PRAGMA_UNROLL_FULL() for (int LANE = 0; LANE < LANES_PER_WARP; LANE++) @@ -280,7 +280,7 @@ struct AgentRadixSortUpsweep * Extracts and aggregates the digit counters for each counter lane * owned by this warp */ - _CCCL_DEVICE _CCCL_FORCEINLINE void UnpackDigitCounts() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void UnpackDigitCounts() { const unsigned int warp_id = threadIdx.x >> LOG_WARP_THREADS; const unsigned int warp_tid = ::cuda::ptx::get_sreg_laneid(); @@ -308,7 +308,7 @@ struct AgentRadixSortUpsweep /** * Processes a single, full tile */ - _CCCL_DEVICE _CCCL_FORCEINLINE void ProcessFullTile(OffsetT block_offset) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ProcessFullTile(OffsetT block_offset) { // Tile of keys bit_ordered_type keys[KEYS_PER_THREAD]; @@ -327,7 +327,7 @@ struct AgentRadixSortUpsweep /** * Processes a single load (may have some threads masked off) */ - _CCCL_DEVICE _CCCL_FORCEINLINE void ProcessPartialTile(OffsetT block_offset, const OffsetT& block_end) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ProcessPartialTile(OffsetT block_offset, const OffsetT& block_end) { // Process partial tile if necessary using single loads for (OffsetT offset = threadIdx.x; offset < block_end - block_offset; offset += BLOCK_THREADS) @@ -345,7 +345,7 @@ struct AgentRadixSortUpsweep /** * Constructor */ - _CCCL_DEVICE _CCCL_FORCEINLINE AgentRadixSortUpsweep( + _CCCL_DEVICE_API _CCCL_FORCEINLINE AgentRadixSortUpsweep( TempStorage& temp_storage, const KeyT* d_keys_in, int current_bit, int num_bits, DecomposerT decomposer = {}) : temp_storage(temp_storage.Alias()) , d_keys_in(reinterpret_cast(d_keys_in)) @@ -357,7 +357,7 @@ struct AgentRadixSortUpsweep /** * Compute radix digit histograms from a segment of input tiles. */ - _CCCL_DEVICE _CCCL_FORCEINLINE void ProcessRegion(OffsetT block_offset, const OffsetT& block_end) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ProcessRegion(OffsetT block_offset, const OffsetT& block_end) { // Reset digit counters in smem and unpacked counters in registers ResetDigitCounters(); @@ -403,7 +403,7 @@ struct AgentRadixSortUpsweep * Extract counts (saving them to the external array) */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void ExtractCounts(OffsetT* counters, int bin_stride = 1, int bin_offset = 0) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExtractCounts(OffsetT* counters, int bin_stride = 1, int bin_offset = 0) { const unsigned int warp_id = threadIdx.x >> LOG_WARP_THREADS; const unsigned int warp_tid = ::cuda::ptx::get_sreg_laneid(); @@ -483,7 +483,7 @@ struct AgentRadixSortUpsweep * 1] */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void ExtractCounts(OffsetT (&bin_count)[BinsTrackedPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExtractCounts(OffsetT (&bin_count)[BinsTrackedPerThread]) { const unsigned int warp_id = threadIdx.x >> LOG_WARP_THREADS; const unsigned int warp_tid = ::cuda::ptx::get_sreg_laneid(); diff --git a/cub/cub/agent/agent_reduce.cuh b/cub/cub/agent/agent_reduce.cuh index 65e2b671a49e..88f9be0565bc 100644 --- a/cub/cub/agent/agent_reduce.cuh +++ b/cub/cub/agent/agent_reduce.cuh @@ -275,7 +275,7 @@ struct AgentReduceImpl * @param d_in Input data to reduce * @param reduction_op Binary reduction operator */ - _CCCL_DEVICE _CCCL_FORCEINLINE AgentReduceImpl( + _CCCL_DEVICE_API _CCCL_FORCEINLINE AgentReduceImpl( TempStorage& temp_storage, InputIteratorT d_in, ReductionOp reduction_op, TransformOp transform_op, int lane_id) : temp_storage(temp_storage.Alias()) , d_in(_CUB::detail::try_make_cache_modified_iterator(d_in)) @@ -295,7 +295,7 @@ struct AgentReduceImpl * @param input_is_vector_aligned Whether we can vectorize loads */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void ConsumeFullTile(AccumT& thread_aggregate, OffsetT block_offset) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ConsumeFullTile(AccumT& thread_aggregate, OffsetT block_offset) { // load items in striped fashion and reduce items within each thread stripe AccumT items[ITEMS_PER_THREAD]; @@ -330,7 +330,8 @@ struct AgentReduceImpl * @param valid_items The number of valid items in the tile */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void ConsumePartialTile(AccumT& thread_aggregate, OffsetT block_offset, int valid_items) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void + ConsumePartialTile(AccumT& thread_aggregate, OffsetT block_offset, int valid_items) { // Partial tile int thread_offset = lane_id; @@ -361,7 +362,7 @@ struct AgentReduceImpl * @param even_share GridEvenShare descriptor */ template - _CCCL_DEVICE _CCCL_FORCEINLINE AccumT ConsumeRange(GridEvenShare& even_share) + _CCCL_DEVICE_API _CCCL_FORCEINLINE AccumT ConsumeRange(GridEvenShare& even_share) { AccumT thread_aggregate{}; @@ -393,7 +394,7 @@ struct AgentReduceImpl * @param[in] block_offset Threadblock begin offset (inclusive) * @param[in] block_end Threadblock end offset (exclusive) */ - _CCCL_DEVICE _CCCL_FORCEINLINE AccumT ConsumeRange(OffsetT block_offset, OffsetT block_end) + _CCCL_DEVICE_API _CCCL_FORCEINLINE AccumT ConsumeRange(OffsetT block_offset, OffsetT block_end) { GridEvenShare even_share; even_share.template BlockInit(block_offset, block_end); @@ -407,7 +408,7 @@ struct AgentReduceImpl * Reduce a contiguous segment of input tiles * @param[in] even_share GridEvenShare descriptor */ - _CCCL_DEVICE _CCCL_FORCEINLINE AccumT ConsumeTiles(GridEvenShare& even_share) + _CCCL_DEVICE_API _CCCL_FORCEINLINE AccumT ConsumeTiles(GridEvenShare& even_share) { // Initialize GRID_MAPPING_STRIP_MINE even-share descriptor for this thread block even_share.template BlockInit(); @@ -422,7 +423,8 @@ private: * @param input_is_vector_aligned Whether we can vectorize loads */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void ConsumeFullTileRange(AccumT& thread_aggregate, GridEvenShare& even_share) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void + ConsumeFullTileRange(AccumT& thread_aggregate, GridEvenShare& even_share) { // At least one full block ConsumeFullTile(thread_aggregate, even_share.block_offset); @@ -512,7 +514,7 @@ struct AgentReduce BlockReduce, AgentReducePolicy::BLOCK_THREADS>; - _CCCL_DEVICE _CCCL_FORCEINLINE AgentReduce( + _CCCL_DEVICE_API _CCCL_FORCEINLINE AgentReduce( typename base_t::TempStorage& temp_storage, InputIteratorT d_in, ReductionOp reduction_op, @@ -576,7 +578,7 @@ struct AgentWarpReduce AgentReducePolicy::WARP_THREADS, true>; - _CCCL_DEVICE _CCCL_FORCEINLINE AgentWarpReduce( + _CCCL_DEVICE_API _CCCL_FORCEINLINE AgentWarpReduce( typename base_t::TempStorage& temp_storage, InputIteratorT d_in, ReductionOp reduction_op, diff --git a/cub/cub/agent/agent_reduce_by_key.cuh b/cub/cub/agent/agent_reduce_by_key.cuh index 35872d2ae9a2..55e865438f94 100644 --- a/cub/cub/agent/agent_reduce_by_key.cuh +++ b/cub/cub/agent/agent_reduce_by_key.cuh @@ -165,14 +165,14 @@ struct AgentReduceByKey int num_remaining; /// Constructor - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE GuardedInequalityWrapper(EqualityOpT op, int num_remaining) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE GuardedInequalityWrapper(EqualityOpT op, int num_remaining) : op(op) , num_remaining(num_remaining) {} /// Boolean inequality operator, returns (a != b) template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE bool operator()(const T& a, const T& b, int idx) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE bool operator()(const T& a, const T& b, int idx) const { if (idx < num_remaining) { @@ -348,7 +348,7 @@ struct AgentReduceByKey * Streaming context providing context about this partition for streaming invocations */ template - _CCCL_DEVICE _CCCL_FORCEINLINE AgentReduceByKey( + _CCCL_DEVICE_API _CCCL_FORCEINLINE AgentReduceByKey( TempStorage& temp_storage, KeysInputIteratorT d_keys_in, UniqueOutputIteratorT d_unique_out, @@ -370,7 +370,7 @@ struct AgentReduceByKey , streaming_context(streaming_context) {} - _CCCL_DEVICE _CCCL_FORCEINLINE AgentReduceByKey( + _CCCL_DEVICE_API _CCCL_FORCEINLINE AgentReduceByKey( TempStorage& temp_storage, KeysInputIteratorT d_keys_in, UniqueOutputIteratorT d_unique_out, @@ -399,7 +399,7 @@ struct AgentReduceByKey /** * Directly scatter flagged items to output offsets */ - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterDirect( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterDirect( KeyValuePairT (&scatter_items)[ITEMS_PER_THREAD], OffsetT (&segment_flags)[ITEMS_PER_THREAD], OffsetT (&segment_indices)[ITEMS_PER_THREAD]) @@ -423,7 +423,7 @@ struct AgentReduceByKey * value aggregate: the scatter offsets must be decremented for value * aggregates */ - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterTwoPhase( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterTwoPhase( KeyValuePairT (&scatter_items)[ITEMS_PER_THREAD], OffsetT (&segment_flags)[ITEMS_PER_THREAD], OffsetT (&segment_indices)[ITEMS_PER_THREAD], @@ -455,7 +455,7 @@ struct AgentReduceByKey /** * Scatter flagged items */ - _CCCL_DEVICE _CCCL_FORCEINLINE void Scatter( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Scatter( KeyValuePairT (&scatter_items)[ITEMS_PER_THREAD], OffsetT (&segment_flags)[ITEMS_PER_THREAD], OffsetT (&segment_indices)[ITEMS_PER_THREAD], @@ -497,7 +497,7 @@ struct AgentReduceByKey * Global tile state descriptor */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ConsumeTile(OffsetT num_remaining, int tile_idx, OffsetT tile_offset, ScanTileStateT& tile_state) { // Tile keys @@ -741,7 +741,7 @@ struct AgentReduceByKey * @param start_tile * The starting tile for the current grid */ - _CCCL_DEVICE _CCCL_FORCEINLINE void ConsumeRange(OffsetT num_items, ScanTileStateT& tile_state, int start_tile) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ConsumeRange(OffsetT num_items, ScanTileStateT& tile_state, int start_tile) { // Blocks are launched in increasing order, so just assign one tile per // block diff --git a/cub/cub/agent/agent_rle.cuh b/cub/cub/agent/agent_rle.cuh index cec500526246..ed5535ef8d0e 100644 --- a/cub/cub/agent/agent_rle.cuh +++ b/cub/cub/agent/agent_rle.cuh @@ -186,13 +186,13 @@ struct AgentRle OffsetT num_remaining; EqualityOpT equality_op; - _CCCL_DEVICE _CCCL_FORCEINLINE OobInequalityOp(OffsetT num_remaining, EqualityOpT equality_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE OobInequalityOp(OffsetT num_remaining, EqualityOpT equality_op) : num_remaining(num_remaining) , equality_op(equality_op) {} template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE bool operator()(T first, T second, Index idx) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE bool operator()(T first, T second, Index idx) { if (!LastTile || (idx < num_remaining)) { @@ -325,7 +325,7 @@ struct AgentRle * Streaming context providing context about this partition for streaming invocations */ template - _CCCL_DEVICE _CCCL_FORCEINLINE AgentRle( + _CCCL_DEVICE_API _CCCL_FORCEINLINE AgentRle( TempStorage& temp_storage, InputIteratorT d_in, OffsetsOutputIteratorT d_offsets_out, @@ -348,7 +348,7 @@ struct AgentRle //--------------------------------------------------------------------- template - _CCCL_DEVICE _CCCL_FORCEINLINE void InitializeSelections( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InitializeSelections( OffsetT tile_offset, OffsetT num_remaining, T (&items)[ITEMS_PER_THREAD], @@ -436,7 +436,7 @@ struct AgentRle /** * Scan of allocations */ - _CCCL_DEVICE _CCCL_FORCEINLINE void WarpScanAllocations( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void WarpScanAllocations( LengthOffsetPair& tile_aggregate, LengthOffsetPair& warp_aggregate, LengthOffsetPair& warp_exclusive_in_tile, @@ -521,7 +521,7 @@ struct AgentRle /** * Two-phase scatter, specialized for warp time-slicing */ - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterTwoPhase( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterTwoPhase( OffsetT tile_num_runs_exclusive_in_global, OffsetT warp_num_runs_aggregate, OffsetT warp_num_runs_exclusive_in_tile, @@ -591,7 +591,7 @@ struct AgentRle /** * Two-phase scatter */ - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterTwoPhase( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterTwoPhase( OffsetT tile_num_runs_exclusive_in_global, OffsetT warp_num_runs_aggregate, OffsetT warp_num_runs_exclusive_in_tile, @@ -657,7 +657,7 @@ struct AgentRle /** * Direct scatter */ - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterDirect( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterDirect( OffsetT tile_num_runs_exclusive_in_global, OffsetT warp_num_runs_aggregate, OffsetT warp_num_runs_exclusive_in_tile, @@ -702,7 +702,7 @@ struct AgentRle /** * Scatter */ - _CCCL_DEVICE _CCCL_FORCEINLINE void Scatter( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Scatter( OffsetT tile_num_runs_aggregate, OffsetT tile_num_runs_exclusive_in_global, OffsetT warp_num_runs_aggregate, @@ -758,7 +758,7 @@ struct AgentRle * Global list of tile status */ template - _CCCL_DEVICE _CCCL_FORCEINLINE LengthOffsetPair + _CCCL_DEVICE_API _CCCL_FORCEINLINE LengthOffsetPair ConsumeTile(OffsetT num_items, OffsetT num_remaining, int tile_idx, OffsetT tile_offset, ScanTileStateT& tile_status) { if (tile_idx == 0) @@ -1020,7 +1020,7 @@ struct AgentRle * Output iterator type for recording number of items selected */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ConsumeRange(int num_tiles, ScanTileStateT& tile_status, NumRunsIteratorT d_num_runs_out) { // Blocks are launched in increasing order, so just assign one tile per block diff --git a/cub/cub/agent/agent_scan.cuh b/cub/cub/agent/agent_scan.cuh index f84ab61c0d4b..9429f6ed9dbf 100644 --- a/cub/cub/agent/agent_scan.cuh +++ b/cub/cub/agent/agent_scan.cuh @@ -266,7 +266,7 @@ struct AgentScan //--------------------------------------------------------------------- template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScanFirstTile(AccumT (&items)[ITEMS_PER_THREAD], InitValueT init_value, ScanOpT scan_op, AccumT& block_aggregate) { BlockScanT blockScan(temp_storage.scan_storage.scan); @@ -290,7 +290,7 @@ struct AgentScan } template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScanFirstTileWithPrefix(AccumT (&items)[ITEMS_PER_THREAD], AccumT prefix, ScanOpT scan_op, AccumT& block_aggregate) { BlockScanT block_scan(temp_storage.scan_storage.scan); @@ -306,7 +306,7 @@ struct AgentScan } template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScanSubsequentTile(AccumT (&items)[ITEMS_PER_THREAD], ScanOpT scan_op, PrefixCallback& prefix_op) { BlockScanT blockScan(temp_storage.scan_storage.scan); @@ -340,7 +340,7 @@ struct AgentScan * @param init_value * Initial value to seed the exclusive scan */ - _CCCL_DEVICE _CCCL_FORCEINLINE AgentScan( + _CCCL_DEVICE_API _CCCL_FORCEINLINE AgentScan( TempStorage& temp_storage, InputIteratorT d_in, OutputIteratorT d_out, ScanOpT scan_op, InitValueT init_value) : temp_storage(temp_storage.Alias()) , d_in(d_in) @@ -371,7 +371,7 @@ struct AgentScan * Global tile state descriptor */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ConsumeTile(OffsetT num_remaining, int tile_idx, OffsetT tile_offset, ScanTileStateT& tile_state) { // Load items @@ -447,7 +447,7 @@ struct AgentScan * @param start_tile * The starting tile for the current grid */ - _CCCL_DEVICE _CCCL_FORCEINLINE void ConsumeRange(OffsetT num_items, ScanTileStateT& tile_state, int start_tile) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ConsumeRange(OffsetT num_items, ScanTileStateT& tile_state, int start_tile) { // Blocks are launched in increasing order, so just assign one tile per // block @@ -487,7 +487,7 @@ struct AgentScan //! @param[in] completion_op Called by all threads after look-back completes. //! Only thread 0 provides a valid inclusive prefix. template - _CCCL_DEVICE _CCCL_FORCEINLINE void ConsumeBatchTile( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ConsumeBatchTile( OffsetT num_remaining, OffsetT tile_idx, int batch_tile_idx, @@ -575,7 +575,7 @@ struct AgentScan * Number of valid items in the tile */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ConsumeTile(OffsetT tile_offset, RunningPrefixCallbackOp& prefix_op, int valid_items = TILE_ITEMS) { // Load items @@ -634,7 +634,7 @@ struct AgentScan * @param[in] range_end * Threadblock end offset (exclusive) */ - _CCCL_DEVICE _CCCL_FORCEINLINE void ConsumeRange(OffsetT range_offset, OffsetT range_end) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ConsumeRange(OffsetT range_offset, OffsetT range_end) { BlockScanRunningPrefixOp prefix_op(scan_op); @@ -678,7 +678,7 @@ struct AgentScan * @param[in] prefix * The prefix to apply to the scan segment */ - _CCCL_DEVICE _CCCL_FORCEINLINE void ConsumeRange(OffsetT range_offset, OffsetT range_end, AccumT prefix) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ConsumeRange(OffsetT range_offset, OffsetT range_end, AccumT prefix) { BlockScanRunningPrefixOp prefix_op(prefix, scan_op); diff --git a/cub/cub/agent/agent_scan_by_key.cuh b/cub/cub/agent/agent_scan_by_key.cuh index a53e71dc23ae..7d692ad558d0 100644 --- a/cub/cub/agent/agent_scan_by_key.cuh +++ b/cub/cub/agent/agent_scan_by_key.cuh @@ -230,7 +230,7 @@ struct AgentScanByKey //--------------------------------------------------------------------- // Exclusive scan specialization - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScanTile(FlagValuePairT (&scan_items)[ITEMS_PER_THREAD], FlagValuePairT& tile_aggregate, ::cuda::std::false_type /* is_inclusive */) @@ -239,7 +239,7 @@ struct AgentScanByKey } // Inclusive scan specialization - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScanTile(FlagValuePairT (&scan_items)[ITEMS_PER_THREAD], FlagValuePairT& tile_aggregate, ::cuda::std::true_type /* is_inclusive */) @@ -252,7 +252,7 @@ struct AgentScanByKey //--------------------------------------------------------------------- // Exclusive scan specialization (with prefix from predecessors) - _CCCL_DEVICE _CCCL_FORCEINLINE void ScanTile( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScanTile( FlagValuePairT (&scan_items)[ITEMS_PER_THREAD], FlagValuePairT& tile_aggregate, TilePrefixCallbackT& prefix_op, @@ -263,7 +263,7 @@ struct AgentScanByKey } // Inclusive scan specialization (with prefix from predecessors) - _CCCL_DEVICE _CCCL_FORCEINLINE void ScanTile( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScanTile( FlagValuePairT (&scan_items)[ITEMS_PER_THREAD], FlagValuePairT& tile_aggregate, TilePrefixCallbackT& prefix_op, @@ -278,7 +278,7 @@ struct AgentScanByKey //--------------------------------------------------------------------- template - _CCCL_DEVICE _CCCL_FORCEINLINE void ZipValuesAndFlags( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ZipValuesAndFlags( OffsetT num_remaining, AccumT (&values)[ITEMS_PER_THREAD], OffsetT (&segment_flags)[ITEMS_PER_THREAD], @@ -299,7 +299,7 @@ struct AgentScanByKey } } - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void UnzipValues(AccumT (&values)[ITEMS_PER_THREAD], FlagValuePairT (&scan_items)[ITEMS_PER_THREAD]) { // Unzip values and segment_flags @@ -311,7 +311,7 @@ struct AgentScanByKey } template , ::cuda::std::enable_if_t = 0> - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void AddInitToScan(AccumT (&items)[ITEMS_PER_THREAD], OffsetT (&flags)[ITEMS_PER_THREAD]) { _CCCL_PRAGMA_UNROLL_FULL() @@ -322,7 +322,7 @@ struct AgentScanByKey } template , ::cuda::std::enable_if_t = 0> - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void AddInitToScan(AccumT (& /*items*/)[ITEMS_PER_THREAD], OffsetT (& /*flags*/)[ITEMS_PER_THREAD]) {} @@ -333,7 +333,7 @@ struct AgentScanByKey // Process a tile of input (dynamic chained scan) // template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ConsumeTile(OffsetT /*num_items*/, OffsetT num_remaining, int tile_idx, OffsetT tile_base, ScanTileStateT& tile_state) { // Load items @@ -429,7 +429,7 @@ struct AgentScanByKey // Dequeue and scan tiles of items as part of a dynamic chained scan // with Init functor - _CCCL_DEVICE _CCCL_FORCEINLINE AgentScanByKey( + _CCCL_DEVICE_API _CCCL_FORCEINLINE AgentScanByKey( TempStorage& storage, KeysInputIteratorT d_keys_in, KeyT* d_keys_prev_in, @@ -461,7 +461,7 @@ struct AgentScanByKey * start_tile * The starting tile for the current grid */ - _CCCL_DEVICE _CCCL_FORCEINLINE void ConsumeRange(OffsetT num_items, ScanTileStateT& tile_state, int start_tile) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ConsumeRange(OffsetT num_items, ScanTileStateT& tile_state, int start_tile) { const int tile_idx = static_cast(blockIdx.x); OffsetT tile_base = OffsetT(ITEMS_PER_TILE) * tile_idx; diff --git a/cub/cub/agent/agent_segmented_radix_sort.cuh b/cub/cub/agent/agent_segmented_radix_sort.cuh index 91f70ebaf410..73f96adf421f 100644 --- a/cub/cub/agent/agent_segmented_radix_sort.cuh +++ b/cub/cub/agent/agent_segmented_radix_sort.cuh @@ -119,14 +119,14 @@ struct AgentSegmentedRadixSort DecomposerT decomposer; - _CCCL_DEVICE _CCCL_FORCEINLINE + _CCCL_DEVICE_API _CCCL_FORCEINLINE AgentSegmentedRadixSort(OffsetT num_items, TempStorage& temp_storage, DecomposerT decomposer = {}) : num_items(num_items) , temp_storage(temp_storage.Alias()) , decomposer(decomposer) {} - _CCCL_DEVICE _CCCL_FORCEINLINE void ProcessSinglePass( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ProcessSinglePass( int begin_bit, int end_bit, const KeyT* d_keys_in, const ValueT* d_values_in, KeyT* d_keys_out, ValueT* d_values_out) { KeyT thread_keys[ITEMS_PER_THREAD]; @@ -171,7 +171,7 @@ struct AgentSegmentedRadixSort } } - _CCCL_DEVICE _CCCL_FORCEINLINE void ProcessIterative( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ProcessIterative( int current_bit, int pass_bits, const KeyT* d_keys_in, diff --git a/cub/cub/agent/agent_select_if.cuh b/cub/cub/agent/agent_select_if.cuh index eba2bbcc76a8..398263a07301 100644 --- a/cub/cub/agent/agent_select_if.cuh +++ b/cub/cub/agent/agent_select_if.cuh @@ -107,7 +107,7 @@ struct guarded_inequality_op template , int> = 0> - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE bool operator()(const T& a, const T& b, int idx) noexcept( + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE bool operator()(const T& a, const T& b, int idx) noexcept( ::cuda::std::__is_nothrow_callable_v) { if (idx < num_remaining) @@ -121,7 +121,7 @@ struct guarded_inequality_op template , int> = 0> - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE bool operator()(const T& a, const T& b, int idx) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE bool operator()(const T& a, const T& b, int idx) const noexcept(::cuda::std::__is_nothrow_callable_v) { if (idx < num_remaining) @@ -372,7 +372,7 @@ struct AgentSelectIf * @param streaming_context * Context for the current partition */ - _CCCL_DEVICE _CCCL_FORCEINLINE AgentSelectIf( + _CCCL_DEVICE_API _CCCL_FORCEINLINE AgentSelectIf( TempStorage& temp_storage, InputIteratorT d_in, FlagsInputIteratorT d_flags_in, @@ -395,7 +395,7 @@ struct AgentSelectIf // Utility methods for initializing the selections //--------------------------------------------------------------------- - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE InputIteratorT GetInputIterator() const + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE InputIteratorT GetInputIterator() const { if constexpr (::cuda::std::is_pointer_v) { @@ -411,7 +411,7 @@ struct AgentSelectIf * Load input items for the current tile. */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void LoadItems(OffsetT tile_offset, int num_tile_items, InputT (&items)[ITEMS_PER_THREAD]) { if (IsLastTile) @@ -429,7 +429,7 @@ struct AgentSelectIf * Load input items and initialize selection flags for the current tile. */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void PrepareTile( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void PrepareTile( OffsetT tile_offset, int num_tile_items, InputT (&items)[ITEMS_PER_THREAD], @@ -474,7 +474,7 @@ struct AgentSelectIf * Initialize selections (specialized for selection operator) */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void InitializeSelections( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InitializeSelections( OffsetT /*tile_offset*/, OffsetT num_tile_items, InputT (&items)[ITEMS_PER_THREAD], @@ -498,7 +498,7 @@ struct AgentSelectIf * Initialize selections (specialized for selection_op applied to d_flags_in) */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void InitializeSelections( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InitializeSelections( OffsetT tile_offset, OffsetT num_tile_items, InputT (& /*items*/)[ITEMS_PER_THREAD], @@ -538,7 +538,7 @@ struct AgentSelectIf * Initialize selections (specialized for valid flags) */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void InitializeSelections( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InitializeSelections( OffsetT tile_offset, OffsetT num_tile_items, InputT (& /*items*/)[ITEMS_PER_THREAD], @@ -570,7 +570,7 @@ struct AgentSelectIf * Initialize selections (specialized for discontinuity detection) */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void InitializeSelections( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InitializeSelections( OffsetT tile_offset, OffsetT num_tile_items, InputT (&items)[ITEMS_PER_THREAD], @@ -657,7 +657,7 @@ struct AgentSelectIf * Scatter flagged items to output offsets (specialized for direct scattering). */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterSelectedDirect( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterSelectedDirect( InputT (&items)[ITEMS_PER_THREAD], OffsetT (&selection_flags)[ITEMS_PER_THREAD], OffsetT (&selection_indices)[ITEMS_PER_THREAD], @@ -698,7 +698,7 @@ struct AgentSelectIf * Marker type indicating whether to keep rejected items in the second partition */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterSelectedTwoPhase( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterSelectedTwoPhase( InputT (&items)[ITEMS_PER_THREAD], OffsetT (&selection_flags)[ITEMS_PER_THREAD], OffsetT (&selection_indices)[ITEMS_PER_THREAD], @@ -747,7 +747,7 @@ struct AgentSelectIf * Total number of selections including this tile */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void Scatter( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Scatter( InputT (&items)[ITEMS_PER_THREAD], OffsetT (&selection_flags)[ITEMS_PER_THREAD], OffsetT (&selection_indices)[ITEMS_PER_THREAD], @@ -791,7 +791,7 @@ struct AgentSelectIf * Marker type indicating whether to keep rejected items in the second partition */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void Scatter( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Scatter( InputT (&items)[ITEMS_PER_THREAD], OffsetT (&selection_flags)[ITEMS_PER_THREAD], OffsetT (&selection_indices)[ITEMS_PER_THREAD], @@ -832,7 +832,7 @@ struct AgentSelectIf * distinct partitions. */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterPartitionsToGlobal( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterPartitionsToGlobal( int num_tile_items, int tile_num_rejections, OffsetT num_selections_prefix, @@ -873,7 +873,7 @@ struct AgentSelectIf * writtem from the iterators end backwards. */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterPartitionsToGlobal( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterPartitionsToGlobal( int num_tile_items, int tile_num_rejections, OffsetT num_selections_prefix, @@ -924,7 +924,7 @@ struct AgentSelectIf * @return The running count of selections (including this tile) */ template - _CCCL_DEVICE _CCCL_FORCEINLINE OffsetT + _CCCL_DEVICE_API _CCCL_FORCEINLINE OffsetT ConsumeFirstTile(int num_tile_items, OffsetT tile_offset, MemoryOrderedTileStateT& tile_state_wrapper) { InputT items[ITEMS_PER_THREAD]; @@ -991,7 +991,7 @@ struct AgentSelectIf * @return The running count of selections (including this tile) */ template - _CCCL_DEVICE _CCCL_FORCEINLINE OffsetT ConsumeSubsequentTile( + _CCCL_DEVICE_API _CCCL_FORCEINLINE OffsetT ConsumeSubsequentTile( int num_tile_items, int tile_idx, OffsetT tile_offset, MemoryOrderedTileStateT& tile_state_wrapper) { InputT items[ITEMS_PER_THREAD]; @@ -1059,7 +1059,7 @@ struct AgentSelectIf * all tile status updates and loads */ template - _CCCL_DEVICE _CCCL_FORCEINLINE OffsetT + _CCCL_DEVICE_API _CCCL_FORCEINLINE OffsetT ConsumeTile(int num_tile_items, int tile_idx, OffsetT tile_offset, MemoryOrderedTileStateT& tile_state_wrapper) { OffsetT num_selections; @@ -1091,7 +1091,7 @@ struct AgentSelectIf * Output iterator type for recording number of items selection_flags */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ConsumeRange(int num_tiles, ScanTileStateT& tile_state, NumSelectedIteratorT d_num_selected_out) { // Ensure consistent memory order across all tile status updates and loads diff --git a/cub/cub/agent/agent_sub_warp_merge_sort.cuh b/cub/cub/agent/agent_sub_warp_merge_sort.cuh index d8c285b2a514..ad7d6fe53ce1 100644 --- a/cub/cub/agent/agent_sub_warp_merge_sort.cuh +++ b/cub/cub/agent/agent_sub_warp_merge_sort.cuh @@ -103,7 +103,7 @@ class AgentSubWarpSort struct BinaryOpT { template - _CCCL_DEVICE bool operator()(T lhs, T rhs) const noexcept + _CCCL_DEVICE_API bool operator()(T lhs, T rhs) const noexcept { if constexpr (IsDescending) { @@ -117,7 +117,7 @@ class AgentSubWarpSort } #if _CCCL_HAS_NVFP16() - _CCCL_DEVICE bool operator()(__half lhs, __half rhs) const noexcept + _CCCL_DEVICE_API bool operator()(__half lhs, __half rhs) const noexcept { // Need to explicitly cast to float for SM <= 52. if constexpr (IsDescending) @@ -133,7 +133,7 @@ class AgentSubWarpSort #endif // _CCCL_HAS_NVFP16() #if _CCCL_HAS_NVBF16() - _CCCL_DEVICE bool operator()(__nv_bfloat16 lhs, __nv_bfloat16 rhs) const noexcept + _CCCL_DEVICE_API bool operator()(__nv_bfloat16 lhs, __nv_bfloat16 rhs) const noexcept { // Need to explicitly cast to float for SM < 80. if constexpr (IsDescending) @@ -152,7 +152,7 @@ class AgentSubWarpSort }; #if _CCCL_HAS_NVFP16() - _CCCL_DEVICE static bool equal(__half lhs, __half rhs) + _CCCL_DEVICE_API static bool equal(__half lhs, __half rhs) { // Need to explicitly cast to float for SM <= 52. NV_IF_ELSE_TARGET(NV_PROVIDES_SM_53, (return __heq(lhs, rhs);), (return __half2float(lhs) == __half2float(rhs);)); @@ -160,7 +160,7 @@ class AgentSubWarpSort #endif // _CCCL_HAS_NVFP16() #if _CCCL_HAS_NVBF16() - _CCCL_DEVICE static bool equal(__nv_bfloat16 lhs, __nv_bfloat16 rhs) + _CCCL_DEVICE_API static bool equal(__nv_bfloat16 lhs, __nv_bfloat16 rhs) { // Need to explicitly cast to float for SM < 80. NV_IF_ELSE_TARGET( @@ -169,7 +169,7 @@ class AgentSubWarpSort #endif // _CCCL_HAS_NVBF16() template - _CCCL_DEVICE static bool equal(T lhs, T rhs) + _CCCL_DEVICE_API static bool equal(T lhs, T rhs) { return lhs == rhs; } @@ -215,11 +215,11 @@ public: _TempStorage& storage; - _CCCL_DEVICE _CCCL_FORCEINLINE explicit AgentSubWarpSort(TempStorage& temp_storage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE explicit AgentSubWarpSort(TempStorage& temp_storage) : storage(temp_storage.Alias()) {} - _CCCL_DEVICE _CCCL_FORCEINLINE void ProcessSegment( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ProcessSegment( int segment_size, KeysLoadItT keys_input, KeyT* keys_output, ItemsLoadItT values_input, ValueT* values_output) { WarpMergeSortT warp_merge_sort(storage.sort); @@ -289,7 +289,7 @@ private: * Only the first thread of a virtual warp is used for soring. */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void ShortCircuit( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ShortCircuit( unsigned int linear_tid, OffsetT segment_size, KeysLoadItT keys_input, diff --git a/cub/cub/agent/agent_three_way_partition.cuh b/cub/cub/agent/agent_three_way_partition.cuh index a40c798ebff7..81af8d651c76 100644 --- a/cub/cub/agent/agent_three_way_partition.cuh +++ b/cub/cub/agent/agent_three_way_partition.cuh @@ -80,7 +80,7 @@ struct pair_pack_t { OffsetT x, y; - _CCCL_DEVICE pair_pack_t operator+(const pair_pack_t& other) const + _CCCL_DEVICE_API pair_pack_t operator+(const pair_pack_t& other) const { return {x + other.x, y + other.y}; } @@ -91,15 +91,15 @@ struct accumulator_pack_base_t { using pack_t = pair_pack_t; - _CCCL_DEVICE static pack_t pack(OffsetT f, OffsetT s) + _CCCL_DEVICE_API static pack_t pack(OffsetT f, OffsetT s) { return {f, s}; } - _CCCL_DEVICE static OffsetT first(pack_t packed) + _CCCL_DEVICE_API static OffsetT first(pack_t packed) { return packed.x; } - _CCCL_DEVICE static OffsetT second(pack_t packed) + _CCCL_DEVICE_API static OffsetT second(pack_t packed) { return packed.y; } @@ -110,17 +110,17 @@ struct accumulator_pack_base_t(f) << 32) | static_cast(s); } - _CCCL_DEVICE static OffsetT first(pack_t packed) + _CCCL_DEVICE_API static OffsetT first(pack_t packed) { return static_cast(packed >> 32); } - _CCCL_DEVICE static OffsetT second(pack_t packed) + _CCCL_DEVICE_API static OffsetT second(pack_t packed) { return static_cast(packed & 0xFFFFFFFF); } @@ -132,17 +132,17 @@ struct accumulator_pack_t : accumulator_pack_base_t using base = accumulator_pack_base_t; using typename base::pack_t; - _CCCL_DEVICE static void subtract(pack_t& packed, OffsetT val) + _CCCL_DEVICE_API static void subtract(pack_t& packed, OffsetT val) { packed = base::pack(base::first(packed) - val, base::second(packed) - val); } - _CCCL_DEVICE static OffsetT sum(pack_t& packed) + _CCCL_DEVICE_API static OffsetT sum(pack_t& packed) { return base::first(packed) + base::second(packed); } - _CCCL_DEVICE static pack_t zero() + _CCCL_DEVICE_API static pack_t zero() { return {}; } @@ -254,7 +254,7 @@ struct AgentThreeWayPartition //--------------------------------------------------------------------- // Constructor - _CCCL_DEVICE _CCCL_FORCEINLINE AgentThreeWayPartition( + _CCCL_DEVICE_API _CCCL_FORCEINLINE AgentThreeWayPartition( TempStorage& temp_storage, InputIteratorT d_in, FirstOutputIteratorT d_first_part_out, @@ -280,7 +280,7 @@ struct AgentThreeWayPartition //--------------------------------------------------------------------- template - _CCCL_DEVICE _CCCL_FORCEINLINE void Initialize( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Initialize( OffsetT num_tile_items, InputT (&items)[ITEMS_PER_THREAD], AccumPackT (&items_selection_flags)[ITEMS_PER_THREAD]) { for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM) @@ -298,7 +298,7 @@ struct AgentThreeWayPartition } template - _CCCL_DEVICE _CCCL_FORCEINLINE void Scatter( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Scatter( InputT (&items)[ITEMS_PER_THREAD], AccumPackT (&items_selection_flags)[ITEMS_PER_THREAD], AccumPackT (&items_selection_indices)[ITEMS_PER_THREAD], @@ -396,7 +396,7 @@ struct AgentThreeWayPartition * @param second_tile_state Global tile state descriptor */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ConsumeFirstTile(int num_tile_items, OffsetT tile_offset, ScanTileStateT& tile_state, AccumPackT& num_items_selected) { InputT items[ITEMS_PER_THREAD]; @@ -461,7 +461,7 @@ struct AgentThreeWayPartition * @param second_tile_state Global tile state descriptor */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void ConsumeSubsequentTile( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ConsumeSubsequentTile( int num_tile_items, int tile_idx, OffsetT tile_offset, ScanTileStateT& tile_state, AccumPackT& num_items_selected) { InputT items[ITEMS_PER_THREAD]; @@ -523,7 +523,7 @@ struct AgentThreeWayPartition * Process a tile of input */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ConsumeTile(int num_tile_items, int tile_idx, OffsetT tile_offset, ScanTileStateT& tile_state, AccumPackT& accum) { if (tile_idx == 0) @@ -555,7 +555,7 @@ struct AgentThreeWayPartition * Output total number selection_flags */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ConsumeRange(int num_tiles, ScanTileStateT& tile_state, NumSelectedIteratorT d_num_selected_out) { // Blocks are launched in increasing order, so just assign one tile per block diff --git a/cub/cub/agent/agent_topk.cuh b/cub/cub/agent/agent_topk.cuh index 01eb5edad927..8dfac1c87c49 100644 --- a/cub/cub/agent/agent_topk.cuh +++ b/cub/cub/agent/agent_topk.cuh @@ -231,7 +231,7 @@ struct AgentTopK //! @param identify_candidates_op //! Filter operator //! - _CCCL_DEVICE _CCCL_FORCEINLINE AgentTopK( + _CCCL_DEVICE_API _CCCL_FORCEINLINE AgentTopK( TempStorage& temp_storage, const KeyInputIteratorT d_keys_in, KeyOutputIteratorT d_keys_out, @@ -260,7 +260,7 @@ struct AgentTopK // Process a range of input data in tiles, calling f(key, index) for each element template - _CCCL_DEVICE _CCCL_FORCEINLINE void process_range(InputItT in, const OffsetT num_items, FuncT f) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void process_range(InputItT in, const OffsetT num_items, FuncT f) { key_in_t thread_data[items_per_thread]; @@ -314,7 +314,7 @@ struct AgentTopK } } - _CCCL_DEVICE _CCCL_FORCEINLINE void init_histograms(OffsetT* histogram) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void init_histograms(OffsetT* histogram) { // Initialize histogram bin counts to zeros int histo_offset = 0; @@ -332,7 +332,7 @@ struct AgentTopK } } - _CCCL_DEVICE _CCCL_FORCEINLINE void merge_histograms(OffsetT* global_histogram) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void merge_histograms(OffsetT* global_histogram) { int histo_offset = 0; @@ -354,7 +354,7 @@ struct AgentTopK } // Fused filtering of the current pass and building histogram for the next pass - _CCCL_DEVICE _CCCL_FORCEINLINE void filter_and_histogram( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void filter_and_histogram( key_in_t* in_buf, OffsetT* in_idx_buf, key_in_t* out_buf, @@ -485,7 +485,7 @@ struct AgentTopK } // Replace histogram with its own prefix sum - _CCCL_DEVICE _CCCL_FORCEINLINE void compute_bin_offsets(volatile OffsetT* histogram) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void compute_bin_offsets(volatile OffsetT* histogram) { OffsetT thread_data[bins_per_thread]{}; @@ -500,7 +500,7 @@ struct AgentTopK } // Identify the bucket that the k-th value falls into - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void choose_bucket(Counter* counter, const OutOffsetT k, const int pass) { // Initialize histogram bin counts to zeros @@ -543,7 +543,7 @@ struct AgentTopK // the histogram for the next pass. The caller-supplied counter_update_fn runs on thread 0 of the // last block to update pass-specific counter state. template - _CCCL_DEVICE _CCCL_FORCEINLINE void finalize_pass( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void finalize_pass( Counter* counter, OffsetT* histogram, OutOffsetT current_k, @@ -589,7 +589,7 @@ struct AgentTopK } } - _CCCL_DEVICE _CCCL_FORCEINLINE void invoke_last_filter( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void invoke_last_filter( key_in_t* in_buf, OffsetT* in_idx_buf, Counter* counter, OutOffsetT k, int pass) { const bool load_from_original_input = (pass <= 1) || counter->previous_len > buffer_length; @@ -647,7 +647,7 @@ struct AgentTopK } } - _CCCL_DEVICE _CCCL_FORCEINLINE void invoke_filter_and_histogram( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void invoke_filter_and_histogram( key_in_t* in_buf, OffsetT* in_idx_buf, key_in_t* out_buf, @@ -711,7 +711,7 @@ struct AgentTopK // Histogram-only pass: computes the histogram over the full input without filtering. // Used for the first radix pass before any candidates have been identified. - _CCCL_DEVICE _CCCL_FORCEINLINE void invoke_histogram_only( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void invoke_histogram_only( Counter* counter, OffsetT* histogram, int pass, bool is_last_pass) { // Initialize shared memory histogram diff --git a/cub/cub/agent/agent_unique_by_key.cuh b/cub/cub/agent/agent_unique_by_key.cuh index 601abc56c003..affb4658bb6a 100644 --- a/cub/cub/agent/agent_unique_by_key.cuh +++ b/cub/cub/agent/agent_unique_by_key.cuh @@ -219,7 +219,7 @@ struct AgentUniqueByKey //--------------------------------------------------------------------- // Constructor - _CCCL_DEVICE _CCCL_FORCEINLINE AgentUniqueByKey( + _CCCL_DEVICE_API _CCCL_FORCEINLINE AgentUniqueByKey( TempStorage& temp_storage_, WrappedKeyInputIteratorT d_keys_in_, WrappedValueInputIteratorT d_values_in_, @@ -245,11 +245,11 @@ struct AgentUniqueByKey struct ValueTagT {}; - _CCCL_DEVICE _CCCL_FORCEINLINE KeyExchangeT& GetShared(KeyTagT) + _CCCL_DEVICE_API _CCCL_FORCEINLINE KeyExchangeT& GetShared(KeyTagT) { return temp_storage.shared_keys.Alias(); } - _CCCL_DEVICE _CCCL_FORCEINLINE ValueExchangeT& GetShared(ValueTagT) + _CCCL_DEVICE_API _CCCL_FORCEINLINE ValueExchangeT& GetShared(ValueTagT) { return temp_storage.shared_values.Alias(); } @@ -258,7 +258,7 @@ struct AgentUniqueByKey // Scatter utility methods //--------------------------------------------------------------------- template - _CCCL_DEVICE _CCCL_FORCEINLINE void Scatter( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Scatter( Tag tag, OutputIt items_out, T (&items)[ITEMS_PER_THREAD], @@ -311,7 +311,7 @@ struct AgentUniqueByKey * @return The running count of selections (including this tile) */ template - _CCCL_DEVICE _CCCL_FORCEINLINE OffsetT + _CCCL_DEVICE_API _CCCL_FORCEINLINE OffsetT ConsumeFirstTile(int num_tile_items, OffsetT tile_offset, ScanTileStateT& tile_state) { KeyT keys[ITEMS_PER_THREAD]; @@ -429,7 +429,7 @@ struct AgentUniqueByKey * @return Returns the running count of selections (including this tile) */ template - _CCCL_DEVICE _CCCL_FORCEINLINE OffsetT + _CCCL_DEVICE_API _CCCL_FORCEINLINE OffsetT ConsumeSubsequentTile(int num_tile_items, int tile_idx, OffsetT tile_offset, ScanTileStateT& tile_state) { KeyT keys[ITEMS_PER_THREAD]; @@ -542,7 +542,7 @@ struct AgentUniqueByKey * Global tile state descriptor */ template - _CCCL_DEVICE _CCCL_FORCEINLINE OffsetT + _CCCL_DEVICE_API _CCCL_FORCEINLINE OffsetT ConsumeTile(int num_tile_items, int tile_idx, OffsetT tile_offset, ScanTileStateT& tile_state) { OffsetT num_selections; @@ -575,7 +575,7 @@ struct AgentUniqueByKey * */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ConsumeRange(int num_tiles, ScanTileStateT& tile_state, NumSelectedIteratorT d_num_selected_out) { // Blocks are launched in increasing order, so just assign one tile per block diff --git a/cub/cub/agent/single_pass_scan_operators.cuh b/cub/cub/agent/single_pass_scan_operators.cuh index 7a5261185e55..5d013fa4396d 100644 --- a/cub/cub/agent/single_pass_scan_operators.cuh +++ b/cub/cub/agent/single_pass_scan_operators.cuh @@ -70,12 +70,12 @@ struct BlockScanRunningPrefixOp T running_total; /// Constructor - _CCCL_DEVICE _CCCL_FORCEINLINE BlockScanRunningPrefixOp(ScanOpT op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockScanRunningPrefixOp(ScanOpT op) : op(op) {} /// Constructor - _CCCL_DEVICE _CCCL_FORCEINLINE BlockScanRunningPrefixOp(T starting_prefix, ScanOpT op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockScanRunningPrefixOp(T starting_prefix, ScanOpT op) : op(op) , running_total(starting_prefix) {} @@ -86,7 +86,7 @@ struct BlockScanRunningPrefixOp * @param block_aggregate * The aggregate sum of the BlockScan inputs */ - _CCCL_DEVICE _CCCL_FORCEINLINE T operator()(const T& block_aggregate) + _CCCL_DEVICE_API _CCCL_FORCEINLINE T operator()(const T& block_aggregate) { T retval = running_total; running_total = op(running_total, block_aggregate); @@ -125,7 +125,7 @@ _CCCL_IKET_CREATE_PUSH_POP_RANGE(Delay); namespace detail { template -_CCCL_DEVICE _CCCL_FORCEINLINE void delay() +_CCCL_DEVICE_API _CCCL_FORCEINLINE void delay() { _CCCL_IKET_RANGE_PUSH(Delay); NV_IF_TARGET(NV_PROVIDES_SM_70, ({ @@ -145,7 +145,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void delay() } template -_CCCL_DEVICE _CCCL_FORCEINLINE void delay(unsigned int ns) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void delay(unsigned int ns) { _CCCL_IKET_RANGE_PUSH(Delay); NV_IF_TARGET(NV_PROVIDES_SM_70, ({ @@ -165,14 +165,14 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void delay(unsigned int ns) } template -_CCCL_DEVICE _CCCL_FORCEINLINE void always_delay() +_CCCL_DEVICE_API _CCCL_FORCEINLINE void always_delay() { _CCCL_IKET_RANGE_PUSH(Delay); NV_IF_TARGET(NV_PROVIDES_SM_70, (__nanosleep(Delay);)); _CCCL_IKET_RANGE_POP(); } -_CCCL_DEVICE _CCCL_FORCEINLINE void always_delay([[maybe_unused]] unsigned int ns) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void always_delay([[maybe_unused]] unsigned int ns) { _CCCL_IKET_RANGE_PUSH(Delay); NV_IF_TARGET(NV_PROVIDES_SM_70, (__nanosleep(ns);)); @@ -180,24 +180,24 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void always_delay([[maybe_unused]] unsigned int n } template -_CCCL_DEVICE _CCCL_FORCEINLINE void delay_or_prevent_hoisting() +_CCCL_DEVICE_API _CCCL_FORCEINLINE void delay_or_prevent_hoisting() { NV_IF_ELSE_TARGET(NV_PROVIDES_SM_70, (_CUB::detail::delay();), (__threadfence_block();)); } template -_CCCL_DEVICE _CCCL_FORCEINLINE void delay_or_prevent_hoisting([[maybe_unused]] unsigned int ns) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void delay_or_prevent_hoisting([[maybe_unused]] unsigned int ns) { NV_IF_ELSE_TARGET(NV_PROVIDES_SM_70, (_CUB::detail::delay(ns);), (__threadfence_block();)); } template -_CCCL_DEVICE _CCCL_FORCEINLINE void always_delay_or_prevent_hoisting() +_CCCL_DEVICE_API _CCCL_FORCEINLINE void always_delay_or_prevent_hoisting() { NV_IF_ELSE_TARGET(NV_PROVIDES_SM_70, (_CUB::detail::always_delay(Delay);), (__threadfence_block();)); } -_CCCL_DEVICE _CCCL_FORCEINLINE void always_delay_or_prevent_hoisting([[maybe_unused]] unsigned int ns) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void always_delay_or_prevent_hoisting([[maybe_unused]] unsigned int ns) { NV_IF_ELSE_TARGET(NV_PROVIDES_SM_70, (_CUB::detail::always_delay(ns);), (__threadfence_block();)); } @@ -207,18 +207,18 @@ struct no_delay_constructor_t { struct delay_t { - _CCCL_DEVICE _CCCL_FORCEINLINE void operator()() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void operator()() { NV_IF_ELSE_TARGET(NV_PROVIDES_SM_70, (), (__threadfence_block();)); } }; - _CCCL_DEVICE _CCCL_FORCEINLINE no_delay_constructor_t(unsigned int /* seed */) + _CCCL_DEVICE_API _CCCL_FORCEINLINE no_delay_constructor_t(unsigned int /* seed */) { _CUB::detail::delay(); } - _CCCL_DEVICE _CCCL_FORCEINLINE delay_t operator()() + _CCCL_DEVICE_API _CCCL_FORCEINLINE delay_t operator()() { return {}; } @@ -229,7 +229,7 @@ struct reduce_by_key_delay_constructor_t { struct delay_t { - _CCCL_DEVICE _CCCL_FORCEINLINE void operator()() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void operator()() { NV_DISPATCH_TARGET( NV_IS_EXACTLY_SM_80, @@ -241,12 +241,12 @@ struct reduce_by_key_delay_constructor_t } }; - _CCCL_DEVICE _CCCL_FORCEINLINE reduce_by_key_delay_constructor_t(unsigned int /* seed */) + _CCCL_DEVICE_API _CCCL_FORCEINLINE reduce_by_key_delay_constructor_t(unsigned int /* seed */) { _CUB::detail::delay(); } - _CCCL_DEVICE _CCCL_FORCEINLINE delay_t operator()() + _CCCL_DEVICE_API _CCCL_FORCEINLINE delay_t operator()() { return {}; } @@ -257,18 +257,18 @@ struct fixed_delay_constructor_t { struct delay_t { - _CCCL_DEVICE _CCCL_FORCEINLINE void operator()() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void operator()() { _CUB::detail::delay_or_prevent_hoisting(); } }; - _CCCL_DEVICE _CCCL_FORCEINLINE fixed_delay_constructor_t(unsigned int /* seed */) + _CCCL_DEVICE_API _CCCL_FORCEINLINE fixed_delay_constructor_t(unsigned int /* seed */) { _CUB::detail::delay(); } - _CCCL_DEVICE _CCCL_FORCEINLINE delay_t operator()() + _CCCL_DEVICE_API _CCCL_FORCEINLINE delay_t operator()() { return {}; } @@ -281,19 +281,19 @@ struct exponential_backoff_constructor_t { unsigned int delay; - _CCCL_DEVICE _CCCL_FORCEINLINE void operator()() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void operator()() { _CUB::detail::always_delay_or_prevent_hoisting(delay); delay <<= 1; } }; - _CCCL_DEVICE _CCCL_FORCEINLINE exponential_backoff_constructor_t(unsigned int /* seed */) + _CCCL_DEVICE_API _CCCL_FORCEINLINE exponential_backoff_constructor_t(unsigned int /* seed */) { _CUB::detail::always_delay(); } - _CCCL_DEVICE _CCCL_FORCEINLINE delay_t operator()() + _CCCL_DEVICE_API _CCCL_FORCEINLINE delay_t operator()() { return {InitialDelay}; } @@ -311,12 +311,12 @@ struct exponential_backoff_jitter_constructor_t unsigned int max_delay; unsigned int& seed; - _CCCL_DEVICE _CCCL_FORCEINLINE unsigned int next(unsigned int min, unsigned int max) + _CCCL_DEVICE_API _CCCL_FORCEINLINE unsigned int next(unsigned int min, unsigned int max) { return ((seed = ((a * seed) + c) % m) % (max + 1 - min)) + min; } - _CCCL_DEVICE _CCCL_FORCEINLINE void operator()() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void operator()() { _CUB::detail::always_delay_or_prevent_hoisting(next(0, max_delay)); max_delay <<= 1; @@ -325,13 +325,13 @@ struct exponential_backoff_jitter_constructor_t unsigned int seed; - _CCCL_DEVICE _CCCL_FORCEINLINE exponential_backoff_jitter_constructor_t(unsigned int seed) + _CCCL_DEVICE_API _CCCL_FORCEINLINE exponential_backoff_jitter_constructor_t(unsigned int seed) : seed(seed) { _CUB::detail::always_delay(); } - _CCCL_DEVICE _CCCL_FORCEINLINE delay_t operator()() + _CCCL_DEVICE_API _CCCL_FORCEINLINE delay_t operator()() { return {InitialDelay, seed}; } @@ -349,12 +349,12 @@ struct exponential_backoff_jitter_window_constructor_t unsigned int max_delay; unsigned int& seed; - _CCCL_DEVICE _CCCL_FORCEINLINE unsigned int next(unsigned int min, unsigned int max) + _CCCL_DEVICE_API _CCCL_FORCEINLINE unsigned int next(unsigned int min, unsigned int max) { return ((seed = ((a * seed) + c) % m) % (max + 1 - min)) + min; } - _CCCL_DEVICE _CCCL_FORCEINLINE void operator()() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void operator()() { unsigned int next_max_delay = max_delay << 1; _CUB::detail::always_delay_or_prevent_hoisting(next(max_delay, next_max_delay)); @@ -363,13 +363,13 @@ struct exponential_backoff_jitter_window_constructor_t }; unsigned int seed; - _CCCL_DEVICE _CCCL_FORCEINLINE exponential_backoff_jitter_window_constructor_t(unsigned int seed) + _CCCL_DEVICE_API _CCCL_FORCEINLINE exponential_backoff_jitter_window_constructor_t(unsigned int seed) : seed(seed) { _CUB::detail::always_delay(); } - _CCCL_DEVICE _CCCL_FORCEINLINE delay_t operator()() + _CCCL_DEVICE_API _CCCL_FORCEINLINE delay_t operator()() { return {InitialDelay, seed}; } @@ -387,12 +387,12 @@ struct exponential_backon_jitter_window_constructor_t unsigned int max_delay; unsigned int& seed; - _CCCL_DEVICE _CCCL_FORCEINLINE unsigned int next(unsigned int min, unsigned int max) + _CCCL_DEVICE_API _CCCL_FORCEINLINE unsigned int next(unsigned int min, unsigned int max) { return ((seed = ((a * seed) + c) % m) % (max + 1 - min)) + min; } - _CCCL_DEVICE _CCCL_FORCEINLINE void operator()() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void operator()() { const int prev_delay = max_delay >> 1; _CUB::detail::always_delay_or_prevent_hoisting(next(prev_delay, max_delay)); @@ -403,13 +403,13 @@ struct exponential_backon_jitter_window_constructor_t unsigned int seed; unsigned int max_delay = InitialDelay; - _CCCL_DEVICE _CCCL_FORCEINLINE exponential_backon_jitter_window_constructor_t(unsigned int seed) + _CCCL_DEVICE_API _CCCL_FORCEINLINE exponential_backon_jitter_window_constructor_t(unsigned int seed) : seed(seed) { _CUB::detail::always_delay(); } - _CCCL_DEVICE _CCCL_FORCEINLINE delay_t operator()() + _CCCL_DEVICE_API _CCCL_FORCEINLINE delay_t operator()() { max_delay >>= 1; return {max_delay, seed}; @@ -428,12 +428,12 @@ struct exponential_backon_jitter_constructor_t unsigned int max_delay; unsigned int& seed; - _CCCL_DEVICE _CCCL_FORCEINLINE unsigned int next(unsigned int min, unsigned int max) + _CCCL_DEVICE_API _CCCL_FORCEINLINE unsigned int next(unsigned int min, unsigned int max) { return ((seed = ((a * seed) + c) % m) % (max + 1 - min)) + min; } - _CCCL_DEVICE _CCCL_FORCEINLINE void operator()() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void operator()() { _CUB::detail::always_delay_or_prevent_hoisting(next(0, max_delay)); max_delay >>= 1; @@ -443,13 +443,13 @@ struct exponential_backon_jitter_constructor_t unsigned int seed; unsigned int max_delay = InitialDelay; - _CCCL_DEVICE _CCCL_FORCEINLINE exponential_backon_jitter_constructor_t(unsigned int seed) + _CCCL_DEVICE_API _CCCL_FORCEINLINE exponential_backon_jitter_constructor_t(unsigned int seed) : seed(seed) { _CUB::detail::always_delay(); } - _CCCL_DEVICE _CCCL_FORCEINLINE delay_t operator()() + _CCCL_DEVICE_API _CCCL_FORCEINLINE delay_t operator()() { max_delay >>= 1; return {max_delay, seed}; @@ -463,7 +463,7 @@ struct exponential_backon_constructor_t { unsigned int delay; - _CCCL_DEVICE _CCCL_FORCEINLINE void operator()() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void operator()() { _CUB::detail::always_delay_or_prevent_hoisting(delay); delay >>= 1; @@ -472,12 +472,12 @@ struct exponential_backon_constructor_t unsigned int max_delay = InitialDelay; - _CCCL_DEVICE _CCCL_FORCEINLINE exponential_backon_constructor_t(unsigned int /* seed */) + _CCCL_DEVICE_API _CCCL_FORCEINLINE exponential_backon_constructor_t(unsigned int /* seed */) { _CUB::detail::always_delay(); } - _CCCL_DEVICE _CCCL_FORCEINLINE delay_t operator()() + _CCCL_DEVICE_API _CCCL_FORCEINLINE delay_t operator()() { max_delay >>= 1; return {max_delay}; @@ -521,7 +521,7 @@ struct tile_state_with_memory_order /** * Update the specified tile's inclusive value and corresponding status */ - _CCCL_DEVICE _CCCL_FORCEINLINE void SetInclusive(int tile_idx, T tile_inclusive) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SetInclusive(int tile_idx, T tile_inclusive) { tile_state.template SetInclusive(tile_idx, tile_inclusive); } @@ -529,7 +529,7 @@ struct tile_state_with_memory_order /** * Update the specified tile's partial value and corresponding status */ - _CCCL_DEVICE _CCCL_FORCEINLINE void SetPartial(int tile_idx, T tile_partial) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SetPartial(int tile_idx, T tile_partial) { tile_state.template SetPartial(tile_idx, tile_partial); } @@ -538,23 +538,23 @@ struct tile_state_with_memory_order * Wait for the corresponding tile to become non-invalid */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void WaitForValid(int tile_idx, StatusWord& status, T& value, DelayT delay = {}) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void WaitForValid(int tile_idx, StatusWord& status, T& value, DelayT delay = {}) { tile_state.template WaitForValid(tile_idx, status, value, delay); } - _CCCL_DEVICE _CCCL_FORCEINLINE T LoadValid(int tile_idx) + _CCCL_DEVICE_API _CCCL_FORCEINLINE T LoadValid(int tile_idx) { return tile_state.template LoadValid(tile_idx); } }; -_CCCL_HOST_DEVICE _CCCL_FORCEINLINE constexpr size_t num_tiles_to_num_tile_states(size_t num_tiles) +_CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE constexpr size_t num_tiles_to_num_tile_states(size_t num_tiles) { return warp_threads + num_tiles; } -_CCCL_HOST_DEVICE _CCCL_FORCEINLINE cudaError_t tile_state_allocation_size( +_CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE cudaError_t tile_state_allocation_size( size_t& temp_storage_bytes, size_t bytes_per_description, size_t bytes_per_payload, size_t num_tiles) { const size_t num_tile_states = _CUB::detail::num_tiles_to_num_tile_states(num_tiles); @@ -571,7 +571,7 @@ _CCCL_HOST_DEVICE _CCCL_FORCEINLINE cudaError_t tile_state_allocation_size( return _CUB::detail::alias_temporaries(nullptr, temp_storage_bytes, allocations, allocation_sizes); }; -_CCCL_HOST_DEVICE _CCCL_FORCEINLINE cudaError_t tile_state_init( +_CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE cudaError_t tile_state_init( size_t bytes_per_description, size_t bytes_per_payload, size_t num_tiles, @@ -652,7 +652,7 @@ struct ScanTileState static constexpr size_t payload_bytes_per_tile = 0; /// Constructor - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE ScanTileState() + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE ScanTileState() : d_tile_descriptors(nullptr) {} @@ -662,7 +662,7 @@ struct ScanTileState * @param[in] d_temp_storage * Device-accessible allocation of temporary storage. */ - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE cudaError_t + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE cudaError_t Init(int /*num_tiles*/, void* d_temp_storage, size_t /*temp_storage_bytes*/) { d_tile_descriptors = reinterpret_cast(d_temp_storage); @@ -678,7 +678,7 @@ struct ScanTileState * @param[out] temp_storage_bytes * Size in bytes of @p d_temp_storage allocation */ - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE static constexpr cudaError_t + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE static constexpr cudaError_t AllocationSize(int num_tiles, size_t& temp_storage_bytes) { return _CUB::detail::tile_state_allocation_size( @@ -688,7 +688,7 @@ struct ScanTileState /** * Initialize (from device) */ - _CCCL_DEVICE _CCCL_FORCEINLINE void InitializeStatus(int num_tiles) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InitializeStatus(int num_tiles) { const int tile_idx = static_cast((blockIdx.x * blockDim.x) + threadIdx.x); @@ -712,28 +712,28 @@ struct ScanTileState private: template - _CCCL_DEVICE _CCCL_FORCEINLINE ::cuda::std::enable_if_t<(Order == MemoryOrder::relaxed), void> + _CCCL_DEVICE_API _CCCL_FORCEINLINE ::cuda::std::enable_if_t<(Order == MemoryOrder::relaxed), void> StoreStatus(TxnWord* ptr, TxnWord alias) { _CUB::detail::store_relaxed(ptr, alias); } template - _CCCL_DEVICE _CCCL_FORCEINLINE ::cuda::std::enable_if_t<(Order == MemoryOrder::acquire_release), void> + _CCCL_DEVICE_API _CCCL_FORCEINLINE ::cuda::std::enable_if_t<(Order == MemoryOrder::acquire_release), void> StoreStatus(TxnWord* ptr, TxnWord alias) { _CUB::detail::store_release(ptr, alias); } template - _CCCL_DEVICE _CCCL_FORCEINLINE ::cuda::std::enable_if_t<(Order == MemoryOrder::relaxed), TxnWord> + _CCCL_DEVICE_API _CCCL_FORCEINLINE ::cuda::std::enable_if_t<(Order == MemoryOrder::relaxed), TxnWord> LoadStatus(TxnWord* ptr) { return _CUB::detail::load_relaxed(ptr); } template - _CCCL_DEVICE _CCCL_FORCEINLINE ::cuda::std::enable_if_t<(Order == MemoryOrder::acquire_release), TxnWord> + _CCCL_DEVICE_API _CCCL_FORCEINLINE ::cuda::std::enable_if_t<(Order == MemoryOrder::acquire_release), TxnWord> LoadStatus(TxnWord* ptr) { // For pre-volta we hoist the memory barrier to outside the loop, i.e., after reading a valid state @@ -743,7 +743,7 @@ private: public: template - _CCCL_DEVICE _CCCL_FORCEINLINE void SetInclusive(int tile_idx, T tile_inclusive) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SetInclusive(int tile_idx, T tile_inclusive) { const TileDescriptor tile_descriptor{SCAN_TILE_INCLUSIVE, tile_inclusive}; @@ -754,7 +754,7 @@ public: } template - _CCCL_DEVICE _CCCL_FORCEINLINE void SetPartial(int tile_idx, T tile_partial) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SetPartial(int tile_idx, T tile_partial) { const TileDescriptor tile_descriptor{SCAN_TILE_PARTIAL, tile_partial}; @@ -777,7 +777,7 @@ public: * Wait for the corresponding tile to become non-invalid */ template , MemoryOrder Order = MemoryOrder::relaxed> - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void WaitForValid(int tile_idx, StatusWord& status, T& value, DelayT delay_or_prevent_hoisting = {}) { // The descriptor is loaded before any field is read. @@ -822,7 +822,7 @@ public: * Loads and returns the tile's value. The returned value is undefined if either (a) the tile's status is invalid or * (b) there is no memory fence between reading a non-invalid status and the call to LoadValid. */ - _CCCL_DEVICE _CCCL_FORCEINLINE T LoadValid(int tile_idx) + _CCCL_DEVICE_API _CCCL_FORCEINLINE T LoadValid(int tile_idx) { TxnWord alias = d_tile_descriptors[TILE_STATUS_PADDING + tile_idx]; const TileDescriptor tile_descriptor = reinterpret_cast(alias); @@ -854,7 +854,7 @@ struct ScanTileState static constexpr size_t payload_bytes_per_tile = sizeof(Uninitialized); /// Constructor - _CCCL_FORCEINLINE ScanTileState() = default; + _CCCL_HIDE_FROM_ABI ScanTileState() = default; /** * @brief Initializer @@ -871,7 +871,8 @@ struct ScanTileState * Size in bytes of @p d_temp_storage allocation */ /// Initializer - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE cudaError_t Init(int num_tiles, void* d_temp_storage, size_t temp_storage_bytes) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE cudaError_t + Init(int num_tiles, void* d_temp_storage, size_t temp_storage_bytes) { cudaError_t error = cudaSuccess; do @@ -901,7 +902,7 @@ struct ScanTileState * @param[out] temp_storage_bytes * Size in bytes of @p d_temp_storage allocation */ - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE static constexpr cudaError_t + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE static constexpr cudaError_t AllocationSize(int num_tiles, size_t& temp_storage_bytes) { return _CUB::detail::tile_state_allocation_size( @@ -910,7 +911,7 @@ struct ScanTileState /** * Initialize (from device) */ - _CCCL_DEVICE _CCCL_FORCEINLINE void InitializeStatus(int num_tiles) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InitializeStatus(int num_tiles) { const int tile_idx = static_cast((blockIdx.x * blockDim.x) + threadIdx.x); if (tile_idx < num_tiles) @@ -930,7 +931,7 @@ struct ScanTileState * Update the specified tile's inclusive value and corresponding status */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void SetInclusive(int tile_idx, T tile_inclusive) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SetInclusive(int tile_idx, T tile_inclusive) { // Update tile inclusive value _CUB::ThreadStore(d_tile_inclusive + TILE_STATUS_PADDING + tile_idx, tile_inclusive); @@ -941,7 +942,7 @@ struct ScanTileState * Update the specified tile's partial value and corresponding status */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void SetPartial(int tile_idx, T tile_partial) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SetPartial(int tile_idx, T tile_partial) { // Update tile partial value _CUB::ThreadStore(d_tile_partial + TILE_STATUS_PADDING + tile_idx, tile_partial); @@ -958,7 +959,7 @@ struct ScanTileState * Wait for the corresponding tile to become non-invalid */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void WaitForValid(int tile_idx, StatusWord& status, T& value, DelayT delay = {}) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void WaitForValid(int tile_idx, StatusWord& status, T& value, DelayT delay = {}) { do { @@ -985,7 +986,7 @@ struct ScanTileState * Loads and returns the tile's value. The returned value is undefined if either (a) the tile's status is invalid or * (b) there is no memory fence between reading a non-invalid status and the call to LoadValid. */ - _CCCL_DEVICE _CCCL_FORCEINLINE T LoadValid(int tile_idx) + _CCCL_DEVICE_API _CCCL_FORCEINLINE T LoadValid(int tile_idx) { return d_tile_inclusive[TILE_STATUS_PADDING + tile_idx]; } @@ -1016,7 +1017,7 @@ struct ReduceByKeyScanTileState : ScanTileState>; /// Constructor - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE ReduceByKeyScanTileState() + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE ReduceByKeyScanTileState() : SuperClass() {} }; @@ -1075,7 +1076,7 @@ struct ReduceByKeyScanTileState TxnWord* d_tile_descriptors; /// Constructor - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE ReduceByKeyScanTileState() + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE ReduceByKeyScanTileState() : d_tile_descriptors(nullptr) {} @@ -1092,7 +1093,7 @@ struct ReduceByKeyScanTileState * @param[in] temp_storage_bytes * Size in bytes of @p d_temp_storage allocation */ - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE cudaError_t + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE cudaError_t Init(int /*num_tiles*/, void* d_temp_storage, size_t /*temp_storage_bytes*/) { d_tile_descriptors = reinterpret_cast(d_temp_storage); @@ -1108,7 +1109,7 @@ struct ReduceByKeyScanTileState * @param[out] temp_storage_bytes * Size in bytes of @p d_temp_storage allocation */ - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE static cudaError_t AllocationSize(int num_tiles, size_t& temp_storage_bytes) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE static cudaError_t AllocationSize(int num_tiles, size_t& temp_storage_bytes) { // bytes needed for tile status descriptors temp_storage_bytes = (num_tiles + TILE_STATUS_PADDING) * sizeof(TxnWord); @@ -1118,7 +1119,7 @@ struct ReduceByKeyScanTileState /** * Initialize (from device) */ - _CCCL_DEVICE _CCCL_FORCEINLINE void InitializeStatus(int num_tiles) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InitializeStatus(int num_tiles) { const int tile_idx = static_cast((blockIdx.x * blockDim.x) + threadIdx.x); TxnWord val = TxnWord(); @@ -1142,7 +1143,7 @@ struct ReduceByKeyScanTileState /** * Update the specified tile's inclusive value and corresponding status */ - _CCCL_DEVICE _CCCL_FORCEINLINE void SetInclusive(int tile_idx, KeyValuePairT tile_inclusive) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SetInclusive(int tile_idx, KeyValuePairT tile_inclusive) { TileDescriptor tile_descriptor; tile_descriptor.status = SCAN_TILE_INCLUSIVE; @@ -1155,7 +1156,7 @@ struct ReduceByKeyScanTileState _CUB::detail::store_relaxed(d_tile_descriptors + TILE_STATUS_PADDING + tile_idx, alias); } - _CCCL_DEVICE _CCCL_FORCEINLINE void SetPartial(int tile_idx, KeyValuePairT tile_partial) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SetPartial(int tile_idx, KeyValuePairT tile_partial) { TileDescriptor tile_descriptor; tile_descriptor.status = SCAN_TILE_PARTIAL; @@ -1172,7 +1173,7 @@ struct ReduceByKeyScanTileState * Wait for the corresponding tile to become non-invalid */ template ::delay_t> - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void WaitForValid(int tile_idx, StatusWord& status, KeyValuePairT& value, DelayT delay_or_prevent_hoisting = {}) { // TxnWord alias = ThreadLoad(d_tile_descriptors + TILE_STATUS_PADDING + @@ -1267,7 +1268,7 @@ struct TilePrefixCallbackOp // Constructs prefix functor for a given tile index. // Precondition: thread blocks processing all of the predecessor tiles were scheduled. - _CCCL_DEVICE _CCCL_FORCEINLINE + _CCCL_DEVICE_API _CCCL_FORCEINLINE TilePrefixCallbackOp(ScanTileStateT& tile_status, TempStorage& temp_storage, ScanOpT scan_op, int tile_idx) : temp_storage(temp_storage.Alias()) , tile_status(tile_status) @@ -1277,7 +1278,7 @@ struct TilePrefixCallbackOp // Computes the tile index and constructs prefix functor with it. // Precondition: thread block per tile assignment. - _CCCL_DEVICE _CCCL_FORCEINLINE + _CCCL_DEVICE_API _CCCL_FORCEINLINE TilePrefixCallbackOp(ScanTileStateT& tile_status, TempStorage& temp_storage, ScanOpT scan_op) : TilePrefixCallbackOp(tile_status, temp_storage, scan_op, blockIdx.x) {} @@ -1295,7 +1296,7 @@ struct TilePrefixCallbackOp * Relevant partial reduction from this window of preceding tiles */ template > - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ProcessWindow(int predecessor_idx, StatusWord& predecessor_status, T& window_aggregate, DelayT delay = {}) { _CCCL_IKET_RANGE_PUSH(ProcessWindow); @@ -1320,7 +1321,7 @@ struct TilePrefixCallbackOp private: // Classic decoupled-lookback prefix computation. - _CCCL_DEVICE _CCCL_FORCEINLINE T lookback(T block_aggregate) + _CCCL_DEVICE_API _CCCL_FORCEINLINE T lookback(T block_aggregate) { _CCCL_IKET_RANGE_PUSH(Lookback); _CCCL_IKET_RANGE_PUSH(SetPartial); @@ -1375,7 +1376,7 @@ private: } // Run-to-run-deterministic K=1 32-batched lookback. Only anchor tiles publish INCLUSIVE. - _CCCL_DEVICE _CCCL_FORCEINLINE T lookback_stable_reduction_order(T block_aggregate) + _CCCL_DEVICE_API _CCCL_FORCEINLINE T lookback_stable_reduction_order(T block_aggregate) { _CCCL_IKET_RANGE_PUSH(Lookback); _CCCL_IKET_RANGE_PUSH(SetPartial); @@ -1433,7 +1434,7 @@ private: public: // BlockScan prefix callback functor. - _CCCL_DEVICE _CCCL_FORCEINLINE T operator()(T block_aggregate) + _CCCL_DEVICE_API _CCCL_FORCEINLINE T operator()(T block_aggregate) { if constexpr (StableReductionOrder) { @@ -1446,24 +1447,24 @@ public: } // Get the exclusive prefix stored in temporary storage - _CCCL_DEVICE _CCCL_FORCEINLINE T GetExclusivePrefix() + _CCCL_DEVICE_API _CCCL_FORCEINLINE T GetExclusivePrefix() { return temp_storage.exclusive_prefix; } // Get the inclusive prefix stored in temporary storage - _CCCL_DEVICE _CCCL_FORCEINLINE T GetInclusivePrefix() + _CCCL_DEVICE_API _CCCL_FORCEINLINE T GetInclusivePrefix() { return temp_storage.inclusive_prefix; } // Get the block aggregate stored in temporary storage - _CCCL_DEVICE _CCCL_FORCEINLINE T GetBlockAggregate() + _CCCL_DEVICE_API _CCCL_FORCEINLINE T GetBlockAggregate() { return temp_storage.block_aggregate; } - _CCCL_DEVICE _CCCL_FORCEINLINE int GetTileIdx() const + _CCCL_DEVICE_API _CCCL_FORCEINLINE int GetTileIdx() const { return tile_idx; } diff --git a/cub/cub/block/block_adjacent_difference.cuh b/cub/cub/block/block_adjacent_difference.cuh index 690f36d31eee..38487a3d7b80 100644 --- a/cub/cub/block/block_adjacent_difference.cuh +++ b/cub/cub/block/block_adjacent_difference.cuh @@ -112,7 +112,7 @@ class BlockAdjacentDifference }; /// Internal storage allocator - _CCCL_DEVICE _CCCL_FORCEINLINE _TempStorage& PrivateStorage() + _CCCL_DEVICE_API _CCCL_FORCEINLINE _TempStorage& PrivateStorage() { __shared__ _TempStorage private_storage; return private_storage; @@ -123,7 +123,7 @@ class BlockAdjacentDifference struct ApplyOp { // Apply flag operator - static _CCCL_DEVICE _CCCL_FORCEINLINE T FlagT(FlagOp flag_op, const T& a, const T& b, int idx) + _CCCL_DEVICE_API static _CCCL_FORCEINLINE T FlagT(FlagOp flag_op, const T& a, const T& b, int idx) { return flag_op(b, a, idx); } @@ -134,7 +134,7 @@ class BlockAdjacentDifference struct ApplyOp { // Apply flag operator - static _CCCL_DEVICE _CCCL_FORCEINLINE T FlagT(FlagOp flag_op, const T& a, const T& b, int /*idx*/) + _CCCL_DEVICE_API static _CCCL_FORCEINLINE T FlagT(FlagOp flag_op, const T& a, const T& b, int /*idx*/) { return flag_op(b, a); } @@ -152,7 +152,7 @@ class BlockAdjacentDifference * @param[in] flag_op Binary boolean flag predicate */ template - static _CCCL_DEVICE _CCCL_FORCEINLINE void FlagHeads( + _CCCL_DEVICE_API static _CCCL_FORCEINLINE void FlagHeads( int linear_tid, FlagT (&flags)[ItemsPerThread], T (&input)[ItemsPerThread], @@ -175,7 +175,7 @@ class BlockAdjacentDifference * @param[in] flag_op Binary boolean flag predicate */ template - static _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API static _CCCL_FORCEINLINE void FlagTails(int linear_tid, FlagT (&flags)[ItemsPerThread], T (&input)[ItemsPerThread], FlagOp flag_op) { _CCCL_PRAGMA_UNROLL_FULL() @@ -205,7 +205,7 @@ public: //! .. versionadded:: 2.2.0 //! First appears in CUDA Toolkit 12.3. //! @endrst - _CCCL_DEVICE _CCCL_FORCEINLINE BlockAdjacentDifference() + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockAdjacentDifference() : temp_storage(PrivateStorage()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} @@ -218,7 +218,7 @@ public: //! @endrst //! //! @param[in] temp_storage Reference to memory allocation having layout type TempStorage - _CCCL_DEVICE _CCCL_FORCEINLINE BlockAdjacentDifference(TempStorage& temp_storage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockAdjacentDifference(TempStorage& temp_storage) : temp_storage(temp_storage.Alias()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} @@ -290,7 +290,7 @@ public: //! @param[in] difference_op //! Binary difference operator template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SubtractLeft(T (&input)[ItemsPerThread], OutputType (&output)[ItemsPerThread], DifferenceOpT difference_op) { // Share last item @@ -389,7 +389,7 @@ public: //! (*input*\ :sub:`0` from *thread*\ :sub:`0`). //! @endrst template - _CCCL_DEVICE _CCCL_FORCEINLINE void SubtractLeft( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SubtractLeft( T (&input)[ItemsPerThread], OutputT (&output)[ItemsPerThread], DifferenceOpT difference_op, T tile_predecessor_item) { // Share last item @@ -483,7 +483,7 @@ public: //! @param[in] valid_items //! Number of valid items in thread block template - _CCCL_DEVICE _CCCL_FORCEINLINE void SubtractLeftPartialTile( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SubtractLeftPartialTile( T (&input)[ItemsPerThread], OutputType (&output)[ItemsPerThread], DifferenceOpT difference_op, int valid_items) { // Share last item @@ -605,7 +605,7 @@ public: //! (*input*\ :sub:`0` from *thread*\ :sub:`0`). //! @endrst template - _CCCL_DEVICE _CCCL_FORCEINLINE void SubtractLeftPartialTile( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SubtractLeftPartialTile( T (&input)[ItemsPerThread], OutputType (&output)[ItemsPerThread], DifferenceOpT difference_op, @@ -726,7 +726,7 @@ public: //! @param[in] difference_op //! Binary difference operator template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SubtractRight(T (&input)[ItemsPerThread], OutputT (&output)[ItemsPerThread], DifferenceOpT difference_op) { // Share first item @@ -826,7 +826,7 @@ public: //! (*input*\ :sub:`ItemsPerThread` from *thread*\ :sub:`BLOCK_THREADS`). //! @endrst template - _CCCL_DEVICE _CCCL_FORCEINLINE void SubtractRight( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SubtractRight( T (&input)[ItemsPerThread], OutputT (&output)[ItemsPerThread], DifferenceOpT difference_op, T tile_successor_item) { // Share first item @@ -918,7 +918,7 @@ public: //! @param[in] valid_items //! Number of valid items in thread block template - _CCCL_DEVICE _CCCL_FORCEINLINE void SubtractRightPartialTile( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SubtractRightPartialTile( T (&input)[ItemsPerThread], OutputT (&output)[ItemsPerThread], DifferenceOpT difference_op, int valid_items) { // Share first item diff --git a/cub/cub/block/block_discontinuity.cuh b/cub/cub/block/block_discontinuity.cuh index 6f6f8e15c48c..27ea535bc117 100644 --- a/cub/cub/block/block_discontinuity.cuh +++ b/cub/cub/block/block_discontinuity.cuh @@ -114,7 +114,7 @@ class BlockDiscontinuity }; /// Internal storage allocator - _CCCL_DEVICE _CCCL_FORCEINLINE _TempStorage& PrivateStorage() + _CCCL_DEVICE_API _CCCL_FORCEINLINE _TempStorage& PrivateStorage() { __shared__ _TempStorage private_storage; return private_storage; @@ -125,7 +125,7 @@ class BlockDiscontinuity struct ApplyOp { // Apply flag operator - static _CCCL_DEVICE _CCCL_FORCEINLINE bool FlagT(FlagOp flag_op, const T& a, const T& b, int idx) + _CCCL_DEVICE_API static _CCCL_FORCEINLINE bool FlagT(FlagOp flag_op, const T& a, const T& b, int idx) { return flag_op(a, b, idx); } @@ -136,7 +136,7 @@ class BlockDiscontinuity struct ApplyOp { // Apply flag operator - static _CCCL_DEVICE _CCCL_FORCEINLINE bool FlagT(FlagOp flag_op, const T& a, const T& b, int /*idx*/) + _CCCL_DEVICE_API static _CCCL_FORCEINLINE bool FlagT(FlagOp flag_op, const T& a, const T& b, int /*idx*/) { return flag_op(a, b); } @@ -161,7 +161,7 @@ class BlockDiscontinuity * Binary boolean flag predicate */ template - static _CCCL_DEVICE _CCCL_FORCEINLINE void FlagHeads( + _CCCL_DEVICE_API static _CCCL_FORCEINLINE void FlagHeads( int linear_tid, FlagT (&flags)[ItemsPerThread], T (&input)[ItemsPerThread], @@ -189,7 +189,7 @@ class BlockDiscontinuity * Binary boolean flag predicate */ template - static _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API static _CCCL_FORCEINLINE void FlagTails(int linear_tid, FlagT (&flags)[ItemsPerThread], T (&input)[ItemsPerThread], FlagOp flag_op) { _CCCL_PRAGMA_UNROLL_FULL() @@ -226,7 +226,7 @@ public: * First appears in CUDA Toolkit 12.3. * @endrst */ - _CCCL_DEVICE _CCCL_FORCEINLINE BlockDiscontinuity() + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockDiscontinuity() : temp_storage(PrivateStorage()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} @@ -242,7 +242,7 @@ public: * @param[in] temp_storage * Reference to memory allocation having layout type TempStorage */ - _CCCL_DEVICE _CCCL_FORCEINLINE BlockDiscontinuity(TempStorage& temp_storage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockDiscontinuity(TempStorage& temp_storage) : temp_storage(temp_storage.Alias()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} @@ -267,7 +267,7 @@ public: * Binary boolean flag predicate */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void FlagHeads(FlagT (&head_flags)[ItemsPerThread], T (&input)[ItemsPerThread], T (&preds)[ItemsPerThread], FlagOp flag_op) { // Share last item @@ -308,7 +308,7 @@ public: * (input0 from thread0). */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void FlagHeads( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void FlagHeads( FlagT (&head_flags)[ItemsPerThread], T (&input)[ItemsPerThread], T (&preds)[ItemsPerThread], @@ -403,7 +403,7 @@ public: //! @param[in] flag_op //! Binary boolean flag predicate template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void FlagHeads(FlagT (&head_flags)[ItemsPerThread], T (&input)[ItemsPerThread], FlagOp flag_op) { T preds[ItemsPerThread]; @@ -490,7 +490,7 @@ public: //! *thread*\ :sub:`0` only item with which to compare the first tile item (``input[0]`` from *thread*\ :sub:`0`). //! @endrst template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void FlagHeads(FlagT (&head_flags)[ItemsPerThread], T (&input)[ItemsPerThread], FlagOp flag_op, T tile_predecessor_item) { T preds[ItemsPerThread]; @@ -573,7 +573,7 @@ public: //! @param[in] flag_op //! Binary boolean flag predicate template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void FlagTails(FlagT (&tail_flags)[ItemsPerThread], T (&input)[ItemsPerThread], FlagOp flag_op) { // Share first item @@ -677,7 +677,7 @@ public: //! *thread*\ :sub:`BLOCK_THREADS - 1`). //! @endrst template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void FlagTails(FlagT (&tail_flags)[ItemsPerThread], T (&input)[ItemsPerThread], FlagOp flag_op, T tile_successor_item) { // Share first item @@ -781,7 +781,7 @@ public: //! @param[in] flag_op //! Binary boolean flag predicate template - _CCCL_DEVICE _CCCL_FORCEINLINE void FlagHeadsAndTails( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void FlagHeadsAndTails( FlagT (&head_flags)[ItemsPerThread], FlagT (&tail_flags)[ItemsPerThread], T (&input)[ItemsPerThread], FlagOp flag_op) { // Share first and last items @@ -912,7 +912,7 @@ public: //! @param[in] flag_op //! Binary boolean flag predicate template - _CCCL_DEVICE _CCCL_FORCEINLINE void FlagHeadsAndTails( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void FlagHeadsAndTails( FlagT (&head_flags)[ItemsPerThread], FlagT (&tail_flags)[ItemsPerThread], T tile_successor_item, @@ -1048,7 +1048,7 @@ public: //! @param[in] flag_op //! Binary boolean flag predicate template - _CCCL_DEVICE _CCCL_FORCEINLINE void FlagHeadsAndTails( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void FlagHeadsAndTails( FlagT (&head_flags)[ItemsPerThread], T tile_predecessor_item, FlagT (&tail_flags)[ItemsPerThread], @@ -1188,7 +1188,7 @@ public: //! @param[in] flag_op //! Binary boolean flag predicate template - _CCCL_DEVICE _CCCL_FORCEINLINE void FlagHeadsAndTails( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void FlagHeadsAndTails( FlagT (&head_flags)[ItemsPerThread], T tile_predecessor_item, FlagT (&tail_flags)[ItemsPerThread], diff --git a/cub/cub/block/block_exchange.cuh b/cub/cub/block/block_exchange.cuh index 1c1f46e44862..b188c740cc50 100644 --- a/cub/cub/block/block_exchange.cuh +++ b/cub/cub/block/block_exchange.cuh @@ -159,7 +159,7 @@ private: unsigned int warp_offset = warp_id * WARP_TIME_SLICED_ITEMS; /// Internal storage allocator - _CCCL_DEVICE _CCCL_FORCEINLINE _TempStorage& PrivateStorage() + _CCCL_DEVICE_API _CCCL_FORCEINLINE _TempStorage& PrivateStorage() { __shared__ _TempStorage private_storage; return private_storage; @@ -174,7 +174,7 @@ private: //! @param[out] output_items //! Items to exchange, converting between **blocked** and **striped** arrangements. template - _CCCL_DEVICE _CCCL_FORCEINLINE void BlockedToStriped( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void BlockedToStriped( const T (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread], ::cuda::std::false_type /*time_slicing*/) @@ -213,7 +213,7 @@ private: //! @param[out] output_items //! Items to exchange, converting between **blocked** and **striped** arrangements. template - _CCCL_DEVICE _CCCL_FORCEINLINE void BlockedToStriped( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void BlockedToStriped( const T (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread], ::cuda::std::true_type /*time_slicing*/) @@ -283,7 +283,7 @@ private: //! @param[out] output_items //! Items to exchange, converting between **blocked** and **striped** arrangements. template - _CCCL_DEVICE _CCCL_FORCEINLINE void BlockedToWarpStriped( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void BlockedToWarpStriped( const T (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread], ::cuda::std::false_type /*time_slicing*/) @@ -322,7 +322,7 @@ private: //! @param[out] output_items //! Items to exchange, converting between **blocked** and **striped** arrangements. template - _CCCL_DEVICE _CCCL_FORCEINLINE void BlockedToWarpStriped( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void BlockedToWarpStriped( const T (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread], ::cuda::std::true_type /*time_slicing*/) @@ -397,7 +397,7 @@ private: //! @param[out] output_items //! Items to exchange, converting between **blocked** and **striped** arrangements. template - _CCCL_DEVICE _CCCL_FORCEINLINE void StripedToBlocked( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void StripedToBlocked( const T (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread], ::cuda::std::false_type /*time_slicing*/) @@ -437,7 +437,7 @@ private: //! @param[out] output_items //! Items to exchange, converting between **blocked** and **striped** arrangements. template - _CCCL_DEVICE _CCCL_FORCEINLINE void StripedToBlocked( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void StripedToBlocked( const T (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread], ::cuda::std::true_type /*time_slicing*/) @@ -508,7 +508,7 @@ private: //! @param[out] output_items //! Items to exchange, converting between **blocked** and **striped** arrangements. template - _CCCL_DEVICE _CCCL_FORCEINLINE void WarpStripedToBlocked( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void WarpStripedToBlocked( const T (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread], ::cuda::std::false_type /*time_slicing*/) @@ -547,7 +547,7 @@ private: //! @param[out] output_items //! Items to exchange, converting between **blocked** and **striped** arrangements. template - _CCCL_DEVICE _CCCL_FORCEINLINE void WarpStripedToBlocked( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void WarpStripedToBlocked( const T (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread], ::cuda::std::true_type /*time_slicing*/) @@ -597,7 +597,7 @@ private: //! @param[in] ranks //! Corresponding scatter ranks template - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterToBlocked( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterToBlocked( const T (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread], OffsetT (&ranks)[ItemsPerThread], @@ -639,7 +639,7 @@ private: //! @param[in] ranks //! Corresponding scatter ranks template - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterToBlocked( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterToBlocked( const T (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread], OffsetT ranks[ItemsPerThread], @@ -704,7 +704,7 @@ private: //! @param[in] ranks //! Corresponding scatter ranks template - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterToStriped( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterToStriped( const T (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread], OffsetT (&ranks)[ItemsPerThread], @@ -746,7 +746,7 @@ private: //! @param[in] ranks //! Corresponding scatter ranks template - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterToStriped( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterToStriped( const T (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread], OffsetT (&ranks)[ItemsPerThread], @@ -818,7 +818,7 @@ public: //! .. versionadded:: 2.2.0 //! First appears in CUDA Toolkit 12.3. //! @endrst - _CCCL_DEVICE _CCCL_FORCEINLINE BlockExchange() + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockExchange() : temp_storage(PrivateStorage()) {} @@ -830,7 +830,7 @@ public: //! @endrst //! //! @param[in] temp_storage Reference to memory allocation having layout type TempStorage - _CCCL_DEVICE _CCCL_FORCEINLINE BlockExchange(TempStorage& temp_storage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockExchange(TempStorage& temp_storage) : temp_storage(temp_storage.Alias()) {} @@ -883,7 +883,7 @@ public: //! @param[out] output_items //! Items from exchange, converting between **striped** and **blocked** arrangements. template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void StripedToBlocked(const T (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread]) { StripedToBlocked(input_items, output_items, detail::bool_constant_v); @@ -900,7 +900,7 @@ public: //! //! @param[in,out] items //! Items to exchange, converting between **striped** and **blocked** arrangements. - _CCCL_DEVICE _CCCL_FORCEINLINE void StripedToBlocked(T (&items)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void StripedToBlocked(T (&items)[ItemsPerThread]) { StripedToBlocked(items, items); } @@ -954,7 +954,7 @@ public: //! @param[out] output_items //! Items from exchange, converting between **striped** and **blocked** arrangements. template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void BlockedToStriped(const T (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread]) { BlockedToStriped(input_items, output_items, detail::bool_constant_v); @@ -971,7 +971,7 @@ public: //! //! @param[in,out] items //! Items to exchange, converting between **blocked** and **striped** arrangements. - _CCCL_DEVICE _CCCL_FORCEINLINE void BlockedToStriped(T (&items)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void BlockedToStriped(T (&items)[ItemsPerThread]) { BlockedToStriped(items, items); } @@ -1025,7 +1025,7 @@ public: //! @param[out] output_items //! Items from exchange, converting between **striped** and **blocked** arrangements. template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void WarpStripedToBlocked(const T (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread]) { WarpStripedToBlocked(input_items, output_items, detail::bool_constant_v); @@ -1042,7 +1042,7 @@ public: //! //! @param[in,out] items //! Items to exchange, converting between **warp-striped** and **blocked** arrangements. - _CCCL_DEVICE _CCCL_FORCEINLINE void WarpStripedToBlocked(T (&items)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void WarpStripedToBlocked(T (&items)[ItemsPerThread]) { WarpStripedToBlocked(items, items); } @@ -1099,7 +1099,7 @@ public: //! @param[out] output_items //! Items from exchange, converting between **striped** and **blocked** arrangements. template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void BlockedToWarpStriped(const T (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread]) { BlockedToWarpStriped(input_items, output_items, detail::bool_constant_v); @@ -1116,7 +1116,7 @@ public: //! //! @param[in,out] items //! Items to exchange, converting between **blocked** and **warp-striped** arrangements. - _CCCL_DEVICE _CCCL_FORCEINLINE void BlockedToWarpStriped(T (&items)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void BlockedToWarpStriped(T (&items)[ItemsPerThread]) { BlockedToWarpStriped(items, items); } @@ -1146,7 +1146,7 @@ public: //! @param[in] ranks //! Corresponding scatter ranks template - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterToBlocked( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterToBlocked( const T (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread], OffsetT (&ranks)[ItemsPerThread]) { ScatterToBlocked(input_items, output_items, ranks, detail::bool_constant_v); @@ -1170,7 +1170,7 @@ public: //! @param[in] ranks //! Corresponding scatter ranks template - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterToBlocked(T (&items)[ItemsPerThread], OffsetT (&ranks)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterToBlocked(T (&items)[ItemsPerThread], OffsetT (&ranks)[ItemsPerThread]) { ScatterToBlocked(items, items, ranks); } @@ -1197,7 +1197,7 @@ public: //! @param[in] ranks //! Corresponding scatter ranks template - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterToStriped( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterToStriped( const T (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread], OffsetT (&ranks)[ItemsPerThread]) { ScatterToStriped(input_items, output_items, ranks, detail::bool_constant_v); @@ -1221,7 +1221,7 @@ public: //! @param[in] ranks //! Corresponding scatter ranks template - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterToStriped(T (&items)[ItemsPerThread], OffsetT (&ranks)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterToStriped(T (&items)[ItemsPerThread], OffsetT (&ranks)[ItemsPerThread]) { ScatterToStriped(items, items, ranks); } @@ -1248,7 +1248,7 @@ public: //! @param[in] ranks //! Corresponding scatter ranks template - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterToStripedGuarded( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterToStripedGuarded( const T (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread], OffsetT (&ranks)[ItemsPerThread]) { _CCCL_PRAGMA_UNROLL_FULL() @@ -1298,7 +1298,7 @@ public: //! @param[in] ranks //! Corresponding scatter ranks template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterToStripedGuarded(T (&items)[ItemsPerThread], OffsetT (&ranks)[ItemsPerThread]) { ScatterToStripedGuarded(items, items, ranks); @@ -1332,7 +1332,7 @@ public: //! @param[in] is_valid //! Corresponding flag denoting item validity template - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterToStripedFlagged( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterToStripedFlagged( const T (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread], OffsetT (&ranks)[ItemsPerThread], @@ -1391,7 +1391,7 @@ public: //! @param[in] is_valid //! Corresponding flag denoting item validity template - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterToStripedFlagged( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterToStripedFlagged( T (&items)[ItemsPerThread], OffsetT (&ranks)[ItemsPerThread], ValidFlag (&is_valid)[ItemsPerThread]) { ScatterToStripedFlagged(items, items, ranks, is_valid); diff --git a/cub/cub/block/block_histogram.cuh b/cub/cub/block/block_histogram.cuh index bd023e90b1db..8d34248a582e 100644 --- a/cub/cub/block/block_histogram.cuh +++ b/cub/cub/block/block_histogram.cuh @@ -185,7 +185,7 @@ class BlockHistogram unsigned int linear_tid; /// Internal storage allocator - _CCCL_DEVICE _CCCL_FORCEINLINE _TempStorage& PrivateStorage() + _CCCL_DEVICE_API _CCCL_FORCEINLINE _TempStorage& PrivateStorage() { __shared__ _TempStorage private_storage; return private_storage; @@ -204,7 +204,7 @@ public: //! .. versionadded:: 2.2.0 //! First appears in CUDA Toolkit 12.3. //! @endrst - _CCCL_DEVICE _CCCL_FORCEINLINE BlockHistogram() + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockHistogram() : temp_storage(PrivateStorage()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} @@ -220,7 +220,7 @@ public: * @param[in] temp_storage * Reference to memory allocation having layout type TempStorage */ - _CCCL_DEVICE _CCCL_FORCEINLINE BlockHistogram(TempStorage& temp_storage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockHistogram(TempStorage& temp_storage) : temp_storage(temp_storage.Alias()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} @@ -272,7 +272,7 @@ public: //! @tparam CounterT //! **[inferred]** Histogram counter type template - _CCCL_DEVICE _CCCL_FORCEINLINE void InitHistogram(CounterT histogram[Bins]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InitHistogram(CounterT histogram[Bins]) { // Initialize histogram bin counts to zeros int histo_offset = 0; @@ -338,7 +338,7 @@ public: //! @param[out] histogram //! Reference to shared/device-accessible memory histogram template - _CCCL_DEVICE _CCCL_FORCEINLINE void Histogram(T (&items)[ItemsPerThread], CounterT histogram[Bins]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Histogram(T (&items)[ItemsPerThread], CounterT histogram[Bins]) { // Initialize histogram bin counts to zeros InitHistogram(histogram); @@ -402,7 +402,7 @@ public: //! @param[out] histogram //! Reference to shared/device-accessible memory histogram template - _CCCL_DEVICE _CCCL_FORCEINLINE void Composite(T (&items)[ItemsPerThread], CounterT histogram[Bins]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Composite(T (&items)[ItemsPerThread], CounterT histogram[Bins]) { InternalBlockHistogram(temp_storage).Composite(items, histogram); } diff --git a/cub/cub/block/block_load.cuh b/cub/cub/block/block_load.cuh index 0883b5731564..8f7136d1b1f6 100644 --- a/cub/cub/block/block_load.cuh +++ b/cub/cub/block/block_load.cuh @@ -80,7 +80,7 @@ CUB_NAMESPACE_BEGIN //! @param[out] dst_items //! Destination to load data into template -_CCCL_DEVICE _CCCL_FORCEINLINE void +_CCCL_DEVICE_API _CCCL_FORCEINLINE void LoadDirectBlocked(int linear_tid, RandomAccessIterator block_src_it, T (&dst_items)[ItemsPerThread]) { // Load directly in thread-blocked order @@ -123,7 +123,7 @@ LoadDirectBlocked(int linear_tid, RandomAccessIterator block_src_it, T (&dst_ite //! @param[in] block_items_end //! First out-of-bounds index when loading from block_src_it template -_CCCL_DEVICE _CCCL_FORCEINLINE void LoadDirectBlocked( +_CCCL_DEVICE_API _CCCL_FORCEINLINE void LoadDirectBlocked( int linear_tid, RandomAccessIterator block_src_it, T (&dst_items)[ItemsPerThread], int block_items_end) { _CCCL_PRAGMA_UNROLL_FULL() @@ -173,7 +173,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void LoadDirectBlocked( //! @param[in] oob_default //! Default value to assign out-of-bound items template -_CCCL_DEVICE _CCCL_FORCEINLINE void LoadDirectBlocked( +_CCCL_DEVICE_API _CCCL_FORCEINLINE void LoadDirectBlocked( int linear_tid, RandomAccessIterator block_src_it, T (&dst_items)[ItemsPerThread], @@ -265,7 +265,7 @@ using block_load_vector_t = typename block_load_vector::type; //! @param[out] dst_items //! destination to load data into template -_CCCL_DEVICE _CCCL_FORCEINLINE void +_CCCL_DEVICE_API _CCCL_FORCEINLINE void LoadDirectBlockedVectorized(int linear_tid, const InputT* block_src_ptr, T (&dst_items)[ItemsPerThread]) { constexpr int vec_size = VecSize == 0 ? ItemsPerThread : VecSize; @@ -312,14 +312,14 @@ LoadDirectBlockedVectorized(int linear_tid, const InputT* block_src_ptr, T (&dst #ifndef _CCCL_DOXYGEN_INVOKED // Do not document // Accepts explicitly specified T and ItemsPerThread, which cannot bind to the leading Modifier of the overload above template -_CCCL_DEVICE _CCCL_FORCEINLINE void +_CCCL_DEVICE_API _CCCL_FORCEINLINE void LoadDirectBlockedVectorized(int linear_tid, T* block_src_ptr, T (&dst_items)[ItemsPerThread]) { _CUB::LoadDirectBlockedVectorized(linear_tid, block_src_ptr, dst_items); } template -CCCL_DEPRECATED_BECAUSE("Use LoadDirectBlockedVectorized instead") _CCCL_DEVICE _CCCL_FORCEINLINE void +CCCL_DEPRECATED_BECAUSE("Use LoadDirectBlockedVectorized instead") _CCCL_DEVICE_API _CCCL_FORCEINLINE void InternalLoadDirectBlockedVectorized(int linear_tid, const T* block_src_ptr, T (&dst_items)[ItemsPerThread]) { _CUB::LoadDirectBlockedVectorized(linear_tid, block_src_ptr, dst_items); @@ -362,7 +362,7 @@ InternalLoadDirectBlockedVectorized(int linear_tid, const T* block_src_ptr, T (& //! @param[out] dst_items //! Destination to load data into template -_CCCL_DEVICE _CCCL_FORCEINLINE void +_CCCL_DEVICE_API _CCCL_FORCEINLINE void LoadDirectStriped(int linear_tid, RandomAccessIterator block_src_it, T (&dst_items)[ItemsPerThread]) { _CCCL_PRAGMA_UNROLL_FULL() @@ -382,7 +382,7 @@ template -_CCCL_DEVICE _CCCL_FORCEINLINE void +_CCCL_DEVICE_API _CCCL_FORCEINLINE void load_direct_striped_vectorized_aligned(int linear_tid, const InputT* block_src_ptr, T (&dst_items)[ItemsPerThread]) { constexpr int vec_size = ::cuda::std::min(ItemsPerThread, static_cast(VecSize)); @@ -465,7 +465,7 @@ template -_CCCL_DEVICE _CCCL_FORCEINLINE void +_CCCL_DEVICE_API _CCCL_FORCEINLINE void LoadDirectStripedVectorized(int linear_tid, const InputT* block_src_ptr, T (&dst_items)[ItemsPerThread]) { static_assert(ItemsPerThread % VecSize == 0, "VecSize must evenly divide ItemsPerThread"); @@ -506,7 +506,7 @@ LoadDirectStripedVectorized(int linear_tid, const InputT* block_src_ptr, T (&dst namespace detail { template -_CCCL_DEVICE _CCCL_FORCEINLINE void load_transform_direct_striped( +_CCCL_DEVICE_API _CCCL_FORCEINLINE void load_transform_direct_striped( int linear_tid, RandomAccessIterator block_src_it, T (&dst_items)[ItemsPerThread], TransformOpT transform_op) { _CCCL_PRAGMA_UNROLL_FULL() @@ -553,7 +553,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void load_transform_direct_striped( //! @param[in] block_items_end //! Number of valid items to load template -_CCCL_DEVICE _CCCL_FORCEINLINE void LoadDirectStriped( +_CCCL_DEVICE_API _CCCL_FORCEINLINE void LoadDirectStriped( int linear_tid, RandomAccessIterator block_src_it, T (&dst_items)[ItemsPerThread], int block_items_end) { _CCCL_PRAGMA_UNROLL_FULL() @@ -606,7 +606,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void LoadDirectStriped( //! @param[in] oob_default //! Default value to assign out-of-bound items template -_CCCL_DEVICE _CCCL_FORCEINLINE void LoadDirectStriped( +_CCCL_DEVICE_API _CCCL_FORCEINLINE void LoadDirectStriped( int linear_tid, RandomAccessIterator block_src_it, T (&dst_items)[ItemsPerThread], @@ -660,7 +660,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void LoadDirectStriped( //! @param[out] dst_items //! Destination to load data into template -_CCCL_DEVICE _CCCL_FORCEINLINE void +_CCCL_DEVICE_API _CCCL_FORCEINLINE void LoadDirectWarpStriped(int linear_tid, RandomAccessIterator block_src_it, T (&dst_items)[ItemsPerThread]) { const int tid = linear_tid & (detail::warp_threads - 1); @@ -713,7 +713,7 @@ LoadDirectWarpStriped(int linear_tid, RandomAccessIterator block_src_it, T (&dst //! @param[in] block_items_end //! Number of valid items to load template -_CCCL_DEVICE _CCCL_FORCEINLINE void LoadDirectWarpStriped( +_CCCL_DEVICE_API _CCCL_FORCEINLINE void LoadDirectWarpStriped( int linear_tid, RandomAccessIterator block_src_it, T (&dst_items)[ItemsPerThread], int block_items_end) { const int tid = linear_tid & (detail::warp_threads - 1); @@ -773,7 +773,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void LoadDirectWarpStriped( //! @param[in] oob_default //! Default value to assign out-of-bound items template -_CCCL_DEVICE _CCCL_FORCEINLINE void LoadDirectWarpStriped( +_CCCL_DEVICE_API _CCCL_FORCEINLINE void LoadDirectWarpStriped( int linear_tid, RandomAccessIterator block_src_it, T (&dst_items)[ItemsPerThread], @@ -970,7 +970,7 @@ namespace detail } } // namespace detail -inline ::std::ostream& operator<<(::std::ostream& os, BlockLoadAlgorithm algo) +_CCCL_HOST_API inline ::std::ostream& operator<<(::std::ostream& os, BlockLoadAlgorithm algo) { return os << _CUB::detail::to_string(algo); } @@ -983,7 +983,7 @@ template <::cuda::std::same_as CharT> struct std::formatter<_CUB::BlockLoadAlgorithm, CharT> : formatter { template - auto format(const _CUB::BlockLoadAlgorithm& algo, FmtCtx& ctx) const + _CCCL_HOST_API auto format(const _CUB::BlockLoadAlgorithm& algo, FmtCtx& ctx) const { return formatter::format(_CUB::detail::to_string(algo), ctx); } @@ -1124,7 +1124,7 @@ class BlockLoad using _TempStorage _CCCL_NODEBUG = decltype(temp_storage_helper()); // Internal storage allocator - _CCCL_DEVICE _CCCL_FORCEINLINE _TempStorage& PrivateStorage() + _CCCL_DEVICE_API _CCCL_FORCEINLINE _TempStorage& PrivateStorage() { __shared__ _TempStorage private_storage; return private_storage; @@ -1146,7 +1146,7 @@ public: //! .. versionadded:: 2.2.0 //! First appears in CUDA Toolkit 12.3. //! @endrst - _CCCL_DEVICE _CCCL_FORCEINLINE BlockLoad() + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockLoad() : temp_storage(PrivateStorage()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} @@ -1159,7 +1159,7 @@ public: //! @endrst //! //! @param[in] temp_storage Reference to memory allocation having layout type TempStorage - _CCCL_DEVICE _CCCL_FORCEINLINE BlockLoad(TempStorage& temp_storage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockLoad(TempStorage& temp_storage) : temp_storage(temp_storage.Alias()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} @@ -1213,7 +1213,7 @@ public: //! @param[out] dst_items //! Destination to load data into template - _CCCL_DEVICE _CCCL_FORCEINLINE void Load(RandomAccessIterator block_src_it, T (&dst_items)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Load(RandomAccessIterator block_src_it, T (&dst_items)[ItemsPerThread]) { if constexpr (Algorithm == BLOCK_LOAD_DIRECT) { @@ -1331,7 +1331,7 @@ public: //! @param[in] block_items_end //! Number of valid items to load template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Load(RandomAccessIterator block_src_it, T (&dst_items)[ItemsPerThread], int block_items_end) { if constexpr (Algorithm == BLOCK_LOAD_DIRECT || Algorithm == BLOCK_LOAD_VECTORIZE) @@ -1407,7 +1407,7 @@ public: //! @param[in] oob_default //! Default value to assign out-of-bound items template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Load(RandomAccessIterator block_src_it, T (&dst_items)[ItemsPerThread], int block_items_end, DefaultT oob_default) { if constexpr (Algorithm == BLOCK_LOAD_DIRECT || Algorithm == BLOCK_LOAD_VECTORIZE) diff --git a/cub/cub/block/block_load_to_shared.cuh b/cub/cub/block/block_load_to_shared.cuh index 192e6e14e0b8..087fafcc50d1 100644 --- a/cub/cub/block/block_load_to_shared.cuh +++ b/cub/cub/block/block_load_to_shared.cuh @@ -242,8 +242,8 @@ private: token_impl& operator=(const token_impl&) = delete; // NOLINTEND(modernize-use-equals-delete) - token_impl(token_impl&&) = default; - token_impl& operator=(token_impl&&) = default; + _CCCL_HIDE_FROM_ABI token_impl(token_impl&&) = default; + _CCCL_HIDE_FROM_ABI token_impl& operator=(token_impl&&) = default; }; public: diff --git a/cub/cub/block/block_merge_sort.cuh b/cub/cub/block/block_merge_sort.cuh index aa84e039135b..8563188c95e6 100644 --- a/cub/cub/block/block_merge_sort.cuh +++ b/cub/cub/block/block_merge_sort.cuh @@ -37,7 +37,7 @@ CUB_NAMESPACE_BEGIN //! \returns The number of elements merged from the first sequence at the intersection of the diagonal with the merge //! path. The number of elements merged from the second sequence is \c diag minus the returned value. template -_CCCL_DEVICE _CCCL_FORCEINLINE OffsetT +_CCCL_DEVICE_API _CCCL_FORCEINLINE OffsetT MergePath(KeyIt1 keys1, KeyIt2 keys2, OffsetT keys1_count, OffsetT keys2_count, OffsetT diag, BinaryPred binary_pred) { OffsetT keys1_begin = diag < keys2_count ? 0 : diag - keys2_count; @@ -64,7 +64,7 @@ MergePath(KeyIt1 keys1, KeyIt2 keys2, OffsetT keys1_count, OffsetT keys2_count, namespace detail { template -_CCCL_DEVICE _CCCL_FORCEINLINE void serial_merge( +_CCCL_DEVICE_API _CCCL_FORCEINLINE void serial_merge( KeyIt keys_shared, int keys1_beg, int keys2_beg, @@ -100,7 +100,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void serial_merge( } template -_CCCL_DEVICE _CCCL_FORCEINLINE void serial_merge( +_CCCL_DEVICE_API _CCCL_FORCEINLINE void serial_merge( KeyIt keys_shared, int keys1_beg, int keys2_beg, @@ -127,7 +127,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void serial_merge( //! \param output The output array //! \param indices The shared memory indices relative to \c keys_shared of the elements written to \c output template -_CCCL_DEVICE _CCCL_FORCEINLINE void SerialMerge( +_CCCL_DEVICE_API _CCCL_FORCEINLINE void SerialMerge( KeyIt keys_shared, int keys1_beg, int keys2_beg, @@ -143,7 +143,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void SerialMerge( } template -_CCCL_DEVICE _CCCL_FORCEINLINE void SerialMerge( +_CCCL_DEVICE_API _CCCL_FORCEINLINE void SerialMerge( KeyIt keys_shared, int keys1_beg, int keys2_beg, @@ -245,7 +245,7 @@ class BlockMergeSortStrategy _TempStorage& temp_storage; /// Internal storage allocator - _CCCL_DEVICE _CCCL_FORCEINLINE _TempStorage& PrivateStorage() + _CCCL_DEVICE_API _CCCL_FORCEINLINE _TempStorage& PrivateStorage() { __shared__ _TempStorage private_storage; return private_storage; @@ -258,17 +258,17 @@ public: using TempStorage = Uninitialized<_TempStorage>; BlockMergeSortStrategy() = delete; - explicit _CCCL_DEVICE _CCCL_FORCEINLINE BlockMergeSortStrategy(unsigned int linear_tid) + explicit _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockMergeSortStrategy(unsigned int linear_tid) : temp_storage(PrivateStorage()) , linear_tid(linear_tid) {} - _CCCL_DEVICE _CCCL_FORCEINLINE BlockMergeSortStrategy(TempStorage& temp_storage, unsigned int linear_tid) + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockMergeSortStrategy(TempStorage& temp_storage, unsigned int linear_tid) : temp_storage(temp_storage.Alias()) , linear_tid(linear_tid) {} - _CCCL_DEVICE _CCCL_FORCEINLINE unsigned int get_linear_tid() const + _CCCL_DEVICE_API _CCCL_FORCEINLINE unsigned int get_linear_tid() const { return linear_tid; } @@ -301,7 +301,7 @@ public: * [Strict Weak Ordering]: https://en.cppreference.com/w/cpp/concepts/strict_weak_order */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void Sort(KeyT (&keys)[ItemsPerThread], CompareOp compare_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Sort(KeyT (&keys)[ItemsPerThread], CompareOp compare_op) { ValueT items[ItemsPerThread]; Sort(keys, items, compare_op, ITEMS_PER_TILE, keys[0]); @@ -348,7 +348,7 @@ public: * [Strict Weak Ordering]: https://en.cppreference.com/w/cpp/concepts/strict_weak_order */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Sort(KeyT (&keys)[ItemsPerThread], CompareOp compare_op, int valid_items, KeyT oob_default) { ValueT items[ItemsPerThread]; @@ -385,7 +385,7 @@ public: * [Strict Weak Ordering]: https://en.cppreference.com/w/cpp/concepts/strict_weak_order */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Sort(KeyT (&keys)[ItemsPerThread], ValueT (&items)[ItemsPerThread], CompareOp compare_op) { Sort(keys, items, compare_op, ITEMS_PER_TILE, keys[0]); @@ -438,7 +438,7 @@ public: * [Strict Weak Ordering]: https://en.cppreference.com/w/cpp/concepts/strict_weak_order */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Sort(KeyT (&keys)[ItemsPerThread], ValueT (&items)[ItemsPerThread], CompareOp compare_op, @@ -505,7 +505,7 @@ public: * [Strict Weak Ordering]: https://en.cppreference.com/w/cpp/concepts/strict_weak_order */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Sort(KeyT (&keys)[ItemsPerThread], ValueT (&items)[ItemsPerThread], CompareOp compare_op, int valid_items) { // keys[0] only binds a reference here; without a sentinel it is never read. @@ -549,7 +549,7 @@ public: * [Strict Weak Ordering]: https://en.cppreference.com/w/cpp/concepts/strict_weak_order */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void Sort(KeyT (&keys)[ItemsPerThread], CompareOp compare_op, int valid_items) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Sort(KeyT (&keys)[ItemsPerThread], CompareOp compare_op, int valid_items) { ValueT items[ItemsPerThread]; Sort(keys, items, compare_op, valid_items); @@ -584,7 +584,7 @@ public: * [Strict Weak Ordering]: https://en.cppreference.com/w/cpp/concepts/strict_weak_order */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void StableSort(KeyT (&keys)[ItemsPerThread], CompareOp compare_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void StableSort(KeyT (&keys)[ItemsPerThread], CompareOp compare_op) { Sort(keys, compare_op); } @@ -621,7 +621,7 @@ public: * [Strict Weak Ordering]: https://en.cppreference.com/w/cpp/concepts/strict_weak_order */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void StableSort(KeyT (&keys)[ItemsPerThread], ValueT (&items)[ItemsPerThread], CompareOp compare_op) { Sort(keys, items, compare_op); @@ -670,7 +670,7 @@ public: * [Strict Weak Ordering]: https://en.cppreference.com/w/cpp/concepts/strict_weak_order */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void StableSort(KeyT (&keys)[ItemsPerThread], CompareOp compare_op, int valid_items, KeyT oob_default) { Sort(keys, compare_op, valid_items, oob_default); @@ -724,7 +724,7 @@ public: * [Strict Weak Ordering]: https://en.cppreference.com/w/cpp/concepts/strict_weak_order */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void StableSort( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void StableSort( KeyT (&keys)[ItemsPerThread], ValueT (&items)[ItemsPerThread], CompareOp compare_op, @@ -772,7 +772,7 @@ public: * [Strict Weak Ordering]: https://en.cppreference.com/w/cpp/concepts/strict_weak_order */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void StableSort(KeyT (&keys)[ItemsPerThread], CompareOp compare_op, int valid_items) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void StableSort(KeyT (&keys)[ItemsPerThread], CompareOp compare_op, int valid_items) { Sort(keys, compare_op, valid_items); } @@ -818,7 +818,7 @@ public: * [Strict Weak Ordering]: https://en.cppreference.com/w/cpp/concepts/strict_weak_order */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void StableSort(KeyT (&keys)[ItemsPerThread], ValueT (&items)[ItemsPerThread], CompareOp compare_op, int valid_items) { Sort(keys, items, compare_op, valid_items); @@ -830,7 +830,7 @@ private: //! run, and the run ending at ITEMS_PER_TILE would otherwise read an uninitialized value //! (see NVIDIA/cccl#5327). The items side needs no such store: the gather indices record //! pre-increment positions, which never exceed ITEMS_PER_TILE - 1. - _CCCL_DEVICE _CCCL_FORCEINLINE void StoreKeys(KeyT (&keys)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void StoreKeys(KeyT (&keys)[ItemsPerThread]) { _CCCL_PRAGMA_UNROLL_FULL() for (int item = 0; item < ItemsPerThread; ++item) @@ -845,7 +845,7 @@ private: } //! Moves this thread's values into the positions given by \p indices, via shared memory. - _CCCL_DEVICE _CCCL_FORCEINLINE void ExchangeItems(ValueT (&items)[ItemsPerThread], int (&indices)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExchangeItems(ValueT (&items)[ItemsPerThread], int (&indices)[ItemsPerThread]) { Sync(); @@ -879,7 +879,7 @@ private: //! indeterminate values, which are copied around but never compared and never placed within the //! valid prefix, because the merge rounds clamp every run to \p valid_items. template - _CCCL_DEVICE _CCCL_FORCEINLINE void SortPartialTile( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SortPartialTile( KeyT (&keys)[ItemsPerThread], ValueT (&items)[ItemsPerThread], CompareOp compare_op, @@ -923,7 +923,7 @@ private: //! end up with unspecified values, and trailing rounds whose input runs already cover all valid //! items are skipped. template - _CCCL_DEVICE _CCCL_FORCEINLINE void MergeRounds( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void MergeRounds( KeyT (&keys)[ItemsPerThread], ValueT (&items)[ItemsPerThread], CompareOp compare_op, @@ -1015,7 +1015,7 @@ private: } } - _CCCL_DEVICE _CCCL_FORCEINLINE void Sync() const + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Sync() const { static_cast(this)->SyncImplementation(); } @@ -1123,16 +1123,17 @@ class BlockMergeSort BlockMergeSortStrategy; public: - _CCCL_DEVICE _CCCL_FORCEINLINE BlockMergeSort() + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockMergeSort() : BlockMergeSortStrategyT(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} - _CCCL_DEVICE _CCCL_FORCEINLINE explicit BlockMergeSort(typename BlockMergeSortStrategyT::TempStorage& temp_storage) + _CCCL_DEVICE_API + _CCCL_FORCEINLINE explicit BlockMergeSort(typename BlockMergeSortStrategyT::TempStorage& temp_storage) : BlockMergeSortStrategyT(temp_storage, _CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} private: - _CCCL_DEVICE _CCCL_FORCEINLINE void SyncImplementation() const + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SyncImplementation() const { __syncthreads(); } diff --git a/cub/cub/block/block_radix_rank.cuh b/cub/cub/block/block_radix_rank.cuh index 4648a460b497..3d0b703b0ec2 100644 --- a/cub/cub/block/block_radix_rank.cuh +++ b/cub/cub/block/block_radix_rank.cuh @@ -98,7 +98,7 @@ namespace detail } } // namespace detail -inline ::std::ostream& operator<<(::std::ostream& os, RadixRankAlgorithm algo) +_CCCL_HOST_API inline ::std::ostream& operator<<(::std::ostream& os, RadixRankAlgorithm algo) { return os << _CUB::detail::to_string(algo); } @@ -124,7 +124,7 @@ CUB_NAMESPACE_BEGIN template struct BlockRadixRankEmptyCallback { - _CCCL_DEVICE _CCCL_FORCEINLINE void operator()(int (&bins)[BinsPerThread]) {} + _CCCL_DEVICE_API _CCCL_FORCEINLINE void operator()(int (&bins)[BinsPerThread]) {} }; #ifndef _CCCL_DOXYGEN_INVOKED // Do not document @@ -133,7 +133,7 @@ namespace detail template struct warp_in_block_matcher_t { - static _CCCL_DEVICE ::cuda::std::uint32_t match_any(::cuda::std::uint32_t label, ::cuda::std::uint32_t warp_id) + _CCCL_DEVICE_API static ::cuda::std::uint32_t match_any(::cuda::std::uint32_t label, ::cuda::std::uint32_t warp_id) { if (warp_id == static_cast<::cuda::std::uint32_t>(PartialWarpId)) { @@ -147,7 +147,7 @@ struct warp_in_block_matcher_t template struct warp_in_block_matcher_t { - static _CCCL_DEVICE ::cuda::std::uint32_t match_any(::cuda::std::uint32_t label, ::cuda::std::uint32_t warp_id) + _CCCL_DEVICE_API static ::cuda::std::uint32_t match_any(::cuda::std::uint32_t label, ::cuda::std::uint32_t warp_id) { return _CUB::MatchAny(label); } @@ -317,7 +317,7 @@ private: /** * Internal storage allocator */ - _CCCL_DEVICE _CCCL_FORCEINLINE _TempStorage& PrivateStorage() + _CCCL_DEVICE_API _CCCL_FORCEINLINE _TempStorage& PrivateStorage() { __shared__ _TempStorage private_storage; return private_storage; @@ -326,7 +326,7 @@ private: /** * Performs upsweep raking reduction, returning the aggregate */ - _CCCL_DEVICE _CCCL_FORCEINLINE PackedCounter Upsweep() + _CCCL_DEVICE_API _CCCL_FORCEINLINE PackedCounter Upsweep() { auto& smem_raking_ptr = temp_storage.aliasable.raking_grid[linear_tid]; if constexpr (MemoizeOuterScan) @@ -346,7 +346,7 @@ private: } /// Performs exclusive downsweep raking scan - _CCCL_DEVICE _CCCL_FORCEINLINE void ExclusiveDownsweep(PackedCounter raking_partial) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveDownsweep(PackedCounter raking_partial) { PackedCounter* smem_raking_ptr = temp_storage.aliasable.raking_grid[linear_tid]; @@ -369,7 +369,7 @@ private: /** * Reset shared memory digit counters */ - _CCCL_DEVICE _CCCL_FORCEINLINE void ResetCounters() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ResetCounters() { // Reset shared memory digit counters _CCCL_PRAGMA_UNROLL_FULL() @@ -384,7 +384,7 @@ private: */ struct PrefixCallBack { - _CCCL_DEVICE _CCCL_FORCEINLINE PackedCounter operator()(PackedCounter block_aggregate) + _CCCL_DEVICE_API _CCCL_FORCEINLINE PackedCounter operator()(PackedCounter block_aggregate) { PackedCounter block_prefix = 0; @@ -402,7 +402,7 @@ private: /** * Scan shared memory digit counters. */ - _CCCL_DEVICE _CCCL_FORCEINLINE void ScanCounters() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScanCounters() { // Upsweep scan const PackedCounter raking_partial = Upsweep(); @@ -425,7 +425,7 @@ public: //! @{ //! @brief Collective constructor using a private static allocation of shared memory as temporary storage. - _CCCL_DEVICE _CCCL_FORCEINLINE BlockRadixRank() + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockRadixRank() : temp_storage(PrivateStorage()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} @@ -436,7 +436,7 @@ public: * @param[in] temp_storage * Reference to memory allocation having layout type TempStorage */ - _CCCL_DEVICE _CCCL_FORCEINLINE BlockRadixRank(TempStorage& temp_storage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockRadixRank(TempStorage& temp_storage) : temp_storage(temp_storage.Alias()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} @@ -458,7 +458,7 @@ public: * The digit extractor */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void RankKeys(UnsignedBits (&keys)[KeysPerThread], int (&ranks)[KeysPerThread], DigitExtractorT digit_extractor) { static_assert(BLOCK_THREADS * KeysPerThread <= max_tile_size, @@ -536,7 +536,7 @@ public: * (threadIdx.x * BINS_TRACKED_PER_THREAD) + BINS_TRACKED_PER_THREAD - 1] */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void RankKeys(UnsignedBits (&keys)[KeysPerThread], int (&ranks)[KeysPerThread], DigitExtractorT digit_extractor, @@ -646,7 +646,7 @@ public: * @param[in] temp_storage * Reference to memory allocation having layout type TempStorage */ - _CCCL_DEVICE _CCCL_FORCEINLINE BlockRadixRankMatch(TempStorage& temp_storage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockRadixRankMatch(TempStorage& temp_storage) : temp_storage(temp_storage.Alias()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} @@ -670,7 +670,7 @@ public: * separate output parameter of RankKeys(). */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void CallBack(CountsCallback callback) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void CallBack(CountsCallback callback) { int bins[BINS_TRACKED_PER_THREAD]; // Get count for each digit @@ -716,7 +716,7 @@ public: * Callback to receive digit counts */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void RankKeys(UnsignedBits (&keys)[KeysPerThread], int (&ranks)[KeysPerThread], DigitExtractorT digit_extractor, @@ -816,7 +816,7 @@ public: } template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void RankKeys(UnsignedBits (&keys)[KeysPerThread], int (&ranks)[KeysPerThread], DigitExtractorT digit_extractor) { RankKeys(keys, ranks, digit_extractor, BlockRadixRankEmptyCallback()); @@ -845,7 +845,7 @@ public: * Callback to receive digit counts */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void RankKeys( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void RankKeys( UnsignedBits (&keys)[KeysPerThread], int (&ranks)[KeysPerThread], DigitExtractorT digit_extractor, @@ -886,7 +886,7 @@ public: * (threadIdx.x * BINS_TRACKED_PER_THREAD) + BINS_TRACKED_PER_THREAD - 1] */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void RankKeys(UnsignedBits (&keys)[KeysPerThread], int (&ranks)[KeysPerThread], DigitExtractorT digit_extractor, @@ -968,19 +968,19 @@ struct BlockRadixRankMatchEarlyCounts int warp; int lane; - _CCCL_DEVICE _CCCL_FORCEINLINE ::cuda::std::uint32_t Digit(UnsignedBits key) + _CCCL_DEVICE_API _CCCL_FORCEINLINE ::cuda::std::uint32_t Digit(UnsignedBits key) { ::cuda::std::uint32_t digit = digit_extractor.Digit(key); return IsDescending ? RADIX_DIGITS - 1 - digit : digit; } - _CCCL_DEVICE _CCCL_FORCEINLINE int ThreadBin(int u) + _CCCL_DEVICE_API _CCCL_FORCEINLINE int ThreadBin(int u) { const int bin = (threadIdx.x * BINS_PER_THREAD) + u; return IsDescending ? RADIX_DIGITS - 1 - bin : bin; } - _CCCL_DEVICE _CCCL_FORCEINLINE void ComputeHistogramsWarp(UnsignedBits (&keys)[KeysPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ComputeHistogramsWarp(UnsignedBits (&keys)[KeysPerThread]) { // int* warp_offsets = &s.warp_offsets[warp][0]; int (&warp_histograms)[RADIX_DIGITS][NumParts] = s.warp_histograms[warp]; @@ -1044,7 +1044,7 @@ struct BlockRadixRankMatchEarlyCounts } } - _CCCL_DEVICE _CCCL_FORCEINLINE void ComputeOffsetsWarpUpsweep(int (&bins)[BINS_PER_THREAD]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ComputeOffsetsWarpUpsweep(int (&bins)[BINS_PER_THREAD]) { // sum up warp-private histograms _CCCL_PRAGMA_UNROLL_FULL() @@ -1065,7 +1065,7 @@ struct BlockRadixRankMatchEarlyCounts } } - _CCCL_DEVICE _CCCL_FORCEINLINE void ComputeOffsetsWarpDownsweep(int (&offsets)[BINS_PER_THREAD]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ComputeOffsetsWarpDownsweep(int (&offsets)[BINS_PER_THREAD]) { _CCCL_PRAGMA_UNROLL_FULL() for (int u = 0; u < BINS_PER_THREAD; ++u) @@ -1083,7 +1083,7 @@ struct BlockRadixRankMatchEarlyCounts } } - _CCCL_DEVICE _CCCL_FORCEINLINE void ComputeRanksItem( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ComputeRanksItem( UnsignedBits (&keys)[KeysPerThread], int (&ranks)[KeysPerThread], detail::constant_t) { // compute key ranks @@ -1119,7 +1119,7 @@ struct BlockRadixRankMatchEarlyCounts } } - _CCCL_DEVICE _CCCL_FORCEINLINE void ComputeRanksItem( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ComputeRanksItem( UnsignedBits (&keys)[KeysPerThread], int (&ranks)[KeysPerThread], detail::constant_t) { // compute key ranks @@ -1146,7 +1146,7 @@ struct BlockRadixRankMatchEarlyCounts } } - _CCCL_DEVICE _CCCL_FORCEINLINE void RankKeys( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void RankKeys( UnsignedBits (&keys)[KeysPerThread], int (&ranks)[KeysPerThread], int (&exclusive_digit_prefix)[BINS_PER_THREAD]) { ComputeHistogramsWarp(keys); @@ -1163,7 +1163,7 @@ struct BlockRadixRankMatchEarlyCounts ComputeRanksItem(keys, ranks, detail::constant_v); } - _CCCL_DEVICE _CCCL_FORCEINLINE + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockRadixRankMatchInternal(TempStorage& temp_storage, DigitExtractorT digit_extractor, CountsCallback callback) : s(temp_storage) , digit_extractor(digit_extractor) @@ -1173,7 +1173,7 @@ struct BlockRadixRankMatchEarlyCounts {} }; - _CCCL_DEVICE _CCCL_FORCEINLINE BlockRadixRankMatchEarlyCounts(TempStorage& temp_storage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockRadixRankMatchEarlyCounts(TempStorage& temp_storage) : temp_storage(temp_storage) {} @@ -1182,7 +1182,7 @@ struct BlockRadixRankMatchEarlyCounts * provided for the corresponding thread. */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void RankKeys( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void RankKeys( UnsignedBits (&keys)[KeysPerThread], int (&ranks)[KeysPerThread], DigitExtractorT digit_extractor, @@ -1195,7 +1195,7 @@ struct BlockRadixRankMatchEarlyCounts } template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void RankKeys(UnsignedBits (&keys)[KeysPerThread], int (&ranks)[KeysPerThread], DigitExtractorT digit_extractor, @@ -1208,7 +1208,7 @@ struct BlockRadixRankMatchEarlyCounts } template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void RankKeys(UnsignedBits (&keys)[KeysPerThread], int (&ranks)[KeysPerThread], DigitExtractorT digit_extractor) { int exclusive_digit_prefix[BINS_PER_THREAD]; diff --git a/cub/cub/block/block_radix_sort.cuh b/cub/cub/block/block_radix_sort.cuh index 6edbd7f15359..37bbafc458db 100644 --- a/cub/cub/block/block_radix_sort.cuh +++ b/cub/cub/block/block_radix_sort.cuh @@ -267,7 +267,7 @@ class BlockRadixSort ******************************************************************************/ /// Internal storage allocator - _CCCL_DEVICE _CCCL_FORCEINLINE _TempStorage& PrivateStorage() + _CCCL_DEVICE_API _CCCL_FORCEINLINE _TempStorage& PrivateStorage() { __shared__ _TempStorage private_storage; return private_storage; @@ -275,7 +275,7 @@ class BlockRadixSort /// Rank keys (specialized for ascending sort) template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void RankKeys(bit_ordered_type (&unsigned_keys)[ItemsPerThread], int (&ranks)[ItemsPerThread], DigitExtractorT digit_extractor, @@ -286,7 +286,7 @@ class BlockRadixSort /// Rank keys (specialized for descending sort) template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void RankKeys(bit_ordered_type (&unsigned_keys)[ItemsPerThread], int (&ranks)[ItemsPerThread], DigitExtractorT digit_extractor, @@ -296,7 +296,7 @@ class BlockRadixSort } /// ExchangeValues (specialized for key-value sort, to-blocked arrangement) - _CCCL_DEVICE _CCCL_FORCEINLINE void ExchangeValues( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExchangeValues( ValueT (&values)[ItemsPerThread], int (&ranks)[ItemsPerThread], ::cuda::std::false_type /*is_keys_only*/, @@ -309,7 +309,7 @@ class BlockRadixSort } /// ExchangeValues (specialized for key-value sort, to-striped arrangement) - _CCCL_DEVICE _CCCL_FORCEINLINE void ExchangeValues( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExchangeValues( ValueT (&values)[ItemsPerThread], int (&ranks)[ItemsPerThread], ::cuda::std::false_type /*is_keys_only*/, @@ -323,7 +323,7 @@ class BlockRadixSort /// ExchangeValues (specialized for keys-only sort) template - _CCCL_DEVICE _CCCL_FORCEINLINE void ExchangeValues( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExchangeValues( ValueT (& /*values*/)[ItemsPerThread], int (& /*ranks*/)[ItemsPerThread], ::cuda::std::true_type /*is_keys_only*/, @@ -356,7 +356,7 @@ class BlockRadixSort * constituent arithmetic types */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void SortBlocked( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SortBlocked( KeyT (&keys)[ItemsPerThread], ValueT (&values)[ItemsPerThread], int begin_bit, @@ -435,7 +435,7 @@ public: * Tag whether is keys-only sort */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void SortBlockedToStriped( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SortBlockedToStriped( KeyT (&keys)[ItemsPerThread], ValueT (&values)[ItemsPerThread], int begin_bit, @@ -510,7 +510,7 @@ public: //! .. versionadded:: 2.2.0 //! First appears in CUDA Toolkit 12.3. //! @endrst - _CCCL_DEVICE _CCCL_FORCEINLINE BlockRadixSort() + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockRadixSort() : temp_storage(PrivateStorage()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} @@ -526,7 +526,7 @@ public: * @param[in] temp_storage * Reference to memory allocation having layout type TempStorage */ - _CCCL_DEVICE _CCCL_FORCEINLINE BlockRadixSort(TempStorage& temp_storage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockRadixSort(TempStorage& temp_storage) : temp_storage(temp_storage.Alias()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} @@ -585,7 +585,7 @@ public: //! //! @param[in] end_bit //! **[optional]** The past-the-end (most-significant) bit index needed for key comparison - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Sort(KeyT (&keys)[ItemsPerThread], int begin_bit = 0, int end_bit = sizeof(KeyT) * 8) { NullType values[ItemsPerThread]; @@ -653,7 +653,7 @@ public: //! The most-significant bit index (exclusive) needed for key //! comparison (e.g., `(sizeof(float) + sizeof(long long int)) * 8`) template - _CCCL_DEVICE _CCCL_FORCEINLINE // + _CCCL_DEVICE_API _CCCL_FORCEINLINE // ::cuda::std::enable_if_t< // !::cuda::std::is_convertible_v> Sort(KeyT (&keys)[ItemsPerThread], DecomposerT decomposer, int begin_bit, int end_bit) @@ -716,7 +716,7 @@ public: //! the tuple is considered the most significant. The call operator must not //! modify members of the key. template - _CCCL_DEVICE _CCCL_FORCEINLINE // + _CCCL_DEVICE_API _CCCL_FORCEINLINE // ::cuda::std::enable_if_t< // !::cuda::std::is_convertible_v> Sort(KeyT (&keys)[ItemsPerThread], DecomposerT decomposer) { @@ -785,7 +785,7 @@ public: //! //! @param[in] end_bit //! **[optional]** The past-the-end (most-significant) bit index needed for key comparison - _CCCL_DEVICE _CCCL_FORCEINLINE void Sort( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Sort( KeyT (&keys)[ItemsPerThread], ValueT (&values)[ItemsPerThread], int begin_bit = 0, int end_bit = sizeof(KeyT) * 8) { SortBlocked(keys, values, begin_bit, end_bit, ::cuda::std::false_type(), detail::bool_constant_v); @@ -859,7 +859,7 @@ public: //! The most-significant bit index (exclusive) needed for key //! comparison (e.g., `(sizeof(float) + sizeof(long long int)) * 8`) template - _CCCL_DEVICE _CCCL_FORCEINLINE // + _CCCL_DEVICE_API _CCCL_FORCEINLINE // ::cuda::std::enable_if_t< // !::cuda::std::is_convertible_v> Sort( @@ -929,7 +929,7 @@ public: //! the tuple is considered the most significant. The call operator must not //! modify members of the key. template - _CCCL_DEVICE _CCCL_FORCEINLINE // + _CCCL_DEVICE_API _CCCL_FORCEINLINE // ::cuda::std::enable_if_t< // !::cuda::std::is_convertible_v> Sort(KeyT (&keys)[ItemsPerThread], ValueT (&values)[ItemsPerThread], DecomposerT decomposer) @@ -988,7 +988,7 @@ public: //! //! @param[in] end_bit //! **[optional]** The past-the-end (most-significant) bit index needed for key comparison - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SortDescending(KeyT (&keys)[ItemsPerThread], int begin_bit = 0, int end_bit = sizeof(KeyT) * 8) { NullType values[ItemsPerThread]; @@ -1056,7 +1056,7 @@ public: //! The most-significant bit index (exclusive) needed for key //! comparison (e.g., `(sizeof(float) + sizeof(long long int)) * 8`) template - _CCCL_DEVICE _CCCL_FORCEINLINE // + _CCCL_DEVICE_API _CCCL_FORCEINLINE // ::cuda::std::enable_if_t< // !::cuda::std::is_convertible_v> SortDescending(KeyT (&keys)[ItemsPerThread], DecomposerT decomposer, int begin_bit, int end_bit) @@ -1119,7 +1119,7 @@ public: //! the tuple is considered the most significant. The call operator must not //! modify members of the key. template - _CCCL_DEVICE _CCCL_FORCEINLINE // + _CCCL_DEVICE_API _CCCL_FORCEINLINE // ::cuda::std::enable_if_t< // !::cuda::std::is_convertible_v> SortDescending(KeyT (&keys)[ItemsPerThread], DecomposerT decomposer) @@ -1196,7 +1196,7 @@ public: //! //! @param[in] end_bit //! **[optional]** The past-the-end (most-significant) bit index needed for key comparison - _CCCL_DEVICE _CCCL_FORCEINLINE void SortDescending( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SortDescending( KeyT (&keys)[ItemsPerThread], ValueT (&values)[ItemsPerThread], int begin_bit = 0, int end_bit = sizeof(KeyT) * 8) { SortBlocked(keys, values, begin_bit, end_bit, ::cuda::std::true_type(), detail::bool_constant_v); @@ -1270,7 +1270,7 @@ public: //! The most-significant bit index (exclusive) needed for key //! comparison (e.g., `(sizeof(float) + sizeof(long long int)) * 8`) template - _CCCL_DEVICE _CCCL_FORCEINLINE // + _CCCL_DEVICE_API _CCCL_FORCEINLINE // ::cuda::std::enable_if_t< // !::cuda::std::is_convertible_v> SortDescending( @@ -1340,7 +1340,7 @@ public: //! the tuple is considered the most significant. The call operator must not //! modify members of the key. template - _CCCL_DEVICE _CCCL_FORCEINLINE // + _CCCL_DEVICE_API _CCCL_FORCEINLINE // ::cuda::std::enable_if_t< // !::cuda::std::is_convertible_v> SortDescending(KeyT (&keys)[ItemsPerThread], ValueT (&values)[ItemsPerThread], DecomposerT decomposer) @@ -1410,7 +1410,7 @@ public: //! //! @param[in] end_bit //! **[optional]** The past-the-end (most-significant) bit index needed for key comparison - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SortBlockedToStriped(KeyT (&keys)[ItemsPerThread], int begin_bit = 0, int end_bit = sizeof(KeyT) * 8) { NullType values[ItemsPerThread]; @@ -1480,7 +1480,7 @@ public: //! The most-significant bit index (exclusive) needed for key //! comparison (e.g., `(sizeof(float) + sizeof(long long int)) * 8`) template - _CCCL_DEVICE _CCCL_FORCEINLINE // + _CCCL_DEVICE_API _CCCL_FORCEINLINE // ::cuda::std::enable_if_t< // !::cuda::std::is_convertible_v> SortBlockedToStriped(KeyT (&keys)[ItemsPerThread], DecomposerT decomposer, int begin_bit, int end_bit) @@ -1544,7 +1544,7 @@ public: //! the tuple is considered the most significant. The call operator must not //! modify members of the key. template - _CCCL_DEVICE _CCCL_FORCEINLINE // + _CCCL_DEVICE_API _CCCL_FORCEINLINE // ::cuda::std::enable_if_t< // !::cuda::std::is_convertible_v> SortBlockedToStriped(KeyT (&keys)[ItemsPerThread], DecomposerT decomposer) @@ -1621,7 +1621,7 @@ public: //! //! @param[in] end_bit //! **[optional]** The past-the-end (most-significant) bit index needed for key comparison - _CCCL_DEVICE _CCCL_FORCEINLINE void SortBlockedToStriped( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SortBlockedToStriped( KeyT (&keys)[ItemsPerThread], ValueT (&values)[ItemsPerThread], int begin_bit = 0, int end_bit = sizeof(KeyT) * 8) { SortBlockedToStriped( @@ -1692,7 +1692,7 @@ public: //! The most-significant bit index (exclusive) needed for key //! comparison (e.g., `(sizeof(float) + sizeof(long long int)) * 8`) template - _CCCL_DEVICE _CCCL_FORCEINLINE // + _CCCL_DEVICE_API _CCCL_FORCEINLINE // ::cuda::std::enable_if_t< // !::cuda::std::is_convertible_v> SortBlockedToStriped( @@ -1758,7 +1758,7 @@ public: //! the tuple is considered the most significant. The call operator must not //! modify members of the key. template - _CCCL_DEVICE _CCCL_FORCEINLINE // + _CCCL_DEVICE_API _CCCL_FORCEINLINE // ::cuda::std::enable_if_t< // !::cuda::std::is_convertible_v> SortBlockedToStriped(KeyT (&keys)[ItemsPerThread], ValueT (&values)[ItemsPerThread], DecomposerT decomposer) @@ -1824,7 +1824,7 @@ public: //! //! @param[in] end_bit //! **[optional]** The past-the-end (most-significant) bit index needed for key comparison - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SortDescendingBlockedToStriped(KeyT (&keys)[ItemsPerThread], int begin_bit = 0, int end_bit = sizeof(KeyT) * 8) { NullType values[ItemsPerThread]; @@ -1893,7 +1893,7 @@ public: //! The most-significant bit index (exclusive) needed for key //! comparison (e.g., `(sizeof(float) + sizeof(long long int)) * 8`) template - _CCCL_DEVICE _CCCL_FORCEINLINE // + _CCCL_DEVICE_API _CCCL_FORCEINLINE // ::cuda::std::enable_if_t< // !::cuda::std::is_convertible_v> SortDescendingBlockedToStriped(KeyT (&keys)[ItemsPerThread], DecomposerT decomposer, int begin_bit, int end_bit) @@ -1957,7 +1957,7 @@ public: //! the tuple is considered the most significant. The call operator must not //! modify members of the key. template - _CCCL_DEVICE _CCCL_FORCEINLINE // + _CCCL_DEVICE_API _CCCL_FORCEINLINE // ::cuda::std::enable_if_t< // !::cuda::std::is_convertible_v> SortDescendingBlockedToStriped(KeyT (&keys)[ItemsPerThread], DecomposerT decomposer) @@ -2034,7 +2034,7 @@ public: //! //! @param[in] end_bit //! **[optional]** The past-the-end (most-significant) bit index needed for key comparison - _CCCL_DEVICE _CCCL_FORCEINLINE void SortDescendingBlockedToStriped( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SortDescendingBlockedToStriped( KeyT (&keys)[ItemsPerThread], ValueT (&values)[ItemsPerThread], int begin_bit = 0, int end_bit = sizeof(KeyT) * 8) { SortBlockedToStriped(keys, values, begin_bit, end_bit, ::cuda::std::true_type(), detail::bool_constant_v); @@ -2104,7 +2104,7 @@ public: //! The most-significant bit index (exclusive) needed for key //! comparison (e.g., `(sizeof(float) + sizeof(long long int)) * 8`) template - _CCCL_DEVICE _CCCL_FORCEINLINE // + _CCCL_DEVICE_API _CCCL_FORCEINLINE // ::cuda::std::enable_if_t< // !::cuda::std::is_convertible_v> SortDescendingBlockedToStriped( @@ -2170,7 +2170,7 @@ public: //! the tuple is considered the most significant. The call operator must not //! modify members of the key. template - _CCCL_DEVICE _CCCL_FORCEINLINE // + _CCCL_DEVICE_API _CCCL_FORCEINLINE // ::cuda::std::enable_if_t< // !::cuda::std::is_convertible_v> SortDescendingBlockedToStriped(KeyT (&keys)[ItemsPerThread], ValueT (&values)[ItemsPerThread], DecomposerT decomposer) diff --git a/cub/cub/block/block_raking_layout.cuh b/cub/cub/block/block_raking_layout.cuh index 7609095e4ac3..324e481c8c12 100644 --- a/cub/cub/block/block_raking_layout.cuh +++ b/cub/cub/block/block_raking_layout.cuh @@ -98,7 +98,7 @@ struct BlockRakingLayout /** * @brief Returns the location for the calling thread to place data into the grid */ - static _CCCL_DEVICE _CCCL_FORCEINLINE T* PlacementPtr(TempStorage& temp_storage, unsigned int linear_tid) + _CCCL_DEVICE_API static _CCCL_FORCEINLINE T* PlacementPtr(TempStorage& temp_storage, unsigned int linear_tid) { // Offset for partial unsigned int offset = linear_tid; @@ -116,7 +116,7 @@ struct BlockRakingLayout /** * @brief Returns the location for the calling thread to begin sequential raking */ - static _CCCL_DEVICE _CCCL_FORCEINLINE T* RakingPtr(TempStorage& temp_storage, unsigned int linear_tid) + _CCCL_DEVICE_API static _CCCL_FORCEINLINE T* RakingPtr(TempStorage& temp_storage, unsigned int linear_tid) { return temp_storage.Alias().buff + (linear_tid * (SEGMENT_LENGTH + USE_SEGMENT_PADDING)); } diff --git a/cub/cub/block/block_reduce.cuh b/cub/cub/block/block_reduce.cuh index 191c11b5d43d..d2c1f5810735 100644 --- a/cub/cub/block/block_reduce.cuh +++ b/cub/cub/block/block_reduce.cuh @@ -177,7 +177,7 @@ namespace detail } } // namespace detail -inline ::std::ostream& operator<<(::std::ostream& os, BlockReduceAlgorithm algo) +_CCCL_HOST_API inline ::std::ostream& operator<<(::std::ostream& os, BlockReduceAlgorithm algo) { return os << _CUB::detail::to_string(algo); } @@ -190,7 +190,7 @@ template <::cuda::std::same_as CharT> struct std::formatter<_CUB::BlockReduceAlgorithm, CharT> : formatter { template - auto format(const _CUB::BlockReduceAlgorithm& algo, FmtCtx& ctx) const + _CCCL_HOST_API auto format(const _CUB::BlockReduceAlgorithm& algo, FmtCtx& ctx) const { return formatter::format(_CUB::detail::to_string(algo), ctx); } @@ -330,7 +330,7 @@ class BlockReduce using _TempStorage _CCCL_NODEBUG = typename InternalBlockReduce::TempStorage; /// Internal storage allocator - _CCCL_DEVICE _CCCL_FORCEINLINE _TempStorage& PrivateStorage() + _CCCL_DEVICE_API _CCCL_FORCEINLINE _TempStorage& PrivateStorage() { __shared__ _TempStorage private_storage; return private_storage; @@ -356,7 +356,7 @@ public: //! .. versionadded:: 2.2.0 //! First appears in CUDA Toolkit 12.3. //! @endrst - _CCCL_DEVICE _CCCL_FORCEINLINE BlockReduce() + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockReduce() : temp_storage(PrivateStorage()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} @@ -372,7 +372,7 @@ public: * @param[in] temp_storage * Reference to memory allocation having layout type TempStorage */ - _CCCL_DEVICE _CCCL_FORCEINLINE BlockReduce(TempStorage& temp_storage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockReduce(TempStorage& temp_storage) : temp_storage(temp_storage.Alias()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} @@ -429,7 +429,7 @@ public: //! @param[in] reduction_op //! Binary reduction functor template - _CCCL_DEVICE _CCCL_FORCEINLINE T Reduce(T input, ReductionOp reduction_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE T Reduce(T input, ReductionOp reduction_op) { return InternalBlockReduce(temp_storage).template Reduce(input, BLOCK_THREADS, reduction_op); } @@ -486,7 +486,7 @@ public: //! @param[in] reduction_op //! Binary reduction functor template - _CCCL_DEVICE _CCCL_FORCEINLINE T Reduce(T (&inputs)[ItemsPerThread], ReductionOp reduction_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE T Reduce(T (&inputs)[ItemsPerThread], ReductionOp reduction_op) { // Reduce partials T partial = _CUB::ThreadReduce(inputs, reduction_op); @@ -544,7 +544,7 @@ public: //! @param[in] num_valid //! Number of threads containing valid elements (may be less than BLOCK_THREADS) template - _CCCL_DEVICE _CCCL_FORCEINLINE T Reduce(T input, ReductionOp reduction_op, int num_valid) + _CCCL_DEVICE_API _CCCL_FORCEINLINE T Reduce(T input, ReductionOp reduction_op, int num_valid) { // Determine if we skip bounds checking if (num_valid >= BLOCK_THREADS) @@ -599,7 +599,7 @@ public: //! //! @param[in] input //! Calling thread's input - _CCCL_DEVICE _CCCL_FORCEINLINE T Sum(T input) + _CCCL_DEVICE_API _CCCL_FORCEINLINE T Sum(T input) { return InternalBlockReduce(temp_storage).template Sum(input, BLOCK_THREADS); } @@ -650,7 +650,7 @@ public: //! @param[in] inputs //! Calling thread's input segment template - _CCCL_DEVICE _CCCL_FORCEINLINE T Sum(T (&inputs)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE T Sum(T (&inputs)[ItemsPerThread]) { // Reduce partials T partial = _CUB::ThreadReduce(inputs, ::cuda::std::plus<>{}); @@ -702,7 +702,7 @@ public: //! //! @param[in] num_valid //! Number of threads containing valid elements (may be less than BLOCK_THREADS) - _CCCL_DEVICE _CCCL_FORCEINLINE T Sum(T input, int num_valid) + _CCCL_DEVICE_API _CCCL_FORCEINLINE T Sum(T input, int num_valid) { // Determine if we skip bounds checking if (num_valid >= BLOCK_THREADS) diff --git a/cub/cub/block/block_run_length_decode.cuh b/cub/cub/block/block_run_length_decode.cuh index 60c7bc9635c6..e93d2132a9d9 100644 --- a/cub/cub/block/block_run_length_decode.cuh +++ b/cub/cub/block/block_run_length_decode.cuh @@ -162,7 +162,7 @@ class BlockRunLengthDecode #endif // _CCCL_DOXYGEN_INVOKED /// Internal storage allocator (used when the user does not provide pre-allocated shared memory) - _CCCL_DEVICE _CCCL_FORCEINLINE _TempStorage& PrivateStorage() + _CCCL_DEVICE_API _CCCL_FORCEINLINE _TempStorage& PrivateStorage() { __shared__ _TempStorage private_storage; return private_storage; @@ -190,7 +190,7 @@ public: //! First appears in CUDA Toolkit 12.3. //! @endrst template - _CCCL_DEVICE _CCCL_FORCEINLINE BlockRunLengthDecode( + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockRunLengthDecode( TempStorage& temp_storage, ItemT (&run_values)[RunsPerThread], RunLengthT (&run_lengths)[RunsPerThread], @@ -210,7 +210,7 @@ public: //! First appears in CUDA Toolkit 12.3. //! @endrst template - _CCCL_DEVICE _CCCL_FORCEINLINE BlockRunLengthDecode( + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockRunLengthDecode( TempStorage& temp_storage, ItemT (&run_values)[RunsPerThread], UserRunOffsetT (&run_offsets)[RunsPerThread]) : temp_storage(temp_storage.Alias()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) @@ -227,7 +227,7 @@ public: * @endrst */ template - _CCCL_DEVICE _CCCL_FORCEINLINE BlockRunLengthDecode( + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockRunLengthDecode( ItemT (&run_values)[RunsPerThread], RunLengthT (&run_lengths)[RunsPerThread], TotalDecodedSizeT& total_decoded_size) : temp_storage(PrivateStorage()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) @@ -244,7 +244,7 @@ public: * @endrst */ template - _CCCL_DEVICE _CCCL_FORCEINLINE + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockRunLengthDecode(ItemT (&run_values)[RunsPerThread], UserRunOffsetT (&run_offsets)[RunsPerThread]) : temp_storage(PrivateStorage()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) @@ -268,7 +268,7 @@ private: * Search key */ template - _CCCL_DEVICE _CCCL_FORCEINLINE OffsetT StaticUpperBound(InputIteratorT input, OffsetT num_items, T val) + _CCCL_DEVICE_API _CCCL_FORCEINLINE OffsetT StaticUpperBound(InputIteratorT input, OffsetT num_items, T val) { OffsetT lower_bound = 0; OffsetT upper_bound = num_items; @@ -293,7 +293,7 @@ private: } template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InitWithRunOffsets(ItemT (&run_values)[RunsPerThread], RunOffsetT (&run_offsets)[RunsPerThread]) { // Keep the runs' items and the offsets of each run's beginning in the temporary storage @@ -312,7 +312,7 @@ private: } template - _CCCL_DEVICE _CCCL_FORCEINLINE void InitWithRunLengths( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InitWithRunLengths( ItemT (&run_values)[RunsPerThread], RunLengthT (&run_lengths)[RunsPerThread], TotalDecodedSizeT& total_decoded_size) { // Compute the offset for the beginning of each run @@ -357,7 +357,7 @@ public: * in undefined behavior. */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void RunLengthDecode( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void RunLengthDecode( ItemT (&decoded_items)[DecodedItemsPerThread], RelativeOffsetT (&item_offsets)[DecodedItemsPerThread], DecodedOffsetT from_decoded_offset = 0) @@ -424,7 +424,7 @@ public: * \param[in] from_decoded_offset If invoked with from_decoded_offset that is larger than total_decoded_size results * in undefined behavior. */ - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void RunLengthDecode(ItemT (&decoded_items)[DecodedItemsPerThread], DecodedOffsetT from_decoded_offset = 0) { DecodedOffsetT item_offsets[DecodedItemsPerThread]; diff --git a/cub/cub/block/block_scan.cuh b/cub/cub/block/block_scan.cuh index f2c69aab74a5..88dd7c132eb6 100644 --- a/cub/cub/block/block_scan.cuh +++ b/cub/cub/block/block_scan.cuh @@ -124,7 +124,7 @@ namespace detail } } // namespace detail -inline ::std::ostream& operator<<(::std::ostream& os, BlockScanAlgorithm algo) +_CCCL_HOST_API inline ::std::ostream& operator<<(::std::ostream& os, BlockScanAlgorithm algo) { return os << _CUB::detail::to_string(algo); } @@ -137,7 +137,7 @@ template <::cuda::std::same_as CharT> struct std::formatter<_CUB::BlockScanAlgorithm, CharT> : formatter { template - auto format(const _CUB::BlockScanAlgorithm& algo, FmtCtx& ctx) const + _CCCL_HOST_API auto format(const _CUB::BlockScanAlgorithm& algo, FmtCtx& ctx) const { return formatter::format(_CUB::detail::to_string(algo), ctx); } @@ -266,7 +266,7 @@ class BlockScan unsigned int linear_tid; /// Internal storage allocator - _CCCL_DEVICE _CCCL_FORCEINLINE _TempStorage& PrivateStorage() + _CCCL_DEVICE_API _CCCL_FORCEINLINE _TempStorage& PrivateStorage() { __shared__ _TempStorage private_storage; return private_storage; @@ -285,7 +285,7 @@ public: //! .. versionadded:: 2.2.0 //! First appears in CUDA Toolkit 12.3. //! @endrst - _CCCL_DEVICE _CCCL_FORCEINLINE BlockScan() + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockScan() : temp_storage(PrivateStorage()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} @@ -301,7 +301,7 @@ public: * @param[in] temp_storage * Reference to memory allocation having layout type TempStorage */ - _CCCL_DEVICE _CCCL_FORCEINLINE BlockScan(TempStorage& temp_storage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockScan(TempStorage& temp_storage) : temp_storage(temp_storage.Alias()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} @@ -344,7 +344,7 @@ public: //! //! @param[out] output //! Calling thread's output item (may be aliased to `input`) - _CCCL_DEVICE _CCCL_FORCEINLINE void ExclusiveSum(T input, T& output) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveSum(T input, T& output) { T initial_value{}; @@ -390,7 +390,7 @@ public: //! //! @param[out] block_aggregate //! block-wide aggregate reduction of input items - _CCCL_DEVICE _CCCL_FORCEINLINE void ExclusiveSum(T input, T& output, T& block_aggregate) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveSum(T input, T& output, T& block_aggregate) { T initial_value{}; @@ -455,7 +455,8 @@ public: //! the logical input sequence. //! @endrst template - _CCCL_DEVICE _CCCL_FORCEINLINE void ExclusiveSum(T input, T& output, BlockPrefixCallbackOp& block_prefix_callback_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void + ExclusiveSum(T input, T& output, BlockPrefixCallbackOp& block_prefix_callback_op) { ExclusiveScan(input, output, ::cuda::std::plus<>{}, block_prefix_callback_op); } @@ -506,7 +507,7 @@ public: //! @param[out] output //! Calling thread's output items (may be aliased to `input`) template - _CCCL_DEVICE _CCCL_FORCEINLINE void ExclusiveSum(T (&input)[ItemsPerThread], T (&output)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveSum(T (&input)[ItemsPerThread], T (&output)[ItemsPerThread]) { T initial_value{}; @@ -560,7 +561,7 @@ public: //! @param[out] block_aggregate //! block-wide aggregate reduction of input items template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveSum(T (&input)[ItemsPerThread], T (&output)[ItemsPerThread], T& block_aggregate) { // Reduce consecutive thread items in registers @@ -634,7 +635,7 @@ public: //! the logical input sequence. //! @endrst template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveSum(T (&input)[ItemsPerThread], T (&output)[ItemsPerThread], BlockPrefixCallbackOp& block_prefix_callback_op) { ExclusiveScan(input, output, ::cuda::std::plus<>{}, block_prefix_callback_op); @@ -689,7 +690,7 @@ public: //! @param[in] scan_op //! Binary scan functor template - _CCCL_DEVICE _CCCL_FORCEINLINE void ExclusiveScan(T input, T& output, T initial_value, ScanOp scan_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveScan(T input, T& output, T initial_value, ScanOp scan_op) { InternalBlockScan(temp_storage).ExclusiveScan(input, output, initial_value, scan_op); } @@ -750,7 +751,7 @@ public: //! @param[out] block_aggregate //! block-wide aggregate reduction of input items template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveScan(T input, T& output, T initial_value, ScanOp scan_op, T& block_aggregate) { InternalBlockScan(temp_storage).ExclusiveScan(input, output, initial_value, scan_op, block_aggregate); @@ -858,7 +859,7 @@ public: //! the logical input sequence. //! @endrst template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveScan(T input, T& output, ScanOp scan_op, BlockPrefixCallbackOp& block_prefix_callback_op) { InternalBlockScan(temp_storage).ExclusiveScan(input, output, scan_op, block_prefix_callback_op); @@ -921,7 +922,7 @@ public: //! @param[in] scan_op //! Binary scan functor template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveScan(T (&input)[ItemsPerThread], T (&output)[ItemsPerThread], T initial_value, ScanOp scan_op) { // Reduce consecutive thread items in registers @@ -1011,7 +1012,7 @@ public: //! @param[out] block_aggregate //! block-wide aggregate reduction of input items template - _CCCL_DEVICE _CCCL_FORCEINLINE void ExclusiveScan( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveScan( T (&input)[ItemsPerThread], T (&output)[ItemsPerThread], T initial_value, ScanOp scan_op, T& block_aggregate) { // Reduce consecutive thread items in registers @@ -1095,7 +1096,7 @@ public: //! the logical input sequence. //! @endrst template - _CCCL_DEVICE _CCCL_FORCEINLINE void ExclusiveScan( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveScan( T (&input)[ItemsPerThread], T (&output)[ItemsPerThread], ScanOp scan_op, @@ -1140,7 +1141,7 @@ public: //! @param[in] scan_op //! Binary scan functor template - _CCCL_DEVICE _CCCL_FORCEINLINE void ExclusiveScan(T input, T& output, ScanOp scan_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveScan(T input, T& output, ScanOp scan_op) { InternalBlockScan(temp_storage).ExclusiveScan(input, output, scan_op); } @@ -1172,7 +1173,7 @@ public: //! @param[out] block_aggregate //! block-wide aggregate reduction of input items template - _CCCL_DEVICE _CCCL_FORCEINLINE void ExclusiveScan(T input, T& output, ScanOp scan_op, T& block_aggregate) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveScan(T input, T& output, ScanOp scan_op, T& block_aggregate) { InternalBlockScan(temp_storage).ExclusiveScan(input, output, scan_op, block_aggregate); } @@ -1208,7 +1209,7 @@ public: //! @param[in] scan_op //! Binary scan functor template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveScan(T (&input)[ItemsPerThread], T (&output)[ItemsPerThread], ScanOp scan_op) { // Reduce consecutive thread items in registers @@ -1252,7 +1253,7 @@ public: //! @param[out] block_aggregate //! block-wide aggregate reduction of input items template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveScan(T (&input)[ItemsPerThread], T (&output)[ItemsPerThread], ScanOp scan_op, T& block_aggregate) { // Reduce consecutive thread items in registers @@ -1303,7 +1304,7 @@ public: //! //! @param[out] output //! Calling thread's output item (may be aliased to `input`) - _CCCL_DEVICE _CCCL_FORCEINLINE void InclusiveSum(T input, T& output) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveSum(T input, T& output) { InclusiveScan(input, output, ::cuda::std::plus<>{}); } @@ -1345,7 +1346,7 @@ public: //! //! @param[out] block_aggregate //! block-wide aggregate reduction of input items - _CCCL_DEVICE _CCCL_FORCEINLINE void InclusiveSum(T input, T& output, T& block_aggregate) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveSum(T input, T& output, T& block_aggregate) { InclusiveScan(input, output, ::cuda::std::plus<>{}, block_aggregate); } @@ -1445,7 +1446,8 @@ public: //! to the logical input sequence. //! @endrst template - _CCCL_DEVICE _CCCL_FORCEINLINE void InclusiveSum(T input, T& output, BlockPrefixCallbackOp& block_prefix_callback_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void + InclusiveSum(T input, T& output, BlockPrefixCallbackOp& block_prefix_callback_op) { InclusiveScan(input, output, ::cuda::std::plus<>{}, block_prefix_callback_op); } @@ -1493,7 +1495,7 @@ public: //! @param[out] output //! Calling thread's output items (may be aliased to `input`) template - _CCCL_DEVICE _CCCL_FORCEINLINE void InclusiveSum(T (&input)[ItemsPerThread], T (&output)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveSum(T (&input)[ItemsPerThread], T (&output)[ItemsPerThread]) { if constexpr (ItemsPerThread == 1) { @@ -1558,7 +1560,7 @@ public: //! @param[out] block_aggregate //! block-wide aggregate reduction of input items template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveSum(T (&input)[ItemsPerThread], T (&output)[ItemsPerThread], T& block_aggregate) { if constexpr (ItemsPerThread == 1) @@ -1642,7 +1644,7 @@ public: //! logical input sequence. //! @endrst template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveSum(T (&input)[ItemsPerThread], T (&output)[ItemsPerThread], BlockPrefixCallbackOp& block_prefix_callback_op) { if constexpr (ItemsPerThread == 1) @@ -1708,7 +1710,7 @@ public: //! @param[in] scan_op //! Binary scan functor template - _CCCL_DEVICE _CCCL_FORCEINLINE void InclusiveScan(T input, T& output, ScanOp scan_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScan(T input, T& output, ScanOp scan_op) { InternalBlockScan(temp_storage).InclusiveScan(input, output, scan_op); } @@ -1773,7 +1775,7 @@ public: //! @param[out] block_aggregate //! Block-wide aggregate reduction of input items template - _CCCL_DEVICE _CCCL_FORCEINLINE void InclusiveScan(T input, T& output, ScanOp scan_op, T& block_aggregate) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScan(T input, T& output, ScanOp scan_op, T& block_aggregate) { InternalBlockScan(temp_storage).InclusiveScan(input, output, scan_op, block_aggregate); } @@ -1844,7 +1846,7 @@ public: //! the logical input sequence. //! @endrst template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScan(T input, T& output, ScanOp scan_op, BlockPrefixCallbackOp& block_prefix_callback_op) { InternalBlockScan(temp_storage).InclusiveScan(input, output, scan_op, block_prefix_callback_op); @@ -1901,7 +1903,7 @@ public: //! @param[in] scan_op //! Binary scan functor template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScan(T (&input)[ItemsPerThread], T (&output)[ItemsPerThread], ScanOp scan_op) { if constexpr (ItemsPerThread == 1) @@ -1966,7 +1968,7 @@ public: //! @param[in] scan_op //! Binary scan functor template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScan(T (&input)[ItemsPerThread], T (&output)[ItemsPerThread], T initial_value, ScanOp scan_op) { // Reduce consecutive thread items in registers @@ -2045,7 +2047,7 @@ public: //! @param[out] block_aggregate //! Block-wide aggregate reduction of input items template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScan(T (&input)[ItemsPerThread], T (&output)[ItemsPerThread], ScanOp scan_op, T& block_aggregate) { if (ItemsPerThread == 1) @@ -2121,7 +2123,7 @@ public: //! @param[out] block_aggregate //! Block-wide aggregate reduction of input items template - _CCCL_DEVICE _CCCL_FORCEINLINE void InclusiveScan( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScan( T (&input)[ItemsPerThread], T (&output)[ItemsPerThread], T initial_value, ScanOp scan_op, T& block_aggregate) { // Reduce consecutive thread items in registers @@ -2250,7 +2252,7 @@ public: //! the logical input sequence. //! @endrst template - _CCCL_DEVICE _CCCL_FORCEINLINE void InclusiveScan( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScan( T (&input)[ItemsPerThread], T (&output)[ItemsPerThread], ScanOp scan_op, diff --git a/cub/cub/block/block_shuffle.cuh b/cub/cub/block/block_shuffle.cuh index d3681856bce0..5a9d82e1a0e6 100644 --- a/cub/cub/block/block_shuffle.cuh +++ b/cub/cub/block/block_shuffle.cuh @@ -75,7 +75,7 @@ private: unsigned int linear_tid; /// Internal storage allocator - _CCCL_DEVICE _CCCL_FORCEINLINE _TempStorage& PrivateStorage() + _CCCL_DEVICE_API _CCCL_FORCEINLINE _TempStorage& PrivateStorage() { __shared__ _TempStorage private_storage; return private_storage; @@ -91,7 +91,7 @@ public: //! .. versionadded:: 2.2.0 //! First appears in CUDA Toolkit 12.3. //! @endrst - _CCCL_DEVICE _CCCL_FORCEINLINE BlockShuffle() + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockShuffle() : temp_storage(PrivateStorage()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} @@ -108,7 +108,7 @@ public: * @param[in] temp_storage * Reference to memory allocation having layout type TempStorage */ - _CCCL_DEVICE _CCCL_FORCEINLINE BlockShuffle(TempStorage& temp_storage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockShuffle(TempStorage& temp_storage) : temp_storage(temp_storage.Alias()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} @@ -142,7 +142,7 @@ public: //! //! @param[in] distance //! Offset distance (may be negative) - _CCCL_DEVICE _CCCL_FORCEINLINE void Offset(T input, T& output, int distance = 1) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Offset(T input, T& output, int distance = 1) { temp_storage[linear_tid] = input; @@ -176,7 +176,7 @@ public: //! //! @param[in] distance //! Offset distance (`0 < distance < `BLOCK_THREADS`) - _CCCL_DEVICE _CCCL_FORCEINLINE void Rotate(T input, T& output, unsigned int distance = 1) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Rotate(T input, T& output, unsigned int distance = 1) { temp_storage[linear_tid] = input; @@ -212,7 +212,7 @@ public: //! The item ``prev[0]`` is not updated for *thread*\ :sub:`0`. //! @endrst template - _CCCL_DEVICE _CCCL_FORCEINLINE void Up(T (&input)[ItemsPerThread], T (&prev)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Up(T (&input)[ItemsPerThread], T (&prev)[ItemsPerThread]) { temp_storage[linear_tid] = input[ItemsPerThread - 1]; @@ -257,7 +257,7 @@ public: //! The item ``input[ItemsPerThread - 1]`` from *thread*\ :sub:`BLOCK_THREADS - 1`, provided to all threads //! @endrst template - _CCCL_DEVICE _CCCL_FORCEINLINE void Up(T (&input)[ItemsPerThread], T (&prev)[ItemsPerThread], T& block_suffix) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Up(T (&input)[ItemsPerThread], T (&prev)[ItemsPerThread], T& block_suffix) { Up(input, prev); block_suffix = temp_storage[BLOCK_THREADS - 1]; @@ -284,7 +284,7 @@ public: //! The value ``prev[0]`` is not updated for *thread*\ :sub:`BLOCK_THREADS - 1`. //! @endrst template - _CCCL_DEVICE _CCCL_FORCEINLINE void Down(T (&input)[ItemsPerThread], T (&prev)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Down(T (&input)[ItemsPerThread], T (&prev)[ItemsPerThread]) { temp_storage[linear_tid] = input[0]; @@ -328,7 +328,7 @@ public: //! The item ``input[0]`` from *thread*\ :sub:`0`, provided to all threads //! @endrst template - _CCCL_DEVICE _CCCL_FORCEINLINE void Down(T (&input)[ItemsPerThread], T (&prev)[ItemsPerThread], T& block_prefix) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Down(T (&input)[ItemsPerThread], T (&prev)[ItemsPerThread], T& block_prefix) { Down(input, prev); block_prefix = temp_storage[0]; diff --git a/cub/cub/block/block_store.cuh b/cub/cub/block/block_store.cuh index 0b0e68f15892..ca0657d8eac5 100644 --- a/cub/cub/block/block_store.cuh +++ b/cub/cub/block/block_store.cuh @@ -73,7 +73,7 @@ CUB_NAMESPACE_BEGIN //! @param[in] items //! Data to store template -_CCCL_DEVICE _CCCL_FORCEINLINE void +_CCCL_DEVICE_API _CCCL_FORCEINLINE void StoreDirectBlocked(int linear_tid, OutputIteratorT block_itr, T (&items)[ItemsPerThread]) { OutputIteratorT thread_itr = block_itr + (linear_tid * ItemsPerThread); // NOLINT(bugprone-misplaced-widening-cast) @@ -118,7 +118,7 @@ StoreDirectBlocked(int linear_tid, OutputIteratorT block_itr, T (&items)[ItemsPe //! @param[in] valid_items //! Number of valid items to write template -_CCCL_DEVICE _CCCL_FORCEINLINE void +_CCCL_DEVICE_API _CCCL_FORCEINLINE void StoreDirectBlocked(int linear_tid, OutputIteratorT block_itr, T (&items)[ItemsPerThread], int valid_items) { OutputIteratorT thread_itr = block_itr + (linear_tid * ItemsPerThread); // NOLINT(bugprone-misplaced-widening-cast) @@ -170,7 +170,7 @@ StoreDirectBlocked(int linear_tid, OutputIteratorT block_itr, T (&items)[ItemsPe //! @param[in] items //! Data to store template -_CCCL_DEVICE _CCCL_FORCEINLINE void +_CCCL_DEVICE_API _CCCL_FORCEINLINE void StoreDirectBlockedVectorized(int linear_tid, T* block_ptr, T (&items)[ItemsPerThread]) { // Find biggest memory access word that T is a whole multiple of @@ -242,7 +242,7 @@ StoreDirectBlockedVectorized(int linear_tid, T* block_ptr, T (&items)[ItemsPerTh //! @param[in] items //! Data to store template -_CCCL_DEVICE _CCCL_FORCEINLINE void +_CCCL_DEVICE_API _CCCL_FORCEINLINE void StoreDirectStriped(int linear_tid, OutputIteratorT block_itr, T (&items)[ItemsPerThread]) { OutputIteratorT thread_itr = block_itr + linear_tid; @@ -290,7 +290,7 @@ StoreDirectStriped(int linear_tid, OutputIteratorT block_itr, T (&items)[ItemsPe //! @param[in] valid_items //! Number of valid items to write template -_CCCL_DEVICE _CCCL_FORCEINLINE void +_CCCL_DEVICE_API _CCCL_FORCEINLINE void StoreDirectStriped(int linear_tid, OutputIteratorT block_itr, T (&items)[ItemsPerThread], int valid_items) { OutputIteratorT thread_itr = block_itr + linear_tid; @@ -344,7 +344,7 @@ StoreDirectStriped(int linear_tid, OutputIteratorT block_itr, T (&items)[ItemsPe //! @param[out] items //! Data to load template -_CCCL_DEVICE _CCCL_FORCEINLINE void +_CCCL_DEVICE_API _CCCL_FORCEINLINE void StoreDirectWarpStriped(int linear_tid, OutputIteratorT block_itr, T (&items)[ItemsPerThread]) { const int tid = linear_tid & (detail::warp_threads - 1); @@ -398,7 +398,7 @@ StoreDirectWarpStriped(int linear_tid, OutputIteratorT block_itr, T (&items)[Ite //! @param[in] valid_items //! Number of valid items to write template -_CCCL_DEVICE _CCCL_FORCEINLINE void +_CCCL_DEVICE_API _CCCL_FORCEINLINE void StoreDirectWarpStriped(int linear_tid, OutputIteratorT block_itr, T (&items)[ItemsPerThread], int valid_items) { const int tid = linear_tid & (detail::warp_threads - 1); @@ -577,7 +577,7 @@ namespace detail } } // namespace detail -inline ::std::ostream& operator<<(::std::ostream& os, BlockStoreAlgorithm algo) +_CCCL_HOST_API inline ::std::ostream& operator<<(::std::ostream& os, BlockStoreAlgorithm algo) { return os << _CUB::detail::to_string(algo); } @@ -590,7 +590,7 @@ template <::cuda::std::same_as CharT> struct std::formatter<_CUB::BlockStoreAlgorithm, CharT> : formatter { template - auto format(const _CUB::BlockStoreAlgorithm& algo, FmtCtx& ctx) const + _CCCL_HOST_API auto format(const _CUB::BlockStoreAlgorithm& algo, FmtCtx& ctx) const { return formatter::format(_CUB::detail::to_string(algo), ctx); } @@ -734,7 +734,7 @@ class BlockStore using _TempStorage _CCCL_NODEBUG = decltype(temp_storage_helper()); - _CCCL_DEVICE _CCCL_FORCEINLINE _TempStorage& PrivateStorage() + _CCCL_DEVICE_API _CCCL_FORCEINLINE _TempStorage& PrivateStorage() { __shared__ _TempStorage private_storage; return private_storage; @@ -759,7 +759,7 @@ public: * First appears in CUDA Toolkit 12.3. * @endrst */ - _CCCL_DEVICE _CCCL_FORCEINLINE BlockStore() + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockStore() : temp_storage(PrivateStorage()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} @@ -775,7 +775,7 @@ public: * @param[in] temp_storage * Reference to memory allocation having layout type TempStorage */ - _CCCL_DEVICE _CCCL_FORCEINLINE BlockStore(TempStorage& temp_storage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockStore(TempStorage& temp_storage) : temp_storage(temp_storage.Alias()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} @@ -833,7 +833,7 @@ public: //! @param[in] items //! Data to store template - _CCCL_DEVICE _CCCL_FORCEINLINE void Store(OutputIteratorT block_itr, T (&items)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Store(OutputIteratorT block_itr, T (&items)[ItemsPerThread]) { if constexpr (Algorithm == BLOCK_STORE_DIRECT) { @@ -923,7 +923,7 @@ public: //! @param[in] valid_items //! Number of valid items to write template - _CCCL_DEVICE _CCCL_FORCEINLINE void Store(OutputIteratorT block_itr, T (&items)[ItemsPerThread], int valid_items) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Store(OutputIteratorT block_itr, T (&items)[ItemsPerThread], int valid_items) { if constexpr (Algorithm == BLOCK_STORE_DIRECT || Algorithm == BLOCK_STORE_VECTORIZE) { diff --git a/cub/cub/block/radix_rank_sort_operations.cuh b/cub/cub/block/radix_rank_sort_operations.cuh index 111b79530b71..be2f0de676a8 100644 --- a/cub/cub/block/radix_rank_sort_operations.cuh +++ b/cub/cub/block/radix_rank_sort_operations.cuh @@ -58,7 +58,7 @@ struct BaseDigitExtractor using TraitsT = Traits; using UnsignedBits = typename TraitsT::UnsignedBits; - static _CCCL_HOST_DEVICE _CCCL_FORCEINLINE UnsignedBits ProcessFloatMinusZero(UnsignedBits key) + _CCCL_HOST_DEVICE_API static _CCCL_FORCEINLINE UnsignedBits ProcessFloatMinusZero(UnsignedBits key) { return key; } @@ -70,7 +70,7 @@ struct BaseDigitExtractor using TraitsT = Traits; using UnsignedBits = typename TraitsT::UnsignedBits; - static _CCCL_HOST_DEVICE _CCCL_FORCEINLINE UnsignedBits ProcessFloatMinusZero(UnsignedBits key) + _CCCL_HOST_DEVICE_API static _CCCL_FORCEINLINE UnsignedBits ProcessFloatMinusZero(UnsignedBits key) { const UnsignedBits TWIDDLED_MINUS_ZERO_BITS = TraitsT::TwiddleIn(UnsignedBits(1) << UnsignedBits((8 * sizeof(UnsignedBits)) - 1)); @@ -89,13 +89,13 @@ struct BFEDigitExtractor : BaseDigitExtractor ::cuda::std::uint32_t bit_start; ::cuda::std::uint32_t num_bits; - explicit _CCCL_DEVICE _CCCL_FORCEINLINE + explicit _CCCL_DEVICE_API _CCCL_FORCEINLINE BFEDigitExtractor(::cuda::std::uint32_t bit_start = 0, ::cuda::std::uint32_t num_bits = 0) : bit_start(bit_start) , num_bits(num_bits) {} - _CCCL_DEVICE _CCCL_FORCEINLINE ::cuda::std::uint32_t Digit(UnsignedBits key) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE ::cuda::std::uint32_t Digit(UnsignedBits key) const { return ::cuda::bitfield_extract(this->ProcessFloatMinusZero(key), bit_start, num_bits); } @@ -111,13 +111,13 @@ struct ShiftDigitExtractor : BaseDigitExtractor ::cuda::std::uint32_t bit_start; ::cuda::std::uint32_t mask; // NOLINT(modernize-use-default-member-init) - explicit _CCCL_DEVICE _CCCL_FORCEINLINE + explicit _CCCL_DEVICE_API _CCCL_FORCEINLINE ShiftDigitExtractor(::cuda::std::uint32_t bit_start = 0, ::cuda::std::uint32_t num_bits = 0) : bit_start(bit_start) , mask((1 << num_bits) - 1) {} - _CCCL_DEVICE _CCCL_FORCEINLINE ::cuda::std::uint32_t Digit(UnsignedBits key) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE ::cuda::std::uint32_t Digit(UnsignedBits key) const { return ::cuda::std::uint32_t(this->ProcessFloatMinusZero(key) >> UnsignedBits(bit_start)) & mask; } @@ -129,14 +129,14 @@ namespace detail struct identity_decomposer_t { template - _CCCL_HOST_DEVICE T& operator()(T& key) const + _CCCL_HOST_DEVICE_API T& operator()(T& key) const { return key; } }; template -_CCCL_HOST_DEVICE void +_CCCL_HOST_DEVICE_API void for_each_member_impl(F f, const ::cuda::std::tuple& tpl, ::cuda::std::index_sequence) { static_assert(sizeof...(Ts), "Empty aggregates are not supported"); @@ -152,7 +152,7 @@ for_each_member_impl(F f, const ::cuda::std::tuple& tpl, ::cuda::std::in } template -_CCCL_HOST_DEVICE void for_each_member(F f, DecomposerT decomposer, T& aggregate) +_CCCL_HOST_DEVICE_API void for_each_member(F f, DecomposerT decomposer, T& aggregate) { const auto& tuple_of_refs = decomposer(aggregate); constexpr int tuple_size = ::cuda::std::tuple_size_v<::cuda::std::remove_reference_t>; @@ -192,12 +192,12 @@ struct bit_ordered_conversion_policy_t { using bit_ordered_type = typename Traits::UnsignedBits; - static _CCCL_HOST_DEVICE bit_ordered_type to_bit_ordered(detail::identity_decomposer_t, bit_ordered_type val) + _CCCL_HOST_DEVICE_API static bit_ordered_type to_bit_ordered(detail::identity_decomposer_t, bit_ordered_type val) { return Traits::TwiddleIn(val); } - static _CCCL_HOST_DEVICE bit_ordered_type from_bit_ordered(detail::identity_decomposer_t, bit_ordered_type val) + _CCCL_HOST_DEVICE_API static bit_ordered_type from_bit_ordered(detail::identity_decomposer_t, bit_ordered_type val) { return Traits::TwiddleOut(val); } @@ -208,7 +208,7 @@ struct bit_ordered_inversion_policy_t { using bit_ordered_type = typename Traits::UnsignedBits; - static _CCCL_HOST_DEVICE bit_ordered_type inverse(detail::identity_decomposer_t, bit_ordered_type val) + _CCCL_HOST_DEVICE_API static bit_ordered_type inverse(detail::identity_decomposer_t, bit_ordered_type val) { return ~val; } @@ -224,23 +224,23 @@ struct traits_t template using digit_extractor_t = FundamentalExtractorT; - static _CCCL_HOST_DEVICE bit_ordered_type min_raw_binary_key(detail::identity_decomposer_t) + _CCCL_HOST_DEVICE_API static bit_ordered_type min_raw_binary_key(detail::identity_decomposer_t) { return Traits::LOWEST_KEY; } - static _CCCL_HOST_DEVICE bit_ordered_type max_raw_binary_key(detail::identity_decomposer_t) + _CCCL_HOST_DEVICE_API static bit_ordered_type max_raw_binary_key(detail::identity_decomposer_t) { return Traits::MAX_KEY; } - static _CCCL_HOST_DEVICE int default_end_bit(detail::identity_decomposer_t) + _CCCL_HOST_DEVICE_API static int default_end_bit(detail::identity_decomposer_t) { return sizeof(T) * 8; } template - static _CCCL_HOST_DEVICE digit_extractor_t + _CCCL_HOST_DEVICE_API static digit_extractor_t digit_extractor(int begin_bit, int num_bits, detail::identity_decomposer_t) { return FundamentalExtractorT(begin_bit, num_bits); @@ -257,7 +257,7 @@ struct min_raw_binary_key_f DecomposerT decomposer; template - _CCCL_HOST_DEVICE void operator()(T& field) + _CCCL_HOST_DEVICE_API void operator()(T& field) { using traits = traits_t; using bit_ordered_type = typename traits::bit_ordered_type; @@ -267,7 +267,7 @@ struct min_raw_binary_key_f }; template -_CCCL_HOST_DEVICE void min_raw_binary_key(DecomposerT decomposer, T& aggregate) +_CCCL_HOST_DEVICE_API void min_raw_binary_key(DecomposerT decomposer, T& aggregate) { _CUB::detail::for_each_member(min_raw_binary_key_f{decomposer}, decomposer, aggregate); } @@ -278,7 +278,7 @@ struct max_raw_binary_key_f DecomposerT decomposer; template - _CCCL_HOST_DEVICE void operator()(T& field) + _CCCL_HOST_DEVICE_API void operator()(T& field) { using traits = traits_t; using bit_ordered_type = typename traits::bit_ordered_type; @@ -288,7 +288,7 @@ struct max_raw_binary_key_f }; template -_CCCL_HOST_DEVICE void max_raw_binary_key(DecomposerT decomposer, T& aggregate) +_CCCL_HOST_DEVICE_API void max_raw_binary_key(DecomposerT decomposer, T& aggregate) { _CUB::detail::for_each_member(max_raw_binary_key_f{decomposer}, decomposer, aggregate); } @@ -299,7 +299,7 @@ struct to_bit_ordered_f DecomposerT decomposer; template - _CCCL_HOST_DEVICE void operator()(T& field) + _CCCL_HOST_DEVICE_API void operator()(T& field) { using traits = traits_t; using bit_ordered_type = typename traits::bit_ordered_type; @@ -312,7 +312,7 @@ struct to_bit_ordered_f }; template -_CCCL_HOST_DEVICE void to_bit_ordered(DecomposerT decomposer, T& aggregate) +_CCCL_HOST_DEVICE_API void to_bit_ordered(DecomposerT decomposer, T& aggregate) { _CUB::detail::for_each_member(to_bit_ordered_f{decomposer}, decomposer, aggregate); } @@ -323,7 +323,7 @@ struct from_bit_ordered_f DecomposerT decomposer; template - _CCCL_HOST_DEVICE void operator()(T& field) + _CCCL_HOST_DEVICE_API void operator()(T& field) { using traits = traits_t; using bit_ordered_type = typename traits::bit_ordered_type; @@ -336,7 +336,7 @@ struct from_bit_ordered_f }; template -_CCCL_HOST_DEVICE void from_bit_ordered(DecomposerT decomposer, T& aggregate) +_CCCL_HOST_DEVICE_API void from_bit_ordered(DecomposerT decomposer, T& aggregate) { _CUB::detail::for_each_member(from_bit_ordered_f{decomposer}, decomposer, aggregate); } @@ -344,7 +344,7 @@ _CCCL_HOST_DEVICE void from_bit_ordered(DecomposerT decomposer, T& aggregate) struct inverse_f { template - _CCCL_HOST_DEVICE void operator()(T& field) + _CCCL_HOST_DEVICE_API void operator()(T& field) { using traits = traits_t; using bit_ordered_type = typename traits::bit_ordered_type; @@ -355,7 +355,7 @@ struct inverse_f }; template -_CCCL_HOST_DEVICE void inverse(DecomposerT decomposer, T& aggregate) +_CCCL_HOST_DEVICE_API void inverse(DecomposerT decomposer, T& aggregate) { _CUB::detail::for_each_member(inverse_f{}, decomposer, aggregate); } @@ -365,14 +365,14 @@ struct default_end_bit_f int& result; template - _CCCL_HOST_DEVICE void operator()(T& /* field */) + _CCCL_HOST_DEVICE_API void operator()(T& /* field */) { result += sizeof(T) * 8; } }; template -_CCCL_HOST_DEVICE int default_end_bit(DecomposerT decomposer, T& aggregate) +_CCCL_HOST_DEVICE_API int default_end_bit(DecomposerT decomposer, T& aggregate) { int result{}; _CUB::detail::for_each_member(default_end_bit_f{result}, decomposer, aggregate); @@ -387,7 +387,7 @@ struct digit_f ::cuda::std::uint32_t& num_bits; template - _CCCL_HOST_DEVICE void operator()(T& src) + _CCCL_HOST_DEVICE_API void operator()(T& src) { constexpr ::cuda::std::uint32_t src_size = sizeof(T) * 8; @@ -419,7 +419,7 @@ struct digit_f }; template -_CCCL_HOST_DEVICE void +_CCCL_HOST_DEVICE_API void digit(DecomposerT decomposer, ::cuda::std::uint32_t& dst, T& src, @@ -437,7 +437,7 @@ struct custom_digit_extractor_t ::cuda::std::uint32_t bit_start; ::cuda::std::uint32_t num_bits; - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE custom_digit_extractor_t(DecomposerT decomposer, ::cuda::std::uint32_t bit_start, ::cuda::std::uint32_t num_bits) : decomposer(decomposer) , bit_start(bit_start) @@ -445,7 +445,7 @@ struct custom_digit_extractor_t {} template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE ::cuda::std::uint32_t Digit(T& key) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE ::cuda::std::uint32_t Digit(T& key) const { ::cuda::std::uint32_t result{}; ::cuda::std::uint32_t dst_bit_start{}; @@ -459,14 +459,14 @@ struct custom_digit_extractor_t struct custom_bit_conversion_policy_t { template - static _CCCL_HOST_DEVICE T to_bit_ordered(DecomposerT decomposer, T val) + _CCCL_HOST_DEVICE_API static T to_bit_ordered(DecomposerT decomposer, T val) { _CUB::detail::radix::to_bit_ordered(decomposer, val); return val; } template - static _CCCL_HOST_DEVICE T from_bit_ordered(DecomposerT decomposer, T val) + _CCCL_HOST_DEVICE_API static T from_bit_ordered(DecomposerT decomposer, T val) { _CUB::detail::radix::from_bit_ordered(decomposer, val); return val; @@ -476,7 +476,7 @@ struct custom_bit_conversion_policy_t struct custom_bit_inversion_policy_t { template - static _CCCL_HOST_DEVICE T inverse(DecomposerT decomposer, T val) + _CCCL_HOST_DEVICE_API static T inverse(DecomposerT decomposer, T val) { _CUB::detail::radix::inverse(decomposer, val); return val; @@ -494,7 +494,7 @@ struct traits_t using digit_extractor_t = custom_digit_extractor_t; template - static _CCCL_HOST_DEVICE bit_ordered_type min_raw_binary_key(DecomposerT decomposer) + _CCCL_HOST_DEVICE_API static bit_ordered_type min_raw_binary_key(DecomposerT decomposer) { T val{}; _CUB::detail::radix::min_raw_binary_key(decomposer, val); @@ -502,7 +502,7 @@ struct traits_t } template - static _CCCL_HOST_DEVICE bit_ordered_type max_raw_binary_key(DecomposerT decomposer) + _CCCL_HOST_DEVICE_API static bit_ordered_type max_raw_binary_key(DecomposerT decomposer) { T val{}; _CUB::detail::radix::max_raw_binary_key(decomposer, val); @@ -510,14 +510,14 @@ struct traits_t } template - static _CCCL_HOST_DEVICE int default_end_bit(DecomposerT decomposer) + _CCCL_HOST_DEVICE_API static int default_end_bit(DecomposerT decomposer) { T aggregate{}; return _CUB::detail::radix::default_end_bit(decomposer, aggregate); } template - static _CCCL_HOST_DEVICE digit_extractor_t + _CCCL_HOST_DEVICE_API static digit_extractor_t digit_extractor(int begin_bit, int num_bits, DecomposerT decomposer) { return custom_digit_extractor_t(decomposer, begin_bit, num_bits); @@ -539,7 +539,7 @@ private: public: template - static _CCCL_HOST_DEVICE _CCCL_FORCEINLINE // + _CCCL_HOST_DEVICE_API static _CCCL_FORCEINLINE // bit_ordered_type In(bit_ordered_type key, DecomposerT decomposer = {}) { key = bit_ordered_conversion_policy::to_bit_ordered(decomposer, key); @@ -551,7 +551,7 @@ public: } template - static _CCCL_HOST_DEVICE _CCCL_FORCEINLINE // + _CCCL_HOST_DEVICE_API static _CCCL_FORCEINLINE // bit_ordered_type Out(bit_ordered_type key, DecomposerT decomposer = {}) { if constexpr (IsDescending) @@ -563,7 +563,7 @@ public: } template - static _CCCL_HOST_DEVICE _CCCL_FORCEINLINE // + _CCCL_HOST_DEVICE_API static _CCCL_FORCEINLINE // bit_ordered_type DefaultKey(DecomposerT decomposer = {}) { return IsDescending ? traits::min_raw_binary_key(decomposer) : traits::max_raw_binary_key(decomposer); diff --git a/cub/cub/block/specializations/block_histogram_atomic.cuh b/cub/cub/block/specializations/block_histogram_atomic.cuh index 2fa71dd02141..92c854b33b08 100644 --- a/cub/cub/block/specializations/block_histogram_atomic.cuh +++ b/cub/cub/block/specializations/block_histogram_atomic.cuh @@ -42,7 +42,7 @@ struct BlockHistogramAtomic {}; /// Constructor - _CCCL_DEVICE _CCCL_FORCEINLINE BlockHistogramAtomic(TempStorage& temp_storage) {} + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockHistogramAtomic(TempStorage& temp_storage) {} /** * @brief Composite data onto an existing histogram @@ -54,7 +54,7 @@ struct BlockHistogramAtomic * Reference to shared/device-accessible memory histogram */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void Composite(const T (&items)[ItemsPerThread], CounterT histogram[Bins]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Composite(const T (&items)[ItemsPerThread], CounterT histogram[Bins]) { // Use unsigned, unsigned long long for all integer counters constexpr bool is_integer_counter = diff --git a/cub/cub/block/specializations/block_histogram_sort.cuh b/cub/cub/block/specializations/block_histogram_sort.cuh index 22510b3d028b..dc83dc847c11 100644 --- a/cub/cub/block/specializations/block_histogram_sort.cuh +++ b/cub/cub/block/specializations/block_histogram_sort.cuh @@ -99,7 +99,7 @@ struct BlockHistogramSort unsigned int linear_tid; /// Constructor - _CCCL_DEVICE _CCCL_FORCEINLINE BlockHistogramSort(TempStorage& temp_storage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockHistogramSort(TempStorage& temp_storage) : temp_storage(temp_storage.Alias()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} @@ -111,12 +111,12 @@ struct BlockHistogramSort _TempStorage& temp_storage; // Constructor - _CCCL_DEVICE _CCCL_FORCEINLINE DiscontinuityOp(_TempStorage& temp_storage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE DiscontinuityOp(_TempStorage& temp_storage) : temp_storage(temp_storage) {} // Discontinuity predicate - _CCCL_DEVICE _CCCL_FORCEINLINE bool operator()(const T& a, const T& b, int b_index) + _CCCL_DEVICE_API _CCCL_FORCEINLINE bool operator()(const T& a, const T& b, int b_index) { if (a != b) { @@ -141,7 +141,7 @@ struct BlockHistogramSort * Reference to shared/device-accessible memory histogram */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void Composite(T (&items)[ItemsPerThread], CounterT histogram[Bins]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Composite(T (&items)[ItemsPerThread], CounterT histogram[Bins]) { static constexpr int TILE_SIZE = BLOCK_THREADS * ItemsPerThread; diff --git a/cub/cub/block/specializations/block_reduce_raking.cuh b/cub/cub/block/specializations/block_reduce_raking.cuh index 25a817f8f3b6..56ac67356be9 100644 --- a/cub/cub/block/specializations/block_reduce_raking.cuh +++ b/cub/cub/block/specializations/block_reduce_raking.cuh @@ -105,7 +105,7 @@ struct BlockReduceRaking unsigned int linear_tid; /// Constructor - _CCCL_DEVICE _CCCL_FORCEINLINE BlockReduceRaking(TempStorage& temp_storage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockReduceRaking(TempStorage& temp_storage) : temp_storage(temp_storage.Alias()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} @@ -121,7 +121,7 @@ struct BlockReduceRaking * Number of valid elements (may be less than BLOCK_THREADS) */ template - _CCCL_DEVICE _CCCL_FORCEINLINE T RakingReduction( + _CCCL_DEVICE_API _CCCL_FORCEINLINE T RakingReduction( ReductionOp reduction_op, T* raking_segment, T partial, int num_valid, constant_t /*iteration*/) { // Update partial if addend is in range @@ -144,7 +144,7 @@ struct BlockReduceRaking * Number of valid elements (may be less than BLOCK_THREADS) */ template - _CCCL_DEVICE _CCCL_FORCEINLINE T RakingReduction( + _CCCL_DEVICE_API _CCCL_FORCEINLINE T RakingReduction( ReductionOp /*reduction_op*/, T* /*raking_segment*/, T partial, @@ -169,7 +169,7 @@ struct BlockReduceRaking * Binary reduction operator */ template - _CCCL_DEVICE _CCCL_FORCEINLINE T Reduce(T partial, int num_valid, ReductionOp reduction_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE T Reduce(T partial, int num_valid, ReductionOp reduction_op) { if (WARP_SYNCHRONOUS) { @@ -220,7 +220,7 @@ struct BlockReduceRaking * Number of valid elements (may be less than BLOCK_THREADS) */ template - _CCCL_DEVICE _CCCL_FORCEINLINE T Sum(T partial, int num_valid) + _CCCL_DEVICE_API _CCCL_FORCEINLINE T Sum(T partial, int num_valid) { const ::cuda::std::plus<> reduction_op; diff --git a/cub/cub/block/specializations/block_reduce_raking_commutative_only.cuh b/cub/cub/block/specializations/block_reduce_raking_commutative_only.cuh index f52e77108657..6c25a2ad6c2d 100644 --- a/cub/cub/block/specializations/block_reduce_raking_commutative_only.cuh +++ b/cub/cub/block/specializations/block_reduce_raking_commutative_only.cuh @@ -107,7 +107,7 @@ struct BlockReduceRakingCommutativeOnly unsigned int linear_tid; /// Constructor - _CCCL_DEVICE _CCCL_FORCEINLINE BlockReduceRakingCommutativeOnly(TempStorage& temp_storage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockReduceRakingCommutativeOnly(TempStorage& temp_storage) : temp_storage(temp_storage.Alias()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} @@ -124,7 +124,7 @@ struct BlockReduceRakingCommutativeOnly * Number of valid elements (may be less than BLOCK_THREADS) */ template - _CCCL_DEVICE _CCCL_FORCEINLINE T Sum(T partial, int num_valid) + _CCCL_DEVICE_API _CCCL_FORCEINLINE T Sum(T partial, int num_valid) { if (USE_FALLBACK || !FullTile) { @@ -168,7 +168,7 @@ struct BlockReduceRakingCommutativeOnly * Binary reduction operator */ template - _CCCL_DEVICE _CCCL_FORCEINLINE T Reduce(T partial, int num_valid, ReductionOp reduction_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE T Reduce(T partial, int num_valid, ReductionOp reduction_op) { if (USE_FALLBACK || !FullTile) { diff --git a/cub/cub/block/specializations/block_reduce_warp_reductions.cuh b/cub/cub/block/specializations/block_reduce_warp_reductions.cuh index d580ed6480f0..8ac161303cd9 100644 --- a/cub/cub/block/specializations/block_reduce_warp_reductions.cuh +++ b/cub/cub/block/specializations/block_reduce_warp_reductions.cuh @@ -110,7 +110,7 @@ struct BlockReduceWarpReductions int warp_id; int lane_id; - _CCCL_DEVICE _CCCL_FORCEINLINE BlockReduceWarpReductions(TempStorage& temp_storage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockReduceWarpReductions(TempStorage& temp_storage) : temp_storage(temp_storage.Alias()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) , warp_id((warps == 1) ? 0 : linear_tid / warp_threads) @@ -130,7 +130,8 @@ struct BlockReduceWarpReductions //! @param[in] warp_aggregate //! **[**\ *lane*\ :sub:`0` **only]** Warp-wide aggregate reduction of input items template - _CCCL_DEVICE _CCCL_FORCEINLINE T ApplyWarpAggregatesNonDeterministic(ReductionOp /*reduction_op*/, T warp_aggregate) + _CCCL_DEVICE_API _CCCL_FORCEINLINE T + ApplyWarpAggregatesNonDeterministic(ReductionOp /*reduction_op*/, T warp_aggregate) { if (linear_tid == 0) { @@ -164,7 +165,7 @@ struct BlockReduceWarpReductions //! @tparam ReductionOp //! **[inferred]** Binary reduction operator type template - _CCCL_DEVICE _CCCL_FORCEINLINE T ApplyWarpAggregates(ReductionOp reduction_op, T warp_aggregate, int num_valid) + _CCCL_DEVICE_API _CCCL_FORCEINLINE T ApplyWarpAggregates(ReductionOp reduction_op, T warp_aggregate, int num_valid) { // Share lane aggregates if (lane_id == 0) @@ -234,7 +235,7 @@ struct BlockReduceWarpReductions //! @param[in] num_valid //! Number of valid elements (may be less than threads_per_block) template - _CCCL_DEVICE _CCCL_FORCEINLINE T Sum(T input, int num_valid) + _CCCL_DEVICE_API _CCCL_FORCEINLINE T Sum(T input, int num_valid) { return Reduce(input, num_valid, ::cuda::std::plus<>{}); } @@ -260,7 +261,7 @@ struct BlockReduceWarpReductions //! @param[in] reduction_op //! Binary reduction operator template - _CCCL_DEVICE _CCCL_FORCEINLINE T Reduce(T input, int num_valid, ReductionOp reduction_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE T Reduce(T input, int num_valid, ReductionOp reduction_op) { const int warp_offset = warp_id * logical_warp_size; const int warp_num_valid = diff --git a/cub/cub/block/specializations/block_scan_raking.cuh b/cub/cub/block/specializations/block_scan_raking.cuh index 62ac33db38df..157501c6df34 100644 --- a/cub/cub/block/specializations/block_scan_raking.cuh +++ b/cub/cub/block/specializations/block_scan_raking.cuh @@ -119,7 +119,7 @@ struct BlockScanRaking * Prefix to seed reduction with */ template - _CCCL_DEVICE _CCCL_FORCEINLINE T + _CCCL_DEVICE_API _CCCL_FORCEINLINE T GuardedReduce(T* raking_ptr, ScanOp scan_op, T raking_partial, constant_t /*iteration*/) { if (BlockRakingLayout::UNGUARDED || (((linear_tid * SEGMENT_LENGTH) + ITERATION) < BLOCK_THREADS)) @@ -144,7 +144,7 @@ struct BlockScanRaking * Prefix to seed reduction with */ template - _CCCL_DEVICE _CCCL_FORCEINLINE T + _CCCL_DEVICE_API _CCCL_FORCEINLINE T GuardedReduce(T* /*raking_ptr*/, ScanOp /*scan_op*/, T raking_partial, constant_t /*iteration*/) { return raking_partial; @@ -160,7 +160,7 @@ struct BlockScanRaking * [in] Input array */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void CopySegment(T* out, T* in, constant_t /*iteration*/) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void CopySegment(T* out, T* in, constant_t /*iteration*/) { out[ITERATION] = in[ITERATION]; CopySegment(out, in, constant_v); @@ -175,11 +175,12 @@ struct BlockScanRaking * @param[in] in * Input array */ - _CCCL_DEVICE _CCCL_FORCEINLINE void CopySegment(T* /*out*/, T* /*in*/, constant_t /*iteration*/) {} + _CCCL_DEVICE_API _CCCL_FORCEINLINE void CopySegment(T* /*out*/, T* /*in*/, constant_t /*iteration*/) + {} /// Performs upsweep raking reduction, returning the aggregate template - _CCCL_DEVICE _CCCL_FORCEINLINE T Upsweep(ScanOp scan_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE T Upsweep(ScanOp scan_op) { T* smem_raking_ptr = BlockRakingLayout::RakingPtr(temp_storage.raking_grid, linear_tid); @@ -193,7 +194,7 @@ struct BlockScanRaking /// Performs exclusive downsweep raking scan template - _CCCL_DEVICE _CCCL_FORCEINLINE void ExclusiveDownsweep(ScanOp scan_op, T raking_partial, bool apply_prefix = true) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveDownsweep(ScanOp scan_op, T raking_partial, bool apply_prefix = true) { T* smem_raking_ptr = BlockRakingLayout::RakingPtr(temp_storage.raking_grid, linear_tid); @@ -211,7 +212,7 @@ struct BlockScanRaking /// Performs inclusive downsweep raking scan template - _CCCL_DEVICE _CCCL_FORCEINLINE void InclusiveDownsweep(ScanOp scan_op, T raking_partial, bool apply_prefix = true) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveDownsweep(ScanOp scan_op, T raking_partial, bool apply_prefix = true) { T* smem_raking_ptr = BlockRakingLayout::RakingPtr(temp_storage.raking_grid, linear_tid); @@ -232,7 +233,7 @@ struct BlockScanRaking //--------------------------------------------------------------------- /// Constructor - _CCCL_DEVICE _CCCL_FORCEINLINE BlockScanRaking(TempStorage& temp_storage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockScanRaking(TempStorage& temp_storage) : temp_storage(temp_storage.Alias()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) {} @@ -256,7 +257,7 @@ struct BlockScanRaking * Binary scan operator */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void ExclusiveScan(T input, T& exclusive_output, ScanOp scan_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveScan(T input, T& exclusive_output, ScanOp scan_op) { if (WARP_SYNCHRONOUS) { @@ -309,7 +310,7 @@ struct BlockScanRaking * Binary scan operator */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void ExclusiveScan(T input, T& output, const T& initial_value, ScanOp scan_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveScan(T input, T& output, const T& initial_value, ScanOp scan_op) { if (WARP_SYNCHRONOUS) { @@ -366,7 +367,7 @@ struct BlockScanRaking * Threadblock-wide aggregate reduction of input items */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void ExclusiveScan(T input, T& output, ScanOp scan_op, T& block_aggregate) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveScan(T input, T& output, ScanOp scan_op, T& block_aggregate) { if (WARP_SYNCHRONOUS) { @@ -433,7 +434,7 @@ struct BlockScanRaking * Threadblock-wide aggregate reduction of input items */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveScan(T input, T& output, const T& initial_value, ScanOp scan_op, T& block_aggregate) { if (WARP_SYNCHRONOUS) @@ -502,7 +503,7 @@ struct BlockScanRaking * block-wide prefix to be applied to all inputs. */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveScan(T input, T& output, ScanOp scan_op, BlockPrefixCallbackOp& block_prefix_callback_op) { if (WARP_SYNCHRONOUS) @@ -583,7 +584,7 @@ struct BlockScanRaking * Binary scan operator */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void InclusiveScan(T input, T& output, ScanOp scan_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScan(T input, T& output, ScanOp scan_op) { if (WARP_SYNCHRONOUS) { @@ -637,7 +638,7 @@ struct BlockScanRaking * Threadblock-wide aggregate reduction of input items */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void InclusiveScan(T input, T& output, ScanOp scan_op, T& block_aggregate) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScan(T input, T& output, ScanOp scan_op, T& block_aggregate) { if (WARP_SYNCHRONOUS) { @@ -705,7 +706,7 @@ struct BlockScanRaking * block-wide prefix to be applied to all inputs. */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScan(T input, T& output, ScanOp scan_op, BlockPrefixCallbackOp& block_prefix_callback_op) { if (WARP_SYNCHRONOUS) diff --git a/cub/cub/block/specializations/block_scan_warp_scans.cuh b/cub/cub/block/specializations/block_scan_warp_scans.cuh index 054896036c16..4c82990d9f64 100644 --- a/cub/cub/block/specializations/block_scan_warp_scans.cuh +++ b/cub/cub/block/specializations/block_scan_warp_scans.cuh @@ -92,7 +92,7 @@ struct BlockScanWarpScans { alignas(16) char storage[sizeof(_TempStorage)]; - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE _TempStorage& Alias() + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE _TempStorage& Alias() { return reinterpret_cast<_TempStorage&>(*this); } @@ -113,7 +113,7 @@ struct BlockScanWarpScans //--------------------------------------------------------------------- /// Constructor - _CCCL_DEVICE _CCCL_FORCEINLINE BlockScanWarpScans(TempStorage& temp_storage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE BlockScanWarpScans(TempStorage& temp_storage) : temp_storage(temp_storage.Alias()) , linear_tid(_CUB::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)) , warp_id((WARPS == 1) ? 0 : linear_tid / WARP_THREADS) @@ -135,7 +135,7 @@ struct BlockScanWarpScans * Threadblock-wide aggregate reduction of input items */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ApplyWarpAggregates(T& warp_prefix, ScanOp scan_op, T& block_aggregate, constant_t /*addend_warp*/) { if (warp_id == WARP) @@ -160,7 +160,7 @@ struct BlockScanWarpScans * Threadblock-wide aggregate reduction of input items */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ApplyWarpAggregates(T& /*warp_prefix*/, ScanOp /*scan_op*/, T& /*block_aggregate*/, constant_t /*addend_warp*/) {} @@ -179,7 +179,7 @@ struct BlockScanWarpScans * Threadblock-wide aggregate reduction of input items */ template - _CCCL_DEVICE _CCCL_FORCEINLINE T ComputeWarpPrefix(ScanOp scan_op, T warp_aggregate, T& block_aggregate) + _CCCL_DEVICE_API _CCCL_FORCEINLINE T ComputeWarpPrefix(ScanOp scan_op, T warp_aggregate, T& block_aggregate) { // Last lane in each warp shares its warp-aggregate if (lane_id == WARP_THREADS - 1) @@ -229,7 +229,7 @@ struct BlockScanWarpScans * Initial value to seed the exclusive scan */ template - _CCCL_DEVICE _CCCL_FORCEINLINE T + _CCCL_DEVICE_API _CCCL_FORCEINLINE T ComputeWarpPrefix(ScanOp scan_op, T warp_aggregate, T& block_aggregate, const T& initial_value) { T warp_prefix = ComputeWarpPrefix(scan_op, warp_aggregate, block_aggregate); @@ -263,7 +263,7 @@ struct BlockScanWarpScans * Binary scan operator */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void ExclusiveScan(T input, T& exclusive_output, ScanOp scan_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveScan(T input, T& exclusive_output, ScanOp scan_op) { // Compute block-wide exclusive scan. The exclusive output from tid0 is invalid. T block_aggregate; @@ -287,7 +287,8 @@ struct BlockScanWarpScans * Binary scan operator */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void ExclusiveScan(T input, T& exclusive_output, const T& initial_value, ScanOp scan_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void + ExclusiveScan(T input, T& exclusive_output, const T& initial_value, ScanOp scan_op) { // ExclusiveScan fills the aggregate before it is read. // NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init) @@ -314,7 +315,7 @@ struct BlockScanWarpScans * Threadblock-wide aggregate reduction of input items */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void ExclusiveScan(T input, T& exclusive_output, ScanOp scan_op, T& block_aggregate) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveScan(T input, T& exclusive_output, ScanOp scan_op, T& block_aggregate) { // Compute warp scan in each warp. The exclusive output from each lane0 is invalid. // WarpScan fills the output before it is read. @@ -357,7 +358,7 @@ struct BlockScanWarpScans * Threadblock-wide aggregate reduction of input items */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveScan(T input, T& exclusive_output, const T& initial_value, ScanOp scan_op, T& block_aggregate) { // Compute warp scan in each warp. The exclusive output from each lane0 is invalid. @@ -399,7 +400,7 @@ struct BlockScanWarpScans * block-wide prefix to be applied to all inputs. */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveScan(T input, T& exclusive_output, ScanOp scan_op, BlockPrefixCallbackOp& block_prefix_callback_op) { // Compute block-wide exclusive scan. The exclusive output from tid0 is invalid. @@ -449,7 +450,7 @@ struct BlockScanWarpScans * Binary scan operator */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void InclusiveScan(T input, T& inclusive_output, ScanOp scan_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScan(T input, T& inclusive_output, ScanOp scan_op) { T block_aggregate; InclusiveScan(input, inclusive_output, scan_op, block_aggregate); @@ -473,7 +474,7 @@ struct BlockScanWarpScans * Threadblock-wide aggregate reduction of input items */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void InclusiveScan(T input, T& inclusive_output, ScanOp scan_op, T& block_aggregate) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScan(T input, T& inclusive_output, ScanOp scan_op, T& block_aggregate) { WarpScanT(temp_storage.warp_scan[warp_id]).InclusiveScan(input, inclusive_output, scan_op); @@ -509,7 +510,7 @@ struct BlockScanWarpScans * block-wide prefix to be applied to all inputs. */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScan(T input, T& exclusive_output, ScanOp scan_op, BlockPrefixCallbackOp& block_prefix_callback_op) { // InclusiveScan fills the aggregate before it is read. diff --git a/cub/cub/detail/array_utils.cuh b/cub/cub/detail/array_utils.cuh index f9a136c0bcb0..beddad2d9be9 100644 --- a/cub/cub/detail/array_utils.cuh +++ b/cub/cub/detail/array_utils.cuh @@ -33,7 +33,7 @@ namespace detail **********************************************************************************************************************/ template -[[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE ::cuda::std::array> +[[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE ::cuda::std::array> to_array_impl(const Input& input, ::cuda::std::index_sequence) { using ArrayType = ::cuda::std::array>; @@ -41,7 +41,7 @@ to_array_impl(const Input& input, ::cuda::std::index_sequence) } template -[[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE ::cuda::std::array> +[[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE ::cuda::std::array> to_array(const Input& input) { using InputType = ::cuda::std::iter_value_t; diff --git a/cub/cub/detail/binary_search_helpers.cuh b/cub/cub/detail/binary_search_helpers.cuh index 10d8c68f0c37..135f23993450 100644 --- a/cub/cub/detail/binary_search_helpers.cuh +++ b/cub/cub/detail/binary_search_helpers.cuh @@ -34,7 +34,7 @@ struct comp_wrapper_t CompareOpT op; template - _CCCL_DEVICE _CCCL_FORCEINLINE void operator()(::cuda::std::tuple args) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE void operator()(::cuda::std::tuple args) const { using DifferenceT = ::cuda::std::iter_difference_t; const auto last = first + static_cast(num_items); @@ -44,7 +44,7 @@ struct comp_wrapper_t }; template -_CCCL_HOST_DEVICE auto make_comp_wrapper(RangeIteratorT first, RangeNumItemsT num_items, CompareOpT comp) +_CCCL_HOST_DEVICE_API auto make_comp_wrapper(RangeIteratorT first, RangeNumItemsT num_items, CompareOpT comp) { return comp_wrapper_t{first, num_items, comp}; } @@ -52,7 +52,7 @@ _CCCL_HOST_DEVICE auto make_comp_wrapper(RangeIteratorT first, RangeNumItemsT nu struct lower_bound { template - _CCCL_DEVICE _CCCL_FORCEINLINE static DifferenceT + _CCCL_DEVICE_API _CCCL_FORCEINLINE static DifferenceT Linear(RangeIteratorT first, DifferenceT num_items, const T& value, CompareOpT comp) { DifferenceT retval = 0; @@ -65,7 +65,7 @@ struct lower_bound } template - _CCCL_DEVICE _CCCL_FORCEINLINE static ::cuda::std::ptrdiff_t + _CCCL_DEVICE_API _CCCL_FORCEINLINE static ::cuda::std::ptrdiff_t Invoke(RangeIteratorT first, RangeIteratorT last, const T& value, CompareOpT comp) { return ::cuda::std::lower_bound(first, last, value, comp) - first; @@ -75,7 +75,7 @@ struct lower_bound struct upper_bound { template - _CCCL_DEVICE _CCCL_FORCEINLINE static DifferenceT + _CCCL_DEVICE_API _CCCL_FORCEINLINE static DifferenceT Linear(RangeIteratorT first, DifferenceT num_items, const T& value, CompareOpT comp) { DifferenceT retval = 0; @@ -88,7 +88,7 @@ struct upper_bound } template - _CCCL_DEVICE _CCCL_FORCEINLINE static ::cuda::std::ptrdiff_t + _CCCL_DEVICE_API _CCCL_FORCEINLINE static ::cuda::std::ptrdiff_t Invoke(RangeIteratorT first, RangeIteratorT last, const T& value, CompareOpT comp) { return ::cuda::std::upper_bound(first, last, value, comp) - first; @@ -103,7 +103,7 @@ struct binary_search_transform_op_t CompareOpT op; template - _CCCL_DEVICE _CCCL_FORCEINLINE ::cuda::std::ptrdiff_t operator()(const Value& value) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE ::cuda::std::ptrdiff_t operator()(const Value& value) const { using DifferenceT = ::cuda::std::iter_difference_t; const auto count = static_cast(num_items); @@ -118,7 +118,8 @@ struct binary_search_transform_op_t }; template -_CCCL_HOST_DEVICE auto make_binary_search_transform_op(RangeIteratorT first, RangeNumItemsT num_items, CompareOpT comp) +_CCCL_HOST_DEVICE_API auto +make_binary_search_transform_op(RangeIteratorT first, RangeNumItemsT num_items, CompareOpT comp) { return binary_search_transform_op_t{first, num_items, comp}; } diff --git a/cub/cub/detail/choose_offset.cuh b/cub/cub/detail/choose_offset.cuh index 4e945bb2b1b4..8fb0316d6532 100644 --- a/cub/cub/detail/choose_offset.cuh +++ b/cub/cub/detail/choose_offset.cuh @@ -119,7 +119,7 @@ struct choose_signed_offset * Checks if the given num_items can be covered by the selected offset type. If not, returns cudaErrorInvalidValue, * otherwise returns cudaSuccess. */ - static _CCCL_HOST_DEVICE _CCCL_FORCEINLINE cudaError_t is_exceeding_offset_type(NumItemsT num_items) + _CCCL_HOST_DEVICE_API static _CCCL_FORCEINLINE cudaError_t is_exceeding_offset_type(NumItemsT num_items) { _CCCL_DIAG_PUSH _CCCL_DIAG_SUPPRESS_MSVC(4127) /* conditional expression is constant */ diff --git a/cub/cub/detail/cub_vector.cuh b/cub/cub/detail/cub_vector.cuh index 94b46556c7d4..714434a80d25 100644 --- a/cub/cub/detail/cub_vector.cuh +++ b/cub/cub/detail/cub_vector.cuh @@ -91,94 +91,94 @@ struct CubVector /** * Macro for expanding partially-specialized built-in vector types */ -# define CUB_DEFINE_VECTOR_TYPE(base_type, short_type) \ - \ - template <> \ - struct CubVector : short_type##1 \ - { \ - using BaseType = base_type; \ - using Type = short_type##1; \ - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE CubVector operator+(const CubVector& other) const \ - { \ - CubVector retval; \ - retval.x = x + other.x; \ - return retval; \ - } \ - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE CubVector operator-(const CubVector& other) const \ - { \ - CubVector retval; \ - retval.x = x - other.x; \ - return retval; \ - } \ - }; \ - \ - template <> \ - struct CubVector : short_type##2 \ - { \ - using BaseType = base_type; \ - using Type = short_type##2; \ - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE CubVector operator+(const CubVector& other) const \ - { \ - CubVector retval; \ - retval.x = x + other.x; \ - retval.y = y + other.y; \ - return retval; \ - } \ - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE CubVector operator-(const CubVector& other) const \ - { \ - CubVector retval; \ - retval.x = x - other.x; \ - retval.y = y - other.y; \ - return retval; \ - } \ - }; \ - \ - template <> \ - struct CubVector : short_type##3 \ - { \ - using BaseType = base_type; \ - using Type = short_type##3; \ - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE CubVector operator+(const CubVector& other) const \ - { \ - CubVector retval; \ - retval.x = x + other.x; \ - retval.y = y + other.y; \ - retval.z = z + other.z; \ - return retval; \ - } \ - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE CubVector operator-(const CubVector& other) const \ - { \ - CubVector retval; \ - retval.x = x - other.x; \ - retval.y = y - other.y; \ - retval.z = z - other.z; \ - return retval; \ - } \ - }; \ - \ - template <> \ - struct CubVector : short_type##4 \ - { \ - using BaseType = base_type; \ - using Type = short_type##4; \ - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE CubVector operator+(const CubVector& other) const \ - { \ - CubVector retval; \ - retval.x = x + other.x; \ - retval.y = y + other.y; \ - retval.z = z + other.z; \ - retval.w = w + other.w; \ - return retval; \ - } \ - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE CubVector operator-(const CubVector& other) const \ - { \ - CubVector retval; \ - retval.x = x - other.x; \ - retval.y = y - other.y; \ - retval.z = z - other.z; \ - retval.w = w - other.w; \ - return retval; \ - } \ +# define CUB_DEFINE_VECTOR_TYPE(base_type, short_type) \ + \ + template <> \ + struct CubVector : short_type##1 \ + { \ + using BaseType = base_type; \ + using Type = short_type##1; \ + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE CubVector operator+(const CubVector& other) const \ + { \ + CubVector retval; \ + retval.x = x + other.x; \ + return retval; \ + } \ + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE CubVector operator-(const CubVector& other) const \ + { \ + CubVector retval; \ + retval.x = x - other.x; \ + return retval; \ + } \ + }; \ + \ + template <> \ + struct CubVector : short_type##2 \ + { \ + using BaseType = base_type; \ + using Type = short_type##2; \ + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE CubVector operator+(const CubVector& other) const \ + { \ + CubVector retval; \ + retval.x = x + other.x; \ + retval.y = y + other.y; \ + return retval; \ + } \ + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE CubVector operator-(const CubVector& other) const \ + { \ + CubVector retval; \ + retval.x = x - other.x; \ + retval.y = y - other.y; \ + return retval; \ + } \ + }; \ + \ + template <> \ + struct CubVector : short_type##3 \ + { \ + using BaseType = base_type; \ + using Type = short_type##3; \ + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE CubVector operator+(const CubVector& other) const \ + { \ + CubVector retval; \ + retval.x = x + other.x; \ + retval.y = y + other.y; \ + retval.z = z + other.z; \ + return retval; \ + } \ + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE CubVector operator-(const CubVector& other) const \ + { \ + CubVector retval; \ + retval.x = x - other.x; \ + retval.y = y - other.y; \ + retval.z = z - other.z; \ + return retval; \ + } \ + }; \ + \ + template <> \ + struct CubVector : short_type##4 \ + { \ + using BaseType = base_type; \ + using Type = short_type##4; \ + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE CubVector operator+(const CubVector& other) const \ + { \ + CubVector retval; \ + retval.x = x + other.x; \ + retval.y = y + other.y; \ + retval.z = z + other.z; \ + retval.w = w + other.w; \ + return retval; \ + } \ + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE CubVector operator-(const CubVector& other) const \ + { \ + CubVector retval; \ + retval.x = x - other.x; \ + retval.y = y - other.y; \ + retval.z = z - other.z; \ + retval.w = w - other.w; \ + return retval; \ + } \ }; // Expand CUDA vector types for built-in primitives diff --git a/cub/cub/detail/device_double_buffer.cuh b/cub/cub/detail/device_double_buffer.cuh index f2b1ccd38f37..03d8cf671e0c 100644 --- a/cub/cub/detail/device_double_buffer.cuh +++ b/cub/cub/detail/device_double_buffer.cuh @@ -65,24 +65,24 @@ public: * @param d_alternate * Alternate storage buffer of the same size as @p d_current */ - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE device_double_buffer(T* current, T* alternate) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE device_double_buffer(T* current, T* alternate) : m_current_buffer(current) , m_alternate_buffer(alternate) {} /// \brief Return pointer to the currently valid buffer - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE T* current() const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE T* current() const { return m_current_buffer; } /// \brief Return pointer to the currently invalid buffer - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE T* alternate() const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE T* alternate() const { return m_alternate_buffer; } - _CCCL_HOST_DEVICE void swap() + _CCCL_HOST_DEVICE_API void swap() { T* tmp = m_current_buffer; m_current_buffer = m_alternate_buffer; diff --git a/cub/cub/detail/double_buffer.cuh b/cub/cub/detail/double_buffer.cuh index dd51ad9ea4e7..575ddc9b9253 100644 --- a/cub/cub/detail/double_buffer.cuh +++ b/cub/cub/detail/double_buffer.cuh @@ -35,23 +35,23 @@ struct DoubleBuffer int selector = 0; /// \brief Constructor - DoubleBuffer() = default; + _CCCL_HIDE_FROM_ABI DoubleBuffer() = default; /// \brief Constructor - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE DoubleBuffer(T* d_current, ///< The currently valid buffer - T* d_alternate) ///< Alternate storage buffer of the same size as \p - ///< d_current + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE DoubleBuffer(T* d_current, ///< The currently valid buffer + T* d_alternate) ///< Alternate storage buffer of the same size as + ///< \p d_current : d_buffers{d_current, d_alternate} {} /// \brief Return pointer to the currently valid buffer - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE T* Current() + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE T* Current() { return d_buffers[selector]; } /// \brief Return pointer to the currently invalid buffer - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE T* Alternate() + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE T* Alternate() { return d_buffers[selector ^ 1]; } diff --git a/cub/cub/detail/future_value.cuh b/cub/cub/detail/future_value.cuh index 5a8d8f85bec9..7ea51be74fbb 100644 --- a/cub/cub/detail/future_value.cuh +++ b/cub/cub/detail/future_value.cuh @@ -49,11 +49,11 @@ struct FutureValue using value_type = T; using iterator_type = IterT; - explicit _CCCL_HOST_DEVICE _CCCL_FORCEINLINE FutureValue(IterT iter) + explicit _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE FutureValue(IterT iter) : m_iter(iter) {} - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE operator T() const noexcept + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE operator T() const noexcept { return *m_iter; } diff --git a/cub/cub/detail/input_value.cuh b/cub/cub/detail/input_value.cuh index 2c894660b44e..85e11ef1fe04 100644 --- a/cub/cub/detail/input_value.cuh +++ b/cub/cub/detail/input_value.cuh @@ -29,7 +29,7 @@ struct InputValue { using value_type = T; using iterator_type = IterT; - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE operator T() const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE operator T() const { if (m_is_future) { @@ -37,15 +37,15 @@ struct InputValue } return m_immediate_value; } - explicit _CCCL_HOST_DEVICE _CCCL_FORCEINLINE InputValue(T immediate_value) + explicit _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE InputValue(T immediate_value) : m_is_future(false) , m_immediate_value(immediate_value) {} - explicit _CCCL_HOST_DEVICE _CCCL_FORCEINLINE InputValue(FutureValue future_value) + explicit _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE InputValue(FutureValue future_value) : m_is_future(true) , m_future_value(future_value) {} - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE InputValue(const InputValue& other) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE InputValue(const InputValue& other) : m_is_future(other.m_is_future) { if (m_is_future) diff --git a/cub/cub/detail/key_value_pair.cuh b/cub/cub/detail/key_value_pair.cuh index 136462c8a7ac..31d35b88acb3 100644 --- a/cub/cub/detail/key_value_pair.cuh +++ b/cub/cub/detail/key_value_pair.cuh @@ -31,28 +31,28 @@ struct KeyValuePair Value value; ///< Item value /// Constructor - _CCCL_FORCEINLINE KeyValuePair() = default; + _CCCL_HIDE_FROM_ABI KeyValuePair() = default; /// Constructor - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE KeyValuePair(Key const& key, Value const& value) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE KeyValuePair(Key const& key, Value const& value) : key(key) , value(value) {} /// Equality operator - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE bool operator==(const KeyValuePair& b) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE bool operator==(const KeyValuePair& b) const { return (value == b.value) && (key == b.key); } /// Inequality operator - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE bool operator!=(const KeyValuePair& b) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE bool operator!=(const KeyValuePair& b) const { return (value != b.value) || (key != b.key); } # if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const KeyValuePair& pair) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const KeyValuePair& pair) { return os << '(' << pair.key << ',' << pair.value << ')'; } diff --git a/cub/cub/detail/null_type.cuh b/cub/cub/detail/null_type.cuh index b0e91184c1d7..297da25c4cfd 100644 --- a/cub/cub/detail/null_type.cuh +++ b/cub/cub/detail/null_type.cuh @@ -23,24 +23,24 @@ struct NullType { using value_type = NullType; - NullType() = default; + _CCCL_HIDE_FROM_ABI NullType() = default; template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE explicit NullType(const T&) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE explicit NullType(const T&) {} template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE NullType& operator=(const T&) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE NullType& operator=(const T&) { return *this; } - friend _CCCL_HOST_DEVICE _CCCL_FORCEINLINE bool operator==(const NullType&, const NullType&) + _CCCL_HOST_DEVICE_API friend _CCCL_FORCEINLINE bool operator==(const NullType&, const NullType&) { return true; } - friend _CCCL_HOST_DEVICE _CCCL_FORCEINLINE bool operator!=(const NullType&, const NullType&) + _CCCL_HOST_DEVICE_API friend _CCCL_FORCEINLINE bool operator!=(const NullType&, const NullType&) { return false; } diff --git a/cub/cub/detail/numeric_traits.cuh b/cub/cub/detail/numeric_traits.cuh index 917f2fc21d8f..e23a9bf7b226 100644 --- a/cub/cub/detail/numeric_traits.cuh +++ b/cub/cub/detail/numeric_traits.cuh @@ -67,18 +67,19 @@ struct BaseTraits static constexpr UnsignedBits LOWEST_KEY = UnsignedBits(0); static constexpr UnsignedBits MAX_KEY = UnsignedBits(-1); - static _CCCL_HOST_DEVICE _CCCL_FORCEINLINE UnsignedBits TwiddleIn(UnsignedBits key) + _CCCL_HOST_DEVICE_API static _CCCL_FORCEINLINE UnsignedBits TwiddleIn(UnsignedBits key) { return key; } - static _CCCL_HOST_DEVICE _CCCL_FORCEINLINE UnsignedBits TwiddleOut(UnsignedBits key) + _CCCL_HOST_DEVICE_API static _CCCL_FORCEINLINE UnsignedBits TwiddleOut(UnsignedBits key) { return key; } //! deprecated [Since 3.0] - CCCL_DEPRECATED_BECAUSE("Use cuda::std::numeric_limits::max()") static _CCCL_HOST_DEVICE _CCCL_FORCEINLINE T Max() + CCCL_DEPRECATED_BECAUSE("Use cuda::std::numeric_limits::max()") _CCCL_HOST_DEVICE_API static _CCCL_FORCEINLINE T + Max() { UnsignedBits retval_bits = MAX_KEY; T retval; @@ -87,7 +88,7 @@ struct BaseTraits } //! deprecated [Since 3.0] - CCCL_DEPRECATED_BECAUSE("Use cuda::std::numeric_limits::lowest()") static _CCCL_HOST_DEVICE _CCCL_FORCEINLINE T + CCCL_DEPRECATED_BECAUSE("Use cuda::std::numeric_limits::lowest()") _CCCL_HOST_DEVICE_API static _CCCL_FORCEINLINE T Lowest() { UnsignedBits retval_bits = LOWEST_KEY; @@ -116,25 +117,26 @@ struct BaseTraits static constexpr UnsignedBits LOWEST_KEY = HIGH_BIT; static constexpr UnsignedBits MAX_KEY = UnsignedBits(-1) ^ HIGH_BIT; - static _CCCL_HOST_DEVICE _CCCL_FORCEINLINE UnsignedBits TwiddleIn(UnsignedBits key) + _CCCL_HOST_DEVICE_API static _CCCL_FORCEINLINE UnsignedBits TwiddleIn(UnsignedBits key) { return key ^ HIGH_BIT; }; - static _CCCL_HOST_DEVICE _CCCL_FORCEINLINE UnsignedBits TwiddleOut(UnsignedBits key) + _CCCL_HOST_DEVICE_API static _CCCL_FORCEINLINE UnsignedBits TwiddleOut(UnsignedBits key) { return key ^ HIGH_BIT; }; //! deprecated [Since 3.0] - CCCL_DEPRECATED_BECAUSE("Use cuda::std::numeric_limits::max()") static _CCCL_HOST_DEVICE _CCCL_FORCEINLINE T Max() + CCCL_DEPRECATED_BECAUSE("Use cuda::std::numeric_limits::max()") _CCCL_HOST_DEVICE_API static _CCCL_FORCEINLINE T + Max() { UnsignedBits retval = MAX_KEY; return reinterpret_cast(retval); } //! deprecated [Since 3.0] - CCCL_DEPRECATED_BECAUSE("Use cuda::std::numeric_limits::lowest()") static _CCCL_HOST_DEVICE _CCCL_FORCEINLINE T + CCCL_DEPRECATED_BECAUSE("Use cuda::std::numeric_limits::lowest()") _CCCL_HOST_DEVICE_API static _CCCL_FORCEINLINE T Lowest() { UnsignedBits retval = LOWEST_KEY; @@ -168,26 +170,27 @@ struct BaseTraits static constexpr UnsignedBits LOWEST_KEY = UnsignedBits(-1); static constexpr UnsignedBits MAX_KEY = UnsignedBits(-1) ^ HIGH_BIT; - static _CCCL_HOST_DEVICE _CCCL_FORCEINLINE UnsignedBits TwiddleIn(UnsignedBits key) + _CCCL_HOST_DEVICE_API static _CCCL_FORCEINLINE UnsignedBits TwiddleIn(UnsignedBits key) { const UnsignedBits mask = (key & HIGH_BIT) ? UnsignedBits(-1) : HIGH_BIT; return key ^ mask; }; - static _CCCL_HOST_DEVICE _CCCL_FORCEINLINE UnsignedBits TwiddleOut(UnsignedBits key) + _CCCL_HOST_DEVICE_API static _CCCL_FORCEINLINE UnsignedBits TwiddleOut(UnsignedBits key) { const UnsignedBits mask = (key & HIGH_BIT) ? HIGH_BIT : UnsignedBits(-1); return key ^ mask; }; //! deprecated [Since 3.0] - CCCL_DEPRECATED_BECAUSE("Use cuda::std::numeric_limits::max()") static _CCCL_HOST_DEVICE _CCCL_FORCEINLINE T Max() + CCCL_DEPRECATED_BECAUSE("Use cuda::std::numeric_limits::max()") _CCCL_HOST_DEVICE_API static _CCCL_FORCEINLINE T + Max() { return ::cuda::std::numeric_limits::max(); } //! deprecated [Since 3.0] - CCCL_DEPRECATED_BECAUSE("Use cuda::std::numeric_limits::lowest()") static _CCCL_HOST_DEVICE _CCCL_FORCEINLINE T + CCCL_DEPRECATED_BECAUSE("Use cuda::std::numeric_limits::lowest()") _CCCL_HOST_DEVICE_API static _CCCL_FORCEINLINE T Lowest() { return ::cuda::std::numeric_limits::lowest(); @@ -240,24 +243,25 @@ struct NumericTraits<__uint128_t> static constexpr UnsignedBits LOWEST_KEY = UnsignedBits(0); static constexpr UnsignedBits MAX_KEY = UnsignedBits(-1); - static _CCCL_HOST_DEVICE _CCCL_FORCEINLINE UnsignedBits TwiddleIn(UnsignedBits key) + _CCCL_HOST_DEVICE_API static _CCCL_FORCEINLINE UnsignedBits TwiddleIn(UnsignedBits key) { return key; } - static _CCCL_HOST_DEVICE _CCCL_FORCEINLINE UnsignedBits TwiddleOut(UnsignedBits key) + _CCCL_HOST_DEVICE_API static _CCCL_FORCEINLINE UnsignedBits TwiddleOut(UnsignedBits key) { return key; } //! deprecated [Since 3.0] - CCCL_DEPRECATED_BECAUSE("Use cuda::std::numeric_limits::max()") static _CCCL_HOST_DEVICE _CCCL_FORCEINLINE T Max() + CCCL_DEPRECATED_BECAUSE("Use cuda::std::numeric_limits::max()") _CCCL_HOST_DEVICE_API static _CCCL_FORCEINLINE T + Max() { return MAX_KEY; } //! deprecated [Since 3.0] - CCCL_DEPRECATED_BECAUSE("Use cuda::std::numeric_limits::lowest()") static _CCCL_HOST_DEVICE _CCCL_FORCEINLINE T + CCCL_DEPRECATED_BECAUSE("Use cuda::std::numeric_limits::lowest()") _CCCL_HOST_DEVICE_API static _CCCL_FORCEINLINE T Lowest() { return LOWEST_KEY; @@ -279,25 +283,26 @@ struct NumericTraits<__int128_t> static constexpr UnsignedBits LOWEST_KEY = HIGH_BIT; static constexpr UnsignedBits MAX_KEY = UnsignedBits(-1) ^ HIGH_BIT; - static _CCCL_HOST_DEVICE _CCCL_FORCEINLINE UnsignedBits TwiddleIn(UnsignedBits key) + _CCCL_HOST_DEVICE_API static _CCCL_FORCEINLINE UnsignedBits TwiddleIn(UnsignedBits key) { return key ^ HIGH_BIT; }; - static _CCCL_HOST_DEVICE _CCCL_FORCEINLINE UnsignedBits TwiddleOut(UnsignedBits key) + _CCCL_HOST_DEVICE_API static _CCCL_FORCEINLINE UnsignedBits TwiddleOut(UnsignedBits key) { return key ^ HIGH_BIT; }; //! deprecated [Since 3.0] - CCCL_DEPRECATED_BECAUSE("Use cuda::std::numeric_limits::max()") static _CCCL_HOST_DEVICE _CCCL_FORCEINLINE T Max() + CCCL_DEPRECATED_BECAUSE("Use cuda::std::numeric_limits::max()") _CCCL_HOST_DEVICE_API static _CCCL_FORCEINLINE T + Max() { UnsignedBits retval = MAX_KEY; return reinterpret_cast(retval); } //! deprecated [Since 3.0] - CCCL_DEPRECATED_BECAUSE("Use cuda::std::numeric_limits::lowest()") static _CCCL_HOST_DEVICE _CCCL_FORCEINLINE T + CCCL_DEPRECATED_BECAUSE("Use cuda::std::numeric_limits::lowest()") _CCCL_HOST_DEVICE_API static _CCCL_FORCEINLINE T Lowest() { UnsignedBits retval = LOWEST_KEY; diff --git a/cub/cub/detail/prefetch.cuh b/cub/cub/detail/prefetch.cuh index b0fbee412eb9..62cfe795b045 100644 --- a/cub/cub/detail/prefetch.cuh +++ b/cub/cub/detail/prefetch.cuh @@ -52,7 +52,7 @@ enum class LoadPrefetch : int }; #if _CCCL_HOSTED() -inline ::std::ostream& operator<<(::std::ostream& os, LoadPrefetch prefetch) +_CCCL_HOST_API inline ::std::ostream& operator<<(::std::ostream& os, LoadPrefetch prefetch) { switch (prefetch) { diff --git a/cub/cub/detail/rfa.cuh b/cub/cub/detail/rfa.cuh index 3c1db86e527f..1ea6d278a8ce 100644 --- a/cub/cub/detail/rfa.cuh +++ b/cub/cub/detail/rfa.cuh @@ -36,7 +36,7 @@ inline constexpr int cub_rfa_max_jump = 5; static_assert(cub_rfa_max_jump <= 5, "cub_rfa_max_jump must be less than or equal to 5"); template -_CCCL_DEVICE FType* get_shared_bin_array() +_CCCL_DEVICE_API FType* get_shared_bin_array() { static __shared__ FType bin_computed_array[Len]; return bin_computed_array; @@ -68,7 +68,7 @@ public: // The maximum floating-point fold supported by the library static constexpr auto max_fold = max_index + 1; - _CCCL_DEVICE static ftype initialize_bin(int index) noexcept + _CCCL_DEVICE_API static ftype initialize_bin(int index) noexcept { if (index == 0) { @@ -101,34 +101,34 @@ private: static constexpr auto exp_bias = max_exp - 2; /// Return a binned floating-point bin - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE static ftype binned_bins(int index) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE static ftype binned_bins(int index) { const ftype* bins = _CUB::detail::rfa::get_shared_bin_array(); return bins[index]; } - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE static uint32_t& get_bit_representation(float& x) noexcept + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE static uint32_t& get_bit_representation(float& x) noexcept { return *reinterpret_cast(&x); } - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE static uint64_t& get_bit_representation(double& x) noexcept + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE static uint64_t& get_bit_representation(double& x) noexcept { return *reinterpret_cast(&x); } - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE static uint32_t get_bit_representation(const float& x) noexcept + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE static uint32_t get_bit_representation(const float& x) noexcept { return ::cuda::std::bit_cast(x); } - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE static uint64_t get_bit_representation(const double& x) noexcept + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE static uint64_t get_bit_representation(const double& x) noexcept { return ::cuda::std::bit_cast(x); } /// Return primary vector value const ref - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE const ftype& primary(int i) const noexcept + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE const ftype& primary(int i) const noexcept { if constexpr (Fold <= cub_rfa_max_jump) { @@ -175,7 +175,7 @@ private: } /// Return carry vector value const ref - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE const ftype& carry(int i) const noexcept + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE const ftype& carry(int i) const noexcept { if (Fold <= cub_rfa_max_jump) { @@ -222,20 +222,20 @@ private: } /// Return primary vector value ref - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE ftype& primary(int i) noexcept + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE ftype& primary(int i) noexcept { const auto& c = *this; return const_cast(c.primary(i)); } /// Return carry vector value ref - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE ftype& carry(int i) noexcept + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE ftype& carry(int i) noexcept { const auto& c = *this; return const_cast(c.carry(i)); } - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE static int exp_val(const ftype x) noexcept + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE static int exp_val(const ftype x) noexcept { const auto bits = get_bit_representation(x); return (bits >> (mant_dig - 1)) & ((2 * max_exp) - 1); @@ -245,7 +245,7 @@ private: /// The index of a non-binned type is the smallest index a binned type would /// need to have to sum it reproducibly. Higher indices correspond to smaller /// bins. - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE static int binned_dindex(const ftype x) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE static int binned_dindex(const ftype x) { int exp = exp_val(x); @@ -264,14 +264,14 @@ private: /// Get index of manually specified binned double precision /// The index of a binned type is the bin that it corresponds to. Higher /// indices correspond to smaller bins. - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE int binned_index() const + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE int binned_index() const { return ((max_exp + mant_dig - bin_width + 1 + exp_bias) - exp_val(primary(0))) / bin_width; } /// Check if index of manually specified binned floating-point is 0 /// A quick check to determine if the index is 0 - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE bool is_binned_index_zero() const + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE bool is_binned_index_zero() const { return exp_val(primary(0)) == max_exp + exp_bias; } @@ -280,7 +280,7 @@ private: //! //! This method updates the binned fp to an index suitable for adding numbers //! with absolute value less than @p max_abs_val - _CCCL_DEVICE void binned_update(const ftype max_abs_val) + _CCCL_DEVICE_API void binned_update(const ftype max_abs_val) { const int X_index = binned_dindex(max_abs_val); const int shift = binned_index() - X_index; @@ -313,7 +313,7 @@ private: //! //! Performs the operation Y += X on an binned type Y where the index of Y is //! larger than the index of @p X - _CCCL_DEVICE void binned_deposit(const ftype X) + _CCCL_DEVICE_API void binned_deposit(const ftype X) { ftype M; ftype x = X; @@ -370,7 +370,7 @@ private: //! //! Renormalization keeps the primary vector within the necessary bins by //! shifting over to the carry vector - _CCCL_DEVICE _CCCL_FORCEINLINE void binned_renorm() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void binned_renorm() { _CCCL_PRAGMA_UNROLL_FULL() for (int i = 0; i < Fold; i++) @@ -389,7 +389,7 @@ private: //! Add scalar to manually specified binned fp (Y += X) //! //! Performs the operation Y += X on an binned type Y - _CCCL_DEVICE _CCCL_FORCEINLINE void binned_add(const ftype x) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void binned_add(const ftype x) { binned_update(x); binned_deposit(x); @@ -400,7 +400,7 @@ private: //! Performs the operation Y += X //! //! @param x Another binned fp of the same type - _CCCL_DEVICE void binned_add(const ReproducibleFloatingAccumulator& x) + _CCCL_DEVICE_API void binned_add(const ReproducibleFloatingAccumulator& x) { const auto X_index = x.binned_index(); const auto Y_index = this->binned_index(); @@ -457,7 +457,7 @@ private: binned_renorm(); } - [[nodiscard]] _CCCL_DEVICE double conv_binned_to_double() const + [[nodiscard]] _CCCL_DEVICE_API double conv_binned_to_double() const { int i = 0; double Y = 0.0; @@ -514,7 +514,7 @@ private: return Y; } - [[nodiscard]] _CCCL_DEVICE float conv_binned_to_float() const + [[nodiscard]] _CCCL_DEVICE_API float conv_binned_to_float() const { int i = 0; double Y = 0.0; @@ -546,15 +546,15 @@ private: } public: - ReproducibleFloatingAccumulator() = default; + _CCCL_HIDE_FROM_ABI ReproducibleFloatingAccumulator() = default; /// Set the binned fp to zero - _CCCL_DEVICE void zero() noexcept + _CCCL_DEVICE_API void zero() noexcept { data = {}; } - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE constexpr int endurance() const noexcept + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE constexpr int endurance() const noexcept { return 1 << (mant_dig - bin_width - 2); } @@ -563,7 +563,7 @@ public: //! NOTE: Casts @p x to the type of the binned fp _CCCL_TEMPLATE(typename U) _CCCL_REQUIRES(::cuda::std::is_arithmetic_v) - _CCCL_DEVICE ReproducibleFloatingAccumulator& operator+=(const U x) + _CCCL_DEVICE_API ReproducibleFloatingAccumulator& operator+=(const U x) { binned_add(static_cast(x)); return *this; @@ -573,14 +573,14 @@ public: //! NOTE: Casts @p x to the type of the binned fp _CCCL_TEMPLATE(typename U) _CCCL_REQUIRES(::cuda::std::is_arithmetic_v) - _CCCL_DEVICE ReproducibleFloatingAccumulator& operator-=(const U x) + _CCCL_DEVICE_API ReproducibleFloatingAccumulator& operator-=(const U x) { binned_add(-static_cast(x)); return *this; } /// Accumulate a binned fp @p x into the binned fp. - _CCCL_DEVICE ReproducibleFloatingAccumulator& operator+=(const ReproducibleFloatingAccumulator& other) + _CCCL_DEVICE_API ReproducibleFloatingAccumulator& operator+=(const ReproducibleFloatingAccumulator& other) { binned_add(other); return *this; @@ -588,18 +588,20 @@ public: //! Accumulate-subtract a binned fp @p other into the binned fp. //! NOTE: Makes a copy and performs arithmetic; slow. - _CCCL_DEVICE ReproducibleFloatingAccumulator& operator-=(const ReproducibleFloatingAccumulator& other) + _CCCL_DEVICE_API ReproducibleFloatingAccumulator& operator-=(const ReproducibleFloatingAccumulator& other) { const auto temp = -other; binned_add(temp); } - _CCCL_DEVICE friend bool operator==(const ReproducibleFloatingAccumulator& a, const ReproducibleFloatingAccumulator& b) + _CCCL_DEVICE_API friend bool + operator==(const ReproducibleFloatingAccumulator& a, const ReproducibleFloatingAccumulator& b) { return a.data == b.data; } - _CCCL_DEVICE friend bool operator!=(const ReproducibleFloatingAccumulator& a, const ReproducibleFloatingAccumulator& b) + _CCCL_DEVICE_API friend bool + operator!=(const ReproducibleFloatingAccumulator& a, const ReproducibleFloatingAccumulator& b) { return !(a == b); } @@ -608,7 +610,7 @@ public: //! NOTE: Casts @p x to the type of the binned fp _CCCL_TEMPLATE(typename U) _CCCL_REQUIRES(::cuda::std::is_arithmetic_v) - _CCCL_DEVICE ReproducibleFloatingAccumulator& operator=(const U x) + _CCCL_DEVICE_API ReproducibleFloatingAccumulator& operator=(const U x) { zero(); binned_add(static_cast(x)); @@ -617,7 +619,7 @@ public: //! Returns the negative of this binned fp //! NOTE: Makes a copy and performs arithmetic; slow. - [[nodiscard]] _CCCL_DEVICE ReproducibleFloatingAccumulator operator-() const + [[nodiscard]] _CCCL_DEVICE_API ReproducibleFloatingAccumulator operator-() const { ReproducibleFloatingAccumulator temp = *this; if (primary(0) != 0.0) @@ -633,7 +635,7 @@ public: } /// Convert this binned fp into its native floating-point representation - [[nodiscard]] _CCCL_DEVICE ftype conv_to_fp() const + [[nodiscard]] _CCCL_DEVICE_API ftype conv_to_fp() const { if (::cuda::std::is_same_v) { @@ -643,7 +645,7 @@ public: } /// Add @p x to the binned fp - _CCCL_DEVICE void add(const ftype x) + _CCCL_DEVICE_API void add(const ftype x) { binned_add(x); } @@ -657,7 +659,7 @@ public: //! Once rebinned, `endurance` values <= @p mav can be added to the accumulator //! with `unsafe_add` after which `renorm()` must be called. See the source of //!`add()` for an example - _CCCL_DEVICE void set_max_val(const ftype mav) + _CCCL_DEVICE_API void set_max_val(const ftype mav) { binned_update(mav); } @@ -665,7 +667,7 @@ public: //! Add @p x to the binned fp //! //! This is intended to be used after a call to `set_max_abs_val()` - _CCCL_DEVICE void unsafe_add(const ftype x) + _CCCL_DEVICE_API void unsafe_add(const ftype x) { binned_deposit(x); } @@ -674,7 +676,7 @@ public: //! //! This is intended to be used after a call to `set_max_abs_val()` and one or //! more calls to `unsafe_add()` - _CCCL_DEVICE void renorm() + _CCCL_DEVICE_API void renorm() { binned_renorm(); } diff --git a/cub/cub/detail/segmented_params.cuh b/cub/cub/detail/segmented_params.cuh index ffb416d8b8f3..471142ee3405 100644 --- a/cub/cub/detail/segmented_params.cuh +++ b/cub/cub/detail/segmented_params.cuh @@ -164,7 +164,7 @@ struct __validate_uniform_integral_param // behavior. Compiled out when assertions are disabled (the bounds are not even computed). Gated on the argument's // integer element type: `bool` and character types are excluded, matching the `cmp_*` comparators. template -_CCCL_HOST_DEVICE constexpr void +_CCCL_HOST_DEVICE_API constexpr void __assert_param_in_bounds([[maybe_unused]] const Arg& __arg, [[maybe_unused]] const Value& __value) noexcept { using __element_t = typename ::cuda::args::__traits::element_type; @@ -188,7 +188,7 @@ __assert_param_in_bounds([[maybe_unused]] const Arg& __arg, [[maybe_unused]] con //! @return The single argument value, or the sequence element at the given index. _CCCL_TEMPLATE(class Tp, class SegmentIndexT) _CCCL_REQUIRES((!::cuda::args::__is_wrapper_v<::cuda::std::remove_cvref_t>) ) -[[nodiscard]] _CCCL_HOST_DEVICE constexpr auto get_param(Tp&& __arg, [[maybe_unused]] SegmentIndexT __index) noexcept +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_param(Tp&& __arg, [[maybe_unused]] SegmentIndexT __index) noexcept { if constexpr (::cuda::args::__traits<::cuda::std::remove_cvref_t>::is_single_value) { @@ -201,21 +201,21 @@ _CCCL_REQUIRES((!::cuda::args::__is_wrapper_v<::cuda::std::remove_cvref_t>) } template -[[nodiscard]] _CCCL_HOST_DEVICE constexpr auto +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_param(const ::cuda::args::constant& __arg, [[maybe_unused]] SegmentIndexT __index) noexcept { return ::cuda::args::__unwrap(__arg); } template -[[nodiscard]] _CCCL_HOST_DEVICE constexpr auto +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_param(const ::cuda::args::immediate& __arg, [[maybe_unused]] SegmentIndexT __index) noexcept { return ::cuda::args::__unwrap(__arg); } template -[[nodiscard]] _CCCL_HOST_DEVICE constexpr auto +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_param(const ::cuda::args::deferred& __arg, [[maybe_unused]] SegmentIndexT __index) noexcept { // A single `deferred` wraps a handle to a device-side value (a pointer or input iterator), not the value itself; @@ -229,7 +229,7 @@ get_param(const ::cuda::args::deferred& __arg, [[maybe_unused } template -[[nodiscard]] _CCCL_HOST_DEVICE constexpr auto +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_param(const ::cuda::args::deferred_sequence& __arg, SegmentIndexT __index) noexcept { // Materialize into the scalar element type: `handle[index]` may yield a proxy reference that must not leak into the @@ -246,7 +246,7 @@ get_param(const ::cuda::args::deferred_sequence& __arg, Segme // negative value as a huge unsigned one. A negative count thus becomes "no work". Deferred forms are range-checked // against their declared bounds by `get_param` in debug builds. template -[[nodiscard]] _CCCL_HOST_DEVICE constexpr auto +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto __get_and_clamp_param_to_nonnegative(const Arg& __arg, SegmentIndexT __index) noexcept { // Materialize into the scalar element type: the generic `get_param` may yield a proxy reference for fancy iterators, @@ -318,7 +318,7 @@ using __retype_static_bounds_t _CCCL_NODEBUG = typename __retype_static_bounds>) ) -[[nodiscard]] _CCCL_HOST_DEVICE constexpr ::cuda::std::remove_cvref_t normalize_param(Tp&& __arg) noexcept +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::remove_cvref_t normalize_param(Tp&& __arg) noexcept { return ::cuda::std::forward(__arg); } @@ -326,7 +326,7 @@ _CCCL_REQUIRES((!::cuda::args::__is_wrapper_v<::cuda::std::remove_cvref_t>) //! @brief Compile-time constants: integer values are re-wrapped with a value-chosen element type. Everything else //! (enumerators, pointers) already spells its type in any constant expression and is kept as is. template -[[nodiscard]] _CCCL_HOST_DEVICE constexpr auto +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto normalize_param([[maybe_unused]] ::cuda::args::constant __arg) noexcept { if constexpr (::cuda::std::__cccl_is_integer_v<::cuda::std::remove_cv_t>) @@ -341,13 +341,14 @@ normalize_param([[maybe_unused]] ::cuda::args::constant __arg) noexce } template -[[nodiscard]] _CCCL_HOST_DEVICE constexpr auto normalize_param(::cuda::args::__constant_sequence __arg) noexcept +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto +normalize_param(::cuda::args::__constant_sequence __arg) noexcept { return __arg; } template -[[nodiscard]] _CCCL_HOST_DEVICE constexpr auto +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto normalize_param(const ::cuda::args::immediate& __arg) noexcept { using __bounds_t = __retype_static_bounds_t<::cuda::args::__element_type_of_t, StaticBounds>; @@ -355,7 +356,7 @@ normalize_param(const ::cuda::args::immediate& __arg) noexcep } template -[[nodiscard]] _CCCL_HOST_DEVICE constexpr auto +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto normalize_param(const ::cuda::args::__immediate_sequence& __arg) noexcept { using __bounds_t = __retype_static_bounds_t<::cuda::args::__element_type_of_t, StaticBounds>; @@ -364,7 +365,7 @@ normalize_param(const ::cuda::args::__immediate_sequence& __a } template -[[nodiscard]] _CCCL_HOST_DEVICE constexpr auto +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto normalize_param(const ::cuda::args::deferred& __arg) noexcept { using __bounds_t = __retype_static_bounds_t<::cuda::args::__element_type_of_t, StaticBounds>; @@ -373,7 +374,7 @@ normalize_param(const ::cuda::args::deferred& __arg) noexcept } template -[[nodiscard]] _CCCL_HOST_DEVICE constexpr auto +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto normalize_param(const ::cuda::args::deferred_sequence& __arg) noexcept { using __bounds_t = __retype_static_bounds_t<::cuda::args::__element_type_of_t, StaticBounds>; @@ -406,7 +407,7 @@ struct static_discrete_param using supported_options_t = supported_options; template - [[nodiscard]] _CCCL_HOST_DEVICE constexpr T get_param(SegmentIndexT) const noexcept + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr T get_param(SegmentIndexT) const noexcept { return Value; } @@ -424,7 +425,7 @@ struct static_discrete_param //! @param[in] f Functor invoked with the matched compile-time constant. //! @return `true` if the value matches one of the supported options. template -[[nodiscard]] _CCCL_HOST_DEVICE bool +[[nodiscard]] _CCCL_HOST_DEVICE_API bool dispatch_impl(T val, [[maybe_unused]] supported_options __supported_options, Functor&& f) { const bool match_found = @@ -442,7 +443,7 @@ dispatch_impl(T val, [[maybe_unused]] supported_options __supported_ //! @param[in] f Functor invoked with the matched compile-time constant. //! @return `true` if the parameter value matches one of its supported options. template -[[nodiscard]] _CCCL_HOST_DEVICE bool dispatch_discrete(ParamT param, SegmentIndexT segment_id, Functor&& f) +[[nodiscard]] _CCCL_HOST_DEVICE_API bool dispatch_discrete(ParamT param, SegmentIndexT segment_id, Functor&& f) { using supported_list = typename ParamT::supported_options_t; auto param_value = param.get_param(segment_id); diff --git a/cub/cub/detail/segmented_scan_helpers.cuh b/cub/cub/detail/segmented_scan_helpers.cuh index db9d654e6546..2d34c54aac7b 100644 --- a/cub/cub/detail/segmented_scan_helpers.cuh +++ b/cub/cub/detail/segmented_scan_helpers.cuh @@ -50,7 +50,7 @@ template struct initial_value_converter { template - _CCCL_DEVICE _CCCL_FORCEINLINE static constexpr ToT cast(FromT v) noexcept + _CCCL_DEVICE_API _CCCL_FORCEINLINE static constexpr ToT cast(FromT v) noexcept { return static_cast(v); } @@ -60,7 +60,7 @@ template struct initial_value_converter> { template - _CCCL_DEVICE _CCCL_FORCEINLINE static constexpr augmented_value_t + _CCCL_DEVICE_API _CCCL_FORCEINLINE static constexpr augmented_value_t cast(augmented_value_t fv) noexcept { return {static_cast(fv.value), static_cast(fv.flag)}; @@ -68,7 +68,7 @@ struct initial_value_converter> }; template -_CCCL_DEVICE _CCCL_FORCEINLINE constexpr ToT convert_initial_value(FromT v) noexcept +_CCCL_DEVICE_API _CCCL_FORCEINLINE constexpr ToT convert_initial_value(FromT v) noexcept { return initial_value_converter::cast(v); } @@ -79,7 +79,7 @@ struct schwarz_scan_op using fv_t = augmented_value_t; mutable BinaryOpT scan_op; - _CCCL_DEVICE _CCCL_FORCEINLINE fv_t operator()(fv_t o1, fv_t o2) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE fv_t operator()(fv_t o1, fv_t o2) const { if (o2.flag) { @@ -94,7 +94,7 @@ struct schwarz_scan_op template struct packer { - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE constexpr auto operator()(V v, F f) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE constexpr auto operator()(V v, F f) const { return augmented_value_t{v, f}; } @@ -106,7 +106,7 @@ struct packer_iv mutable ScanOp op; V init_v; - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE constexpr auto operator()(V v, F f) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE constexpr auto operator()(V v, F f) const { V res = v; if (f) @@ -120,7 +120,7 @@ struct packer_iv template struct projector { - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE constexpr auto operator()(V v, F) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE constexpr auto operator()(V v, F) const { return v; } @@ -131,7 +131,7 @@ struct projector_iv { V init_v; - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE constexpr auto operator()(V v, F f) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE constexpr auto operator()(V v, F f) const { return f ? init_v : v; } @@ -159,11 +159,11 @@ public: logical_offset_t logical_offset; }; - _CCCL_DEVICE _CCCL_FORCEINLINE bag_of_segments(::cuda::std::span cum_sizes) + _CCCL_DEVICE_API _CCCL_FORCEINLINE bag_of_segments(::cuda::std::span cum_sizes) : m_offsets(cum_sizes) {} - _CCCL_DEVICE _CCCL_FORCEINLINE search_data_t find(logical_offset_t elem_id) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE search_data_t find(logical_offset_t elem_id) const { const bool is_small = (m_offsets.size() < LinearBinarySearchThreshold); const auto pos = is_small ? locate_linear_search(elem_id) : locate_binary_search(elem_id); @@ -179,7 +179,7 @@ private: // prefix sum of segment sizes. Expectation: 0 <= pos < last element of m_offsets // Linear search - _CCCL_DEVICE _CCCL_FORCEINLINE search_data_t locate_linear_search(logical_offset_t pos) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE search_data_t locate_linear_search(logical_offset_t pos) const { const auto offset_size = m_offsets.size(); @@ -201,7 +201,7 @@ private: } // Binary search - _CCCL_DEVICE _CCCL_FORCEINLINE search_data_t locate_binary_search(logical_offset_t pos) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE search_data_t locate_binary_search(logical_offset_t pos) const { const auto offset_size = m_offsets.size(); const auto beg_it = m_offsets.data(); @@ -235,12 +235,12 @@ public: static constexpr segment_id_t max_offset_size = MaxBagSize; - _CCCL_DEVICE _CCCL_FORCEINLINE statically_bound_bag_of_segments( + _CCCL_DEVICE_API _CCCL_FORCEINLINE statically_bound_bag_of_segments( ::cuda::std::span cum_sizes, ::cuda::std::integral_constant) : m_offsets(cum_sizes) {} - _CCCL_DEVICE _CCCL_FORCEINLINE search_data_t find(logical_offset_t elem_id) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE search_data_t find(logical_offset_t elem_id) const { if constexpr (max_offset_size < LinearBinarySearchThreshold) { @@ -253,7 +253,7 @@ public: } private: - _CCCL_DEVICE _CCCL_FORCEINLINE bool + _CCCL_DEVICE_API _CCCL_FORCEINLINE bool is_it_past_this_segment(logical_offset_t offset, segment_id_t segment_id, segment_id_t size) const { return ((segment_id < size) && (m_offsets[segment_id] <= offset)); @@ -267,7 +267,7 @@ private: // prefix sum of segment sizes. Expectation: 0 <= pos < last element of m_offsets // Linear search - _CCCL_DEVICE _CCCL_FORCEINLINE search_data_t locate_linear_search(logical_offset_t pos) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE search_data_t locate_linear_search(logical_offset_t pos) const { const auto n_offsets = m_offsets.size(); @@ -284,7 +284,7 @@ private: } // Branchless binary search - _CCCL_DEVICE _CCCL_FORCEINLINE search_data_t locate_binary_search(logical_offset_t pos) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE search_data_t locate_binary_search(logical_offset_t pos) const { constexpr segment_id_t start = ::cuda::std::bit_ceil(max_offset_size) >> 1; @@ -308,7 +308,7 @@ private: }; template -_CCCL_DEVICE _CCCL_FORCEINLINE auto make_statically_bound_bag_of_segments(::cuda::std::span span) +_CCCL_DEVICE_API _CCCL_FORCEINLINE auto make_statically_bound_bag_of_segments(::cuda::std::span span) { return statically_bound_bag_of_segments{span, {}}; } @@ -329,11 +329,11 @@ public: SizeT logical_offset; }; - _CCCL_DEVICE _CCCL_FORCEINLINE bag_of_fixed_size_segments(SizeT segment_size) + _CCCL_DEVICE_API _CCCL_FORCEINLINE bag_of_fixed_size_segments(SizeT segment_size) : m_segment_size(segment_size) {} - _CCCL_DEVICE _CCCL_FORCEINLINE search_data_t find(logical_offset_t elem_id) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE search_data_t find(logical_offset_t elem_id) const { const SizeT segment_id = elem_id / m_segment_size; const SizeT relative_offset = elem_id - segment_id * m_segment_size; @@ -374,35 +374,35 @@ struct multi_segmented_input_iterator bool m_head_flag; ReadTransformT m_read_fn; - _CCCL_DEVICE _CCCL_FORCEINLINE operator value_type() const + _CCCL_DEVICE_API _CCCL_FORCEINLINE operator value_type() const { return m_read_fn(m_it[m_offset], m_head_flag); } - _CCCL_DEVICE _CCCL_FORCEINLINE __mapping_proxy& operator=(const __mapping_proxy& other) + _CCCL_DEVICE_API _CCCL_FORCEINLINE __mapping_proxy& operator=(const __mapping_proxy& other) { return (*this = static_cast(other)); } }; - _CCCL_DEVICE _CCCL_FORCEINLINE __mapping_proxy operator*() const + _CCCL_DEVICE_API _CCCL_FORCEINLINE __mapping_proxy operator*() const { return make_proxy(0); } - _CCCL_DEVICE _CCCL_FORCEINLINE __mapping_proxy operator[](difference_type n) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE __mapping_proxy operator[](difference_type n) const { return make_proxy(n); } - _CCCL_DEVICE _CCCL_FORCEINLINE friend multi_segmented_input_iterator + _CCCL_DEVICE_API _CCCL_FORCEINLINE friend multi_segmented_input_iterator operator+(const multi_segmented_input_iterator& iter, difference_type n) { return {iter.m_it, iter.m_start + n, iter.m_searcher, iter.m_it_idx_begin, iter.m_read_transform_fn}; } private: - _CCCL_DEVICE _CCCL_FORCEINLINE __mapping_proxy make_proxy(difference_type n) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE __mapping_proxy make_proxy(difference_type n) const { const auto [segment_id, rel_offset] = m_searcher.find(m_start + n); @@ -448,31 +448,31 @@ struct multi_segmented_output_iterator WriteTransformT m_write_fn; template - _CCCL_DEVICE _CCCL_FORCEINLINE __mapping_proxy& operator=(augmented_value_t new_value) + _CCCL_DEVICE_API _CCCL_FORCEINLINE __mapping_proxy& operator=(augmented_value_t new_value) { m_it[m_offset] = m_write_fn(new_value.value, m_head_flag); return *this; } }; - _CCCL_DEVICE _CCCL_FORCEINLINE __mapping_proxy operator*() const + _CCCL_DEVICE_API _CCCL_FORCEINLINE __mapping_proxy operator*() const { return make_proxy(0); } - _CCCL_DEVICE _CCCL_FORCEINLINE __mapping_proxy operator[](difference_type n) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE __mapping_proxy operator[](difference_type n) const { return make_proxy(n); } - _CCCL_DEVICE _CCCL_FORCEINLINE friend multi_segmented_output_iterator + _CCCL_DEVICE_API _CCCL_FORCEINLINE friend multi_segmented_output_iterator operator+(const multi_segmented_output_iterator& iter, difference_type n) { return {iter.m_it, iter.m_start + n, iter.m_searcher, iter.m_it_idx_begin, iter.m_write_transform_fn}; } private: - _CCCL_DEVICE _CCCL_FORCEINLINE __mapping_proxy make_proxy(difference_type n) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE __mapping_proxy make_proxy(difference_type n) const { const auto [segment_id, rel_offset] = m_searcher.find(m_start + n); @@ -498,12 +498,12 @@ struct worker_prefix_callback_t PrefixT& m_exclusive_prefix; BinaryOpT m_scan_op; - _CCCL_DEVICE _CCCL_FORCEINLINE worker_prefix_callback_t(PrefixT& prefix, BinaryOpT& op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE worker_prefix_callback_t(PrefixT& prefix, BinaryOpT& op) : m_exclusive_prefix(prefix) , m_scan_op(op) {} - _CCCL_DEVICE _CCCL_FORCEINLINE PrefixT operator()(PrefixT block_aggregate) + _CCCL_DEVICE_API _CCCL_FORCEINLINE PrefixT operator()(PrefixT block_aggregate) { const PrefixT previous_prefix = m_exclusive_prefix; m_exclusive_prefix = m_scan_op(m_exclusive_prefix, block_aggregate); diff --git a/cub/cub/detail/strong_load.cuh b/cub/cub/detail/strong_load.cuh index 597fc6572d92..3a63aa187b77 100644 --- a/cub/cub/detail/strong_load.cuh +++ b/cub/cub/detail/strong_load.cuh @@ -26,7 +26,7 @@ CUB_NAMESPACE_BEGIN namespace detail { -_CCCL_DEVICE _CCCL_FORCEINLINE uint4 load_relaxed(uint4 const* ptr) +_CCCL_DEVICE_API _CCCL_FORCEINLINE uint4 load_relaxed(uint4 const* ptr) { uint4 retval; NV_IF_ELSE_TARGET( @@ -42,7 +42,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE uint4 load_relaxed(uint4 const* ptr) return retval; } -_CCCL_DEVICE _CCCL_FORCEINLINE ulonglong2 load_relaxed(ulonglong2 const* ptr) +_CCCL_DEVICE_API _CCCL_FORCEINLINE ulonglong2 load_relaxed(ulonglong2 const* ptr) { ulonglong2 retval; NV_IF_ELSE_TARGET( @@ -52,7 +52,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE ulonglong2 load_relaxed(ulonglong2 const* ptr) return retval; } -_CCCL_DEVICE _CCCL_FORCEINLINE ushort4 load_relaxed(ushort4 const* ptr) +_CCCL_DEVICE_API _CCCL_FORCEINLINE ushort4 load_relaxed(ushort4 const* ptr) { ushort4 retval; NV_IF_ELSE_TARGET( @@ -68,7 +68,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE ushort4 load_relaxed(ushort4 const* ptr) return retval; } -_CCCL_DEVICE _CCCL_FORCEINLINE uint2 load_relaxed(uint2 const* ptr) +_CCCL_DEVICE_API _CCCL_FORCEINLINE uint2 load_relaxed(uint2 const* ptr) { uint2 retval; NV_IF_ELSE_TARGET( @@ -78,7 +78,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE uint2 load_relaxed(uint2 const* ptr) return retval; } -_CCCL_DEVICE _CCCL_FORCEINLINE unsigned long long load_relaxed(unsigned long long const* ptr) +_CCCL_DEVICE_API _CCCL_FORCEINLINE unsigned long long load_relaxed(unsigned long long const* ptr) { unsigned long long retval; NV_IF_ELSE_TARGET(NV_PROVIDES_SM_70, @@ -87,7 +87,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE unsigned long long load_relaxed(unsigned long lon return retval; } -_CCCL_DEVICE _CCCL_FORCEINLINE unsigned int load_relaxed(unsigned int const* ptr) +_CCCL_DEVICE_API _CCCL_FORCEINLINE unsigned int load_relaxed(unsigned int const* ptr) { unsigned int retval; NV_IF_ELSE_TARGET(NV_PROVIDES_SM_70, @@ -97,7 +97,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE unsigned int load_relaxed(unsigned int const* ptr return retval; } -_CCCL_DEVICE _CCCL_FORCEINLINE unsigned short load_relaxed(unsigned short const* ptr) +_CCCL_DEVICE_API _CCCL_FORCEINLINE unsigned short load_relaxed(unsigned short const* ptr) { unsigned short retval; NV_IF_ELSE_TARGET(NV_PROVIDES_SM_70, @@ -106,7 +106,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE unsigned short load_relaxed(unsigned short const* return retval; } -_CCCL_DEVICE _CCCL_FORCEINLINE unsigned char load_relaxed(unsigned char const* ptr) +_CCCL_DEVICE_API _CCCL_FORCEINLINE unsigned char load_relaxed(unsigned char const* ptr) { unsigned short retval; NV_IF_ELSE_TARGET( @@ -124,7 +124,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE unsigned char load_relaxed(unsigned char const* p return (unsigned char) retval; } -_CCCL_DEVICE _CCCL_FORCEINLINE ulonglong2 load_acquire(ulonglong2 const* ptr) +_CCCL_DEVICE_API _CCCL_FORCEINLINE ulonglong2 load_acquire(ulonglong2 const* ptr) { ulonglong2 retval; NV_IF_ELSE_TARGET( @@ -137,7 +137,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE ulonglong2 load_acquire(ulonglong2 const* ptr) return retval; } -_CCCL_DEVICE _CCCL_FORCEINLINE uint2 load_acquire(uint2 const* ptr) +_CCCL_DEVICE_API _CCCL_FORCEINLINE uint2 load_acquire(uint2 const* ptr) { uint2 retval; NV_IF_ELSE_TARGET( @@ -150,7 +150,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE uint2 load_acquire(uint2 const* ptr) return retval; } -_CCCL_DEVICE _CCCL_FORCEINLINE unsigned int load_acquire(unsigned int const* ptr) +_CCCL_DEVICE_API _CCCL_FORCEINLINE unsigned int load_acquire(unsigned int const* ptr) { unsigned int retval; NV_IF_ELSE_TARGET(NV_PROVIDES_SM_70, diff --git a/cub/cub/detail/strong_store.cuh b/cub/cub/detail/strong_store.cuh index 57ca7c038f59..a42aad5bedcc 100644 --- a/cub/cub/detail/strong_store.cuh +++ b/cub/cub/detail/strong_store.cuh @@ -26,7 +26,7 @@ CUB_NAMESPACE_BEGIN namespace detail { -_CCCL_DEVICE _CCCL_FORCEINLINE void store_relaxed(uint4* ptr, uint4 val) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void store_relaxed(uint4* ptr, uint4 val) { NV_IF_ELSE_TARGET( NV_PROVIDES_SM_70, @@ -39,7 +39,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void store_relaxed(uint4* ptr, uint4 val) "st.cg.v4.u32 [%0], {%1, %2, %3, %4};" : : "l"(ptr), "r"(val.x), "r"(val.y), "r"(val.z), "r"(val.w) : "memory");)); } -_CCCL_DEVICE _CCCL_FORCEINLINE void store_relaxed(ulonglong2* ptr, ulonglong2 val) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void store_relaxed(ulonglong2* ptr, ulonglong2 val) { NV_IF_ELSE_TARGET( NV_PROVIDES_SM_70, @@ -47,7 +47,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void store_relaxed(ulonglong2* ptr, ulonglong2 va (asm volatile("st.cg.v2.u64 [%0], {%1, %2};" : : "l"(ptr), "l"(val.x), "l"(val.y) : "memory");)); } -_CCCL_DEVICE _CCCL_FORCEINLINE void store_relaxed(ushort4* ptr, ushort4 val) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void store_relaxed(ushort4* ptr, ushort4 val) { NV_IF_ELSE_TARGET( NV_PROVIDES_SM_70, @@ -60,7 +60,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void store_relaxed(ushort4* ptr, ushort4 val) "st.cg.v4.u16 [%0], {%1, %2, %3, %4};" : : "l"(ptr), "h"(val.x), "h"(val.y), "h"(val.z), "h"(val.w) : "memory");)); } -_CCCL_DEVICE _CCCL_FORCEINLINE void store_relaxed(uint2* ptr, uint2 val) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void store_relaxed(uint2* ptr, uint2 val) { NV_IF_ELSE_TARGET( NV_PROVIDES_SM_70, @@ -69,7 +69,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void store_relaxed(uint2* ptr, uint2 val) } // NOLINTNEXTLINE(readability-non-const-parameter) -_CCCL_DEVICE _CCCL_FORCEINLINE void store_relaxed(unsigned long long* ptr, unsigned long long val) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void store_relaxed(unsigned long long* ptr, unsigned long long val) { NV_IF_ELSE_TARGET(NV_PROVIDES_SM_70, (asm volatile("st.relaxed.gpu.u64 [%0], %1;" : : "l"(ptr), "l"(val) : "memory");), @@ -77,7 +77,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void store_relaxed(unsigned long long* ptr, unsig } // NOLINTNEXTLINE(readability-non-const-parameter) -_CCCL_DEVICE _CCCL_FORCEINLINE void store_relaxed(unsigned int* ptr, unsigned int val) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void store_relaxed(unsigned int* ptr, unsigned int val) { NV_IF_ELSE_TARGET(NV_PROVIDES_SM_70, (asm volatile("st.relaxed.gpu.u32 [%0], %1;" : : "l"(ptr), "r"(val) : "memory");), @@ -85,7 +85,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void store_relaxed(unsigned int* ptr, unsigned in } // NOLINTNEXTLINE(readability-non-const-parameter) -_CCCL_DEVICE _CCCL_FORCEINLINE void store_relaxed(unsigned short* ptr, unsigned short val) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void store_relaxed(unsigned short* ptr, unsigned short val) { NV_IF_ELSE_TARGET(NV_PROVIDES_SM_70, (asm volatile("st.relaxed.gpu.u16 [%0], %1;" : : "l"(ptr), "h"(val) : "memory");), @@ -93,7 +93,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void store_relaxed(unsigned short* ptr, unsigned } // NOLINTNEXTLINE(readability-non-const-parameter) -_CCCL_DEVICE _CCCL_FORCEINLINE void store_relaxed(unsigned char* ptr, unsigned char val) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void store_relaxed(unsigned char* ptr, unsigned char val) { NV_IF_ELSE_TARGET( NV_PROVIDES_SM_70, @@ -111,7 +111,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void store_relaxed(unsigned char* ptr, unsigned c "h"((unsigned short) val) : "memory");)); } -_CCCL_DEVICE _CCCL_FORCEINLINE void store_release(uint4* ptr, uint4 val) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void store_release(uint4* ptr, uint4 val) { NV_IF_ELSE_TARGET( NV_PROVIDES_SM_70, @@ -129,7 +129,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void store_release(uint4* ptr, uint4 val) })); } -_CCCL_DEVICE _CCCL_FORCEINLINE void store_release(ulonglong2* ptr, ulonglong2 val) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void store_release(ulonglong2* ptr, ulonglong2 val) { NV_IF_ELSE_TARGET( NV_PROVIDES_SM_70, @@ -140,7 +140,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void store_release(ulonglong2* ptr, ulonglong2 va })); } -_CCCL_DEVICE _CCCL_FORCEINLINE void store_release(ushort4* ptr, ushort4 val) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void store_release(ushort4* ptr, ushort4 val) { NV_IF_ELSE_TARGET( NV_PROVIDES_SM_70, @@ -158,7 +158,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void store_release(ushort4* ptr, ushort4 val) })); } -_CCCL_DEVICE _CCCL_FORCEINLINE void store_release(uint2* ptr, uint2 val) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void store_release(uint2* ptr, uint2 val) { NV_IF_ELSE_TARGET( NV_PROVIDES_SM_70, @@ -170,7 +170,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void store_release(uint2* ptr, uint2 val) } // NOLINTNEXTLINE(readability-non-const-parameter) -_CCCL_DEVICE _CCCL_FORCEINLINE void store_release(unsigned long long* ptr, unsigned long long val) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void store_release(unsigned long long* ptr, unsigned long long val) { NV_IF_ELSE_TARGET( NV_PROVIDES_SM_70, (asm volatile("st.release.gpu.u64 [%0], %1;" : : "l"(ptr), "l"(val) : "memory");), ({ @@ -180,7 +180,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void store_release(unsigned long long* ptr, unsig } // NOLINTNEXTLINE(readability-non-const-parameter) -_CCCL_DEVICE _CCCL_FORCEINLINE void store_release(unsigned int* ptr, unsigned int val) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void store_release(unsigned int* ptr, unsigned int val) { NV_IF_ELSE_TARGET( NV_PROVIDES_SM_70, (asm volatile("st.release.gpu.u32 [%0], %1;" : : "l"(ptr), "r"(val) : "memory");), ({ @@ -190,7 +190,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void store_release(unsigned int* ptr, unsigned in } // NOLINTNEXTLINE(readability-non-const-parameter) -_CCCL_DEVICE _CCCL_FORCEINLINE void store_release(unsigned short* ptr, unsigned short val) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void store_release(unsigned short* ptr, unsigned short val) { NV_IF_ELSE_TARGET( NV_PROVIDES_SM_70, (asm volatile("st.release.gpu.u16 [%0], %1;" : : "l"(ptr), "h"(val) : "memory");), ({ @@ -200,7 +200,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void store_release(unsigned short* ptr, unsigned } // NOLINTNEXTLINE(readability-non-const-parameter) -_CCCL_DEVICE _CCCL_FORCEINLINE void store_release(unsigned char* ptr, unsigned char val) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void store_release(unsigned char* ptr, unsigned char val) { NV_IF_ELSE_TARGET( NV_PROVIDES_SM_70, diff --git a/cub/cub/detail/temporary_storage.cuh b/cub/cub/detail/temporary_storage.cuh index e3fe42a24c55..2bf6e798a4a4 100644 --- a/cub/cub/detail/temporary_storage.cuh +++ b/cub/cub/detail/temporary_storage.cuh @@ -86,7 +86,7 @@ class slot void* m_pointer{}; public: - slot() = default; + _CCCL_HIDE_FROM_ABI slot() = default; /** * @brief Returns an array of type @p T and length @p elements @@ -95,21 +95,21 @@ public: _CCCL_HOST_DEVICE alias create_alias(size_t elements = 0); private: - _CCCL_HOST_DEVICE void set_bytes_required(size_t new_size) + _CCCL_HOST_DEVICE_API void set_bytes_required(size_t new_size) { m_size = (::cuda::std::max) (m_size, new_size); } - _CCCL_HOST_DEVICE size_t get_bytes_required() const + _CCCL_HOST_DEVICE_API size_t get_bytes_required() const { return m_size; } - _CCCL_HOST_DEVICE void set_storage(void* ptr) + _CCCL_HOST_DEVICE_API void set_storage(void* ptr) { m_pointer = ptr; } - _CCCL_HOST_DEVICE void* get_storage() const + _CCCL_HOST_DEVICE_API void* get_storage() const { return m_pointer; } @@ -138,14 +138,14 @@ class alias slot& m_slot; size_t m_elements{}; - _CCCL_HOST_DEVICE explicit alias(slot& slot, size_t elements = 0) + _CCCL_HOST_DEVICE_API explicit alias(slot& slot, size_t elements = 0) : m_slot(slot) , m_elements(elements) { this->update_slot(); } - _CCCL_HOST_DEVICE void update_slot() + _CCCL_HOST_DEVICE_API void update_slot() { m_slot.set_bytes_required(m_elements * sizeof(T)); } @@ -163,7 +163,7 @@ public: * temporary slot to fit up to @p new_elements items * of type @p T. */ - _CCCL_HOST_DEVICE void grow(size_t new_elements) + _CCCL_HOST_DEVICE_API void grow(size_t new_elements) { m_elements = new_elements; this->update_slot(); @@ -175,7 +175,7 @@ public: * If the @p elements number is equal to zero, or storage layout isn't mapped, * @p nullptr is returned. */ - _CCCL_HOST_DEVICE T* get() const + _CCCL_HOST_DEVICE_API T* get() const { if (m_elements == 0) { @@ -189,7 +189,7 @@ public: }; template -_CCCL_HOST_DEVICE alias slot::create_alias(size_t elements) +_CCCL_HOST_DEVICE_API alias slot::create_alias(size_t elements) { return alias(*this, elements); } @@ -261,9 +261,9 @@ class layout bool m_layout_was_mapped{}; public: - layout() = default; + _CCCL_HIDE_FROM_ABI layout() = default; - _CCCL_HOST_DEVICE slot* get_slot(int slot_id) + _CCCL_HOST_DEVICE_API slot* get_slot(int slot_id) { if (slot_id < SlotsCount) { @@ -276,7 +276,7 @@ public: /** * @brief Returns required temporary storage size in bytes */ - _CCCL_HOST_DEVICE size_t get_size() + _CCCL_HOST_DEVICE_API size_t get_size() { this->prepare_interface(); @@ -306,7 +306,7 @@ public: /** * @brief Maps the layout to the temporary storage buffer. */ - _CCCL_HOST_DEVICE cudaError_t map_to_buffer(void* d_temp_storage, size_t temp_storage_bytes) + _CCCL_HOST_DEVICE_API cudaError_t map_to_buffer(void* d_temp_storage, size_t temp_storage_bytes) { if (m_layout_was_mapped) { @@ -331,7 +331,7 @@ public: } private: - _CCCL_HOST_DEVICE void prepare_interface() + _CCCL_HOST_DEVICE_API void prepare_interface() { if (m_layout_was_mapped) { diff --git a/cub/cub/detail/topk_helpers.cuh b/cub/cub/detail/topk_helpers.cuh index 6542e28a99d8..d98f005d6b82 100644 --- a/cub/cub/detail/topk_helpers.cuh +++ b/cub/cub/detail/topk_helpers.cuh @@ -42,13 +42,13 @@ struct key_prefix_storage_t // Calculates the number of passes needed for a type T with BitsPerPass bits processed per pass. template -[[nodiscard]] _CCCL_HOST_DEVICE _CCCL_FORCEINLINE constexpr int calc_num_passes(int bits_per_pass) +[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE constexpr int calc_num_passes(int bits_per_pass) { return ::cuda::ceil_div(sizeof(T) * CHAR_BIT, bits_per_pass); } template -[[nodiscard]] _CCCL_HOST_DEVICE _CCCL_FORCEINLINE int calc_num_passes(const int total_bits) +[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE int calc_num_passes(const int total_bits) { return ::cuda::ceil_div(total_bits, BitsPerPass); } @@ -56,7 +56,7 @@ template // Calculates the starting bit for a given pass (bit 0 is the least significant (rightmost) bit). // We process the input from the most to the least significant bit. This way, we can skip some passes in the end. template -[[nodiscard]] _CCCL_HOST_DEVICE _CCCL_FORCEINLINE constexpr int calc_start_bit(const int pass) +[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE constexpr int calc_start_bit(const int pass) { int start_bit = (int{sizeof(T)} * CHAR_BIT) - ((pass + 1) * BitsPerPass); start_bit = ::cuda::std::max(start_bit, 0); @@ -64,7 +64,7 @@ template } template -[[nodiscard]] _CCCL_HOST_DEVICE _CCCL_FORCEINLINE int calc_start_bit(const int total_bits, const int pass) +[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE int calc_start_bit(const int total_bits, const int pass) { int start_bit = total_bits - ((pass + 1) * BitsPerPass); start_bit = ::cuda::std::max(start_bit, 0); @@ -83,7 +83,7 @@ struct key_prefix_storage_t // Funnel-shifts the entire bit-vector left by `shift` positions and inserts `value` into the // vacated low bits. Each word receives carry bits from its lower neighbor (high-to-low order // so each word reads its neighbor's original value). The final word is filled from `value`. - _CCCL_DEVICE _CCCL_FORCEINLINE void shift_or(int shift, unsigned int value) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void shift_or(int shift, unsigned int value) { _CCCL_ASSERT(shift > 0 && shift < 32, "shift_or requires 0 < shift < 32"); _CCCL_PRAGMA_UNROLL_FULL() @@ -96,7 +96,7 @@ struct key_prefix_storage_t }; template -_CCCL_DEVICE _CCCL_FORCEINLINE void +_CCCL_DEVICE_API _CCCL_FORCEINLINE void set_kth_key_bits(key_prefix_storage_t& prefix, const int pass, const int bin_index) { if constexpr (detail::radix::can_twiddle) @@ -124,7 +124,7 @@ enum class candidate_class // Used in the bin ID calculation to exclude bits unrelated to the current pass template -[[nodiscard]] _CCCL_HOST_DEVICE _CCCL_FORCEINLINE constexpr unsigned calc_mask(const int pass) +[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE constexpr unsigned calc_mask(const int pass) { const int num_bits = _CUB::detail::topk::calc_start_bit(pass - 1) - _CUB::detail::topk::calc_start_bit(pass); @@ -138,7 +138,7 @@ template // fix-up. Every sign-clear pattern is unchanged and every sign-set pattern moves by exactly +1, so the relative order // is preserved everywhere (-NaN stays below -inf, +NaN above +inf) and the only new equivalence is the two zeros. template -[[nodiscard]] _CCCL_HOST_DEVICE _CCCL_FORCEINLINE typename Traits::UnsignedBits +[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE typename Traits::UnsignedBits twiddle_in_bits(typename Traits::UnsignedBits bits) { using bits_t = typename Traits::UnsignedBits; @@ -182,13 +182,13 @@ struct extract_bin_op_t(pass)) , mask(_CUB::detail::topk::calc_mask(pass)) {} - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE int operator()(T key) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE int operator()(T key) const { auto bits = reinterpret_cast::UnsignedBits&>(key); bits = _CUB::detail::topk::twiddle_in_bits(bits); @@ -212,7 +212,7 @@ struct extract_bin_op_t>( _CUB::detail::topk::calc_start_bit(total_bits, pass), @@ -221,7 +221,7 @@ struct extract_bin_op_t::In(ordered, decomposer); @@ -245,13 +245,13 @@ struct identify_candidates_op_t; unsigned_bits_t* kth_key_bits; int start_bit; - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE identify_candidates_op_t(key_prefix_t* kth_key_bits, int pass, int /*total_bits*/, DecomposerT /*decomposer*/) : kth_key_bits(&kth_key_bits->bits) , start_bit(_CUB::detail::topk::calc_start_bit(pass - 1)) {} - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE candidate_class operator()(T key) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE candidate_class operator()(T key) const { auto bits = reinterpret_cast(key); bits = _CUB::detail::topk::twiddle_in_bits(bits); // must match extract_bin_op_t, which built @@ -284,7 +284,7 @@ struct identify_candidates_op_t(*this); } diff --git a/cub/cub/detail/uninitialized_copy.cuh b/cub/cub/detail/uninitialized_copy.cuh index b07b4c47342c..8088f8176f1f 100644 --- a/cub/cub/detail/uninitialized_copy.cuh +++ b/cub/cub/detail/uninitialized_copy.cuh @@ -31,14 +31,14 @@ _CCCL_HOST_DEVICE void uninitialized_copy_single(T* ptr, U&& val) } #else // ^^^ _CCCL_CUDA_COMPILER(NVHPC) ^^^ / vvv !_CCCL_CUDA_COMPILER(NVHPC) vvv template , int> = 0> -_CCCL_HOST_DEVICE void uninitialized_copy_single(T* ptr, U&& val) +_CCCL_HOST_DEVICE_API void uninitialized_copy_single(T* ptr, U&& val) { // gevtushenko: placement new should work here as well, but the code generated for copy assignment is sometimes better *ptr = ::cuda::std::forward(val); } template , int> = 0> -_CCCL_HOST_DEVICE void uninitialized_copy_single(T* ptr, U&& val) +_CCCL_HOST_DEVICE_API void uninitialized_copy_single(T* ptr, U&& val) { new (ptr) T(::cuda::std::forward(val)); } diff --git a/cub/cub/detail/unsafe_bitcast.cuh b/cub/cub/detail/unsafe_bitcast.cuh index bb1341c78af6..208801de862e 100644 --- a/cub/cub/detail/unsafe_bitcast.cuh +++ b/cub/cub/detail/unsafe_bitcast.cuh @@ -20,7 +20,7 @@ namespace detail // NOTE: bit_cast cannot be always used because __half, __nv_bfloat16, etc. are not trivially copyable template -[[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE Output unsafe_bitcast(const Input& input) +[[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE Output unsafe_bitcast(const Input& input) { // memcpy overwrites the complete object representation. Output output; // NOLINT(cppcoreguidelines-pro-type-member-init) diff --git a/cub/cub/detail/warpspeed/sync_handler.cuh b/cub/cub/detail/warpspeed/sync_handler.cuh index 673d06c9f92a..1978ae43b0f4 100644 --- a/cub/cub/detail/warpspeed/sync_handler.cuh +++ b/cub/cub/detail/warpspeed/sync_handler.cuh @@ -45,7 +45,7 @@ struct SyncHandler int mNumOwningThreads[mMaxNumResources][mMaxNumPhases]{}; ::cuda::std::uint64_t* mPtrBar[mMaxNumResources][mMaxNumPhases]{}; - constexpr SyncHandler() = default; + _CCCL_HIDE_FROM_ABI constexpr SyncHandler() = default; // we need constant destruction for the host side single stage SMEM amount, which is only possible in C++20 #if _CCCL_STD_VER >= 2020 diff --git a/cub/cub/device/device_for.cuh b/cub/cub/device/device_for.cuh index 543666d39acd..d9864577398c 100644 --- a/cub/cub/device/device_for.cuh +++ b/cub/cub/device/device_for.cuh @@ -80,7 +80,7 @@ struct DeviceFor RandomAccessIteratorT input; OpT op; - _CCCL_DEVICE _CCCL_FORCEINLINE void operator()(OffsetT i) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void operator()(OffsetT i) { // Dereferencing `thrust::device_vector` iterators returns a `thrust::device_reference` // instead of `T`. Since user-provided operator expects `T` as an argument, we need to unwrap. @@ -108,7 +108,7 @@ struct DeviceFor // Type of the vector that is used to load the input data using vector_t = typename CubVector::Type; - _CCCL_DEVICE _CCCL_FORCEINLINE void operator()(OffsetT i) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void operator()(OffsetT i) { // Surrounding `Bulk` call doesn't invoke this operator on invalid indices, so we don't need to // check for out-of-bounds access here. diff --git a/cub/cub/device/device_histogram.cuh b/cub/cub/device/device_histogram.cuh index 21990489febd..3b3281bb0f2b 100644 --- a/cub/cub/device/device_histogram.cuh +++ b/cub/cub/device/device_histogram.cuh @@ -554,7 +554,7 @@ struct DeviceHistogram private: template - _CCCL_HOST_DEVICE static auto to_array(T* ptr) + _CCCL_HOST_DEVICE_API static auto to_array(T* ptr) { ::cuda::std::array<::cuda::std::remove_const_t, N> a{}; ::cuda::std::copy(ptr, ptr + N, a.begin()); diff --git a/cub/cub/device/device_merge_sort.cuh b/cub/cub/device/device_merge_sort.cuh index d46acfdb542f..40b4c5f8d3ec 100644 --- a/cub/cub/device/device_merge_sort.cuh +++ b/cub/cub/device/device_merge_sort.cuh @@ -123,7 +123,7 @@ struct DeviceMergeSort { private: // Name reported for NVTX ranges - _CCCL_HOST_DEVICE static constexpr auto GetName() -> const char* + _CCCL_HOST_DEVICE_API static constexpr auto GetName() -> const char* { return "cub::DeviceMergeSort"; } diff --git a/cub/cub/device/device_radix_sort.cuh b/cub/cub/device/device_radix_sort.cuh index 260db9c1c5d5..1eada66aaeda 100644 --- a/cub/cub/device/device_radix_sort.cuh +++ b/cub/cub/device/device_radix_sort.cuh @@ -310,7 +310,7 @@ private: } // Name reported for NVTX ranges - _CCCL_HOST_DEVICE static constexpr auto GetName() -> const char* + _CCCL_HOST_DEVICE_API static constexpr auto GetName() -> const char* { return "cub::DeviceRadixSort"; } diff --git a/cub/cub/device/device_segmented_radix_sort.cuh b/cub/cub/device/device_segmented_radix_sort.cuh index 62bea4f7b45f..b2ff1a74e0c4 100644 --- a/cub/cub/device/device_segmented_radix_sort.cuh +++ b/cub/cub/device/device_segmented_radix_sort.cuh @@ -88,7 +88,7 @@ struct DeviceSegmentedRadixSort { private: // Name reported for NVTX ranges - _CCCL_HOST_DEVICE static constexpr auto GetName() -> const char* + _CCCL_HOST_DEVICE_API static constexpr auto GetName() -> const char* { return "cub::DeviceSegmentedRadixSort"; } diff --git a/cub/cub/device/device_segmented_sort.cuh b/cub/cub/device/device_segmented_sort.cuh index 4423b2d20170..dbeb48b14c1d 100644 --- a/cub/cub/device/device_segmented_sort.cuh +++ b/cub/cub/device/device_segmented_sort.cuh @@ -137,7 +137,7 @@ struct DeviceSegmentedSort { private: // Name reported for NVTX ranges - _CCCL_HOST_DEVICE static constexpr auto GetName() -> const char* + _CCCL_HOST_DEVICE_API static constexpr auto GetName() -> const char* { return "cub::DeviceSegmentedSort"; } diff --git a/cub/cub/device/device_transform.cuh b/cub/cub/device/device_transform.cuh index 90509558388d..e1be6767d999 100644 --- a/cub/cub/device/device_transform.cuh +++ b/cub/cub/device/device_transform.cuh @@ -61,7 +61,7 @@ struct __return_constant T value; template - _CCCL_HOST_DEVICE auto operator()(Args&&...) const -> T + _CCCL_HOST_DEVICE_API auto operator()(Args&&...) const -> T { return value; } diff --git a/cub/cub/device/dispatch/dispatch_batched_topk.cuh b/cub/cub/device/dispatch/dispatch_batched_topk.cuh index 5de98eb760bb..bdf439fa29cf 100644 --- a/cub/cub/device/dispatch/dispatch_batched_topk.cuh +++ b/cub/cub/device/dispatch/dispatch_batched_topk.cuh @@ -75,7 +75,7 @@ namespace detail::batched_topk // value-less static_discrete_param. Because the direction is fixed at compile time and carries no runtime value, it // can never disagree with its only supported option, so dispatch can never silently degrade to a no-op. template -[[nodiscard]] _CCCL_HOST_DEVICE auto wrap_select_direction(::cuda::args::constant) +[[nodiscard]] _CCCL_HOST_DEVICE_API auto wrap_select_direction(::cuda::args::constant) { return params::static_discrete_param{}; } @@ -87,7 +87,7 @@ template // reach the algorithm through the min/max device entry points (DeviceBatchedTopK::{Max,Min}{Keys,Pairs}), which // construct the matching `constant` internally, so `dispatch` is only ever invoked with a direction we create. template -[[nodiscard]] _CCCL_HOST_DEVICE auto wrap_select_direction(SelectDirectionT) +[[nodiscard]] _CCCL_HOST_DEVICE_API auto wrap_select_direction(SelectDirectionT) { static_assert(::cuda::std::__always_false_v, "DeviceBatchedTopK currently supports only compile-time selection directions: the min/max entry " @@ -112,7 +112,7 @@ struct segment_size_to_tile_count_op int large_segment_agent_tile_size; template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE constexpr TotalNumItemsValueType operator()(SegmentIndexT segment_id) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE constexpr TotalNumItemsValueType operator()(SegmentIndexT segment_id) const { return static_cast( ::cuda::ceil_div(_CUB::detail::params::__get_and_clamp_param_to_nonnegative(segment_sizes, segment_id), diff --git a/cub/cub/device/dispatch/dispatch_copy_mdspan.cuh b/cub/cub/device/dispatch/dispatch_copy_mdspan.cuh index a91153bbaa77..d3e0c18465fd 100644 --- a/cub/cub/device/dispatch/dispatch_copy_mdspan.cuh +++ b/cub/cub/device/dispatch/dispatch_copy_mdspan.cuh @@ -115,7 +115,7 @@ copy(::cuda::std::mdspan mdspan_in, ::cuda::std::mdspan output; const EnvT& environment; - _CCCL_HOST ::cudaError_t operator()() const + _CCCL_HOST_API ::cudaError_t operator()() const { _CCCL_TRY { diff --git a/cub/cub/device/dispatch/dispatch_reduce_by_key.cuh b/cub/cub/device/dispatch/dispatch_reduce_by_key.cuh index 21826cb42346..f865894ad5c7 100644 --- a/cub/cub/device/dispatch/dispatch_reduce_by_key.cuh +++ b/cub/cub/device/dispatch/dispatch_reduce_by_key.cuh @@ -68,47 +68,47 @@ struct streaming_context GlobalOffsetT* d_num_previous_uniques_in; GlobalOffsetT* d_num_accumulated_uniques_out; - _CCCL_DEVICE _CCCL_FORCEINLINE GlobalOffsetT num_accumulated_uniques_out() const + _CCCL_DEVICE_API _CCCL_FORCEINLINE GlobalOffsetT num_accumulated_uniques_out() const { return first_partition ? GlobalOffsetT{0} : *d_num_previous_uniques_in; }; - _CCCL_FORCEINLINE _CCCL_HOST_DEVICE bool is_first_partition() const + _CCCL_FORCEINLINE _CCCL_HOST_DEVICE_API bool is_first_partition() const { return first_partition; } - _CCCL_FORCEINLINE _CCCL_HOST_DEVICE bool is_last_partition() const + _CCCL_FORCEINLINE _CCCL_HOST_DEVICE_API bool is_last_partition() const { return last_partition; } - _CCCL_FORCEINLINE _CCCL_HOST_DEVICE auto predecessor_key() const + _CCCL_FORCEINLINE _CCCL_HOST_DEVICE_API auto predecessor_key() const { return *preceding_key_it; } - _CCCL_FORCEINLINE _CCCL_HOST_DEVICE AccumT prefix() const + _CCCL_FORCEINLINE _CCCL_HOST_DEVICE_API AccumT prefix() const { return *preceding_prefix; } - _CCCL_FORCEINLINE _CCCL_HOST_DEVICE void write_prefix(AccumT prefix) const + _CCCL_FORCEINLINE _CCCL_HOST_DEVICE_API void write_prefix(AccumT prefix) const { *prefix_out = prefix; } - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE GlobalOffsetT* previous_uniques_ptr() const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE GlobalOffsetT* previous_uniques_ptr() const { return d_num_previous_uniques_in; } - _CCCL_FORCEINLINE _CCCL_HOST_DEVICE GlobalOffsetT num_uniques() const + _CCCL_FORCEINLINE _CCCL_HOST_DEVICE_API GlobalOffsetT num_uniques() const { return num_accumulated_uniques_out(); } template - _CCCL_FORCEINLINE _CCCL_HOST_DEVICE GlobalOffsetT add_num_uniques(NumUniquesT num_uniques) const + _CCCL_FORCEINLINE _CCCL_HOST_DEVICE_API GlobalOffsetT add_num_uniques(NumUniquesT num_uniques) const { GlobalOffsetT total_uniques = num_accumulated_uniques_out() + static_cast(num_uniques); diff --git a/cub/cub/device/dispatch/dispatch_reduce_deterministic.cuh b/cub/cub/device/dispatch/dispatch_reduce_deterministic.cuh index a264c86ec456..43e4b9f977da 100644 --- a/cub/cub/device/dispatch/dispatch_reduce_deterministic.cuh +++ b/cub/cub/device/dispatch/dispatch_reduce_deterministic.cuh @@ -66,25 +66,25 @@ struct deterministic_sum_t { using DeterministicAcc = ReproducibleFloatingAccumulator; - _CCCL_DEVICE DeterministicAcc operator()(DeterministicAcc acc, FloatType f) + _CCCL_DEVICE_API DeterministicAcc operator()(DeterministicAcc acc, FloatType f) { acc += f; return acc; } - _CCCL_DEVICE DeterministicAcc operator()(FloatType f, DeterministicAcc acc) + _CCCL_DEVICE_API DeterministicAcc operator()(FloatType f, DeterministicAcc acc) { return this->operator()(acc, f); } - _CCCL_DEVICE DeterministicAcc operator()(DeterministicAcc lhs, DeterministicAcc rhs) + _CCCL_DEVICE_API DeterministicAcc operator()(DeterministicAcc lhs, DeterministicAcc rhs) { DeterministicAcc rtn = lhs; rtn += rhs; return rtn; } - _CCCL_DEVICE FloatType operator()(FloatType lhs, FloatType rhs) + _CCCL_DEVICE_API FloatType operator()(FloatType lhs, FloatType rhs) { return lhs + rhs; } diff --git a/cub/cub/device/dispatch/dispatch_rle.cuh b/cub/cub/device/dispatch/dispatch_rle.cuh index 5062ca20d4ac..867de737c30c 100644 --- a/cub/cub/device/dispatch/dispatch_rle.cuh +++ b/cub/cub/device/dispatch/dispatch_rle.cuh @@ -70,38 +70,38 @@ struct streaming_context GlobalOffsetT* d_num_previous_uniques_in; GlobalOffsetT* d_num_accumulated_uniques_out; - _CCCL_DEVICE _CCCL_FORCEINLINE GlobalOffsetT num_accumulated_uniques_out() const + _CCCL_DEVICE_API _CCCL_FORCEINLINE GlobalOffsetT num_accumulated_uniques_out() const { return first_partition ? GlobalOffsetT{0} : *d_num_previous_uniques_in; }; - _CCCL_FORCEINLINE _CCCL_HOST_DEVICE RunLengthT prefix() const + _CCCL_FORCEINLINE _CCCL_HOST_DEVICE_API RunLengthT prefix() const { return *preceding_length; } - _CCCL_FORCEINLINE _CCCL_HOST_DEVICE void write_prefix(RunLengthT prefix) const + _CCCL_FORCEINLINE _CCCL_HOST_DEVICE_API void write_prefix(RunLengthT prefix) const { *length_out = prefix; } - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE GlobalOffsetT* previous_uniques_ptr() const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE GlobalOffsetT* previous_uniques_ptr() const { return d_num_previous_uniques_in; } - _CCCL_FORCEINLINE _CCCL_HOST_DEVICE GlobalOffsetT num_uniques() const + _CCCL_FORCEINLINE _CCCL_HOST_DEVICE_API GlobalOffsetT num_uniques() const { return num_accumulated_uniques_out(); } - _CCCL_FORCEINLINE _CCCL_HOST_DEVICE GlobalOffsetT base_offset() const + _CCCL_FORCEINLINE _CCCL_HOST_DEVICE_API GlobalOffsetT base_offset() const { return current_base_offset; } template - _CCCL_FORCEINLINE _CCCL_HOST_DEVICE GlobalOffsetT add_num_uniques(NumUniquesT num_uniques) const + _CCCL_FORCEINLINE _CCCL_HOST_DEVICE_API GlobalOffsetT add_num_uniques(NumUniquesT num_uniques) const { GlobalOffsetT total_uniques = num_accumulated_uniques_out() + static_cast(num_uniques); diff --git a/cub/cub/device/dispatch/dispatch_scan.cuh b/cub/cub/device/dispatch/dispatch_scan.cuh index 190498d584a6..88f4e0a338c2 100644 --- a/cub/cub/device/dispatch/dispatch_scan.cuh +++ b/cub/cub/device/dispatch/dispatch_scan.cuh @@ -848,7 +848,7 @@ struct CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceScan") DispatchScan struct policy_getter { // host-device not api, because clang warns about exclude_from_explicit_instantiation in local types - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE constexpr auto operator()() const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE constexpr auto operator()() const { return _CUB::detail::scan::convert_policy(); } diff --git a/cub/cub/device/dispatch/dispatch_segmented_reduce.cuh b/cub/cub/device/dispatch/dispatch_segmented_reduce.cuh index 94ca00deb55b..cd24dfca5b64 100644 --- a/cub/cub/device/dispatch/dispatch_segmented_reduce.cuh +++ b/cub/cub/device/dispatch/dispatch_segmented_reduce.cuh @@ -669,12 +669,12 @@ private: int segment_size; public: - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE generate_idx_value(Iterator it, int segment_size) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE generate_idx_value(Iterator it, int segment_size) : it(it) , segment_size(segment_size) {} - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE auto operator()(::cuda::std::int64_t idx) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE auto operator()(::cuda::std::int64_t idx) const { return ::cuda::std::pair(static_cast(idx % segment_size), it[idx]); } diff --git a/cub/cub/device/dispatch/dispatch_select_if.cuh b/cub/cub/device/dispatch/dispatch_select_if.cuh index 7440f493e7e0..e37c8a831874 100644 --- a/cub/cub/device/dispatch/dispatch_select_if.cuh +++ b/cub/cub/device/dispatch/dispatch_select_if.cuh @@ -75,7 +75,7 @@ class streaming_context_t public: using total_num_items_t = TotalNumItemsT; - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE streaming_context_t( + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE streaming_context_t( TotalNumItemsT* d_num_selected_in, TotalNumItemsT* d_num_selected_out, TotalNumItemsT total_num_items, @@ -86,7 +86,7 @@ public: , d_num_selected_out(d_num_selected_out) {} - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void advance(TotalNumItemsT num_items, bool next_partition_is_the_last) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE void advance(TotalNumItemsT num_items, bool next_partition_is_the_last) { using ::cuda::std::swap; swap(d_num_selected_in, d_num_selected_out); @@ -95,34 +95,34 @@ public: total_previous_num_items += num_items; }; - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE TotalNumItemsT input_offset() const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE TotalNumItemsT input_offset() const { return first_partition ? TotalNumItemsT{0} : total_previous_num_items; }; - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE TotalNumItemsT is_first_partition() const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE TotalNumItemsT is_first_partition() const { return first_partition; }; - _CCCL_DEVICE _CCCL_FORCEINLINE TotalNumItemsT num_previously_selected() const + _CCCL_DEVICE_API _CCCL_FORCEINLINE TotalNumItemsT num_previously_selected() const { return first_partition ? TotalNumItemsT{0} : *d_num_selected_in; }; - _CCCL_DEVICE _CCCL_FORCEINLINE TotalNumItemsT num_previously_rejected() const + _CCCL_DEVICE_API _CCCL_FORCEINLINE TotalNumItemsT num_previously_rejected() const { return first_partition ? TotalNumItemsT{0} : (total_previous_num_items - num_previously_selected()); }; template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE TotalNumItemsT num_total_items(OffsetT) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE TotalNumItemsT num_total_items(OffsetT) const { return total_num_items; } template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE void update_num_selected(NumSelectedIteratorT user_num_selected_out_it, OffsetT num_selections) const { if (last_partition) @@ -142,38 +142,38 @@ class streaming_context_t public: using total_num_items_t = TotalNumItemsT; - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE streaming_context_t(TotalNumItemsT*, TotalNumItemsT*, TotalNumItemsT, bool) {} + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE streaming_context_t(TotalNumItemsT*, TotalNumItemsT*, TotalNumItemsT, bool) {} - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void advance(TotalNumItemsT, bool) {}; + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE void advance(TotalNumItemsT, bool) {}; - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE TotalNumItemsT input_offset() const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE TotalNumItemsT input_offset() const { return TotalNumItemsT{0}; }; - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE TotalNumItemsT is_first_partition() const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE TotalNumItemsT is_first_partition() const { return true; }; - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE TotalNumItemsT num_previously_selected() const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE TotalNumItemsT num_previously_selected() const { return TotalNumItemsT{0}; }; - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE TotalNumItemsT num_previously_rejected() const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE TotalNumItemsT num_previously_rejected() const { return TotalNumItemsT{0}; }; template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE TotalNumItemsT num_total_items(OffsetT num_partition_items) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE TotalNumItemsT num_total_items(OffsetT num_partition_items) const { return num_partition_items; } template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE void update_num_selected(NumSelectedIteratorT user_num_selected_out_it, OffsetT num_selections) const { *user_num_selected_out_it = num_selections; diff --git a/cub/cub/device/dispatch/dispatch_streaming_reduce.cuh b/cub/cub/device/dispatch/dispatch_streaming_reduce.cuh index 0ddf3c35a49d..c78c1404acd8 100644 --- a/cub/cub/device/dispatch/dispatch_streaming_reduce.cuh +++ b/cub/cub/device/dispatch/dispatch_streaming_reduce.cuh @@ -59,7 +59,7 @@ struct accumulating_transform_output_op GlobalReductionOpT reduce_op; template - _CCCL_DEVICE _CCCL_FORCEINLINE void operator()(IndexT, AccumT per_partition_aggregate) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void operator()(IndexT, AccumT per_partition_aggregate) { // Add this partitions aggregate to the global aggregate if (first_partition) @@ -82,7 +82,7 @@ struct accumulating_transform_output_op * This is a helper function that's invoked after a partition has been fully processed */ template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void advance(GlobalOffsetT partition_size, bool next_partition_is_the_last) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE void advance(GlobalOffsetT partition_size, bool next_partition_is_the_last) { promote_op.advance(partition_size); using ::cuda::std::swap; @@ -104,7 +104,7 @@ struct local_to_global_op /** * This helper function is invoked after a partition has been fully processed, in preparation for the next partition. */ - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void advance(GlobalOffsetT partition_size) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE void advance(GlobalOffsetT partition_size) { current_partition_offset += partition_size; } @@ -114,7 +114,7 @@ struct local_to_global_op * that is used to reduce across partitions). */ template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE KeyValuePair + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE KeyValuePair operator()(KeyValuePair partition_aggregate) { return KeyValuePair{ @@ -129,7 +129,7 @@ struct unzip_and_write_arg_extremum_op IndexOutIteratorT index_out_it; template - _CCCL_DEVICE _CCCL_FORCEINLINE void operator()(IndexT, KeyValuePairT reduced_result) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void operator()(IndexT, KeyValuePairT reduced_result) { *result_out_it = reduced_result.value; *index_out_it = reduced_result.key; @@ -140,7 +140,7 @@ struct unzip_and_write_arg_extremum_op struct kvp_to_argminmax_accum { template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE auto operator()(KeyValuePair kv) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE auto operator()(KeyValuePair kv) const -> argminmax_accum_t { return {kv.value, kv.value, kv.key, kv.key}; @@ -153,13 +153,13 @@ struct local_to_global_minmax_op { GlobalOffsetT current_partition_offset; - _CCCL_HOST_DEVICE void advance(GlobalOffsetT partition_size) + _CCCL_HOST_DEVICE_API void advance(GlobalOffsetT partition_size) { current_partition_offset += partition_size; } template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE detail::argminmax_accum_t + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE detail::argminmax_accum_t operator()(const detail::argminmax_accum_t& p) const { return {p.min_value, @@ -178,7 +178,7 @@ struct write_arg_minmax_result_op MaxIndexOutT max_index_out; template - _CCCL_DEVICE _CCCL_FORCEINLINE void operator()(IndexT, const argminmax_accum_t& result) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void operator()(IndexT, const argminmax_accum_t& result) { *min_out = result.min_value; *min_index_out = result.min_index; diff --git a/cub/cub/device/dispatch/kernels/kernel_batched_topk.cuh b/cub/cub/device/dispatch/kernels/kernel_batched_topk.cuh index 082f48454b57..bb30e88a711e 100644 --- a/cub/cub/device/dispatch/kernels/kernel_batched_topk.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_batched_topk.cuh @@ -53,7 +53,7 @@ struct find_sort_policy_index { private: template - [[nodiscard]] static constexpr int find_index() + [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr int find_index() { if constexpr (Index >= sorted_output_policy_count) { @@ -149,7 +149,7 @@ private: static constexpr topk_policy active_policy = PolicyGetter{}(); template - [[nodiscard]] static constexpr int find_index() + [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr int find_index() { if constexpr (Index >= active_policy.baseline.worker_per_segment_policies.size()) { diff --git a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh index 9c9aa4eec6f3..6be1e7b128a1 100644 --- a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh @@ -60,7 +60,7 @@ struct Transforms //! //! @param d_levels_ Pointer to levels array //! @param num_output_levels_ Number of levels in array - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void Init(LevelIteratorT d_levels_, int num_output_levels_) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE void Init(LevelIteratorT d_levels_, int num_output_levels_) { this->d_levels = d_levels_; this->num_output_levels = num_output_levels_; @@ -68,7 +68,7 @@ struct Transforms // Method for converting samples to bin-ids template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(Sample sample, int& bin, bool valid) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE void BinSelect(Sample sample, int& bin, bool valid) const { /// Level iterator wrapper type // Wrap the native input pointer with CacheModifiedInputIterator @@ -161,7 +161,7 @@ struct Transforms using FractionStorageT = decltype(FractionStorageType()); template - [[nodiscard]] _CCCL_HOST_DEVICE _CCCL_FORCEINLINE static auto subtract_as_unsigned(T lhs, T rhs) noexcept + [[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE static auto subtract_as_unsigned(T lhs, T rhs) noexcept { if constexpr (::cuda::std::is_same_v) { @@ -192,7 +192,7 @@ struct Transforms ScaleT m_scale; // Bin scaling template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE ScaleT + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE ScaleT ComputeScale(int num_levels, T max_level, T min_level, ::cuda::std::true_type /* is_fp */) { // The active scale representation is assigned below. @@ -203,7 +203,7 @@ struct Transforms } template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE ScaleT + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE ScaleT ComputeScale(int num_levels, T max_level, T min_level, ::cuda::std::false_type /* is_fp */) { // The active scale representation is assigned below. @@ -222,13 +222,13 @@ struct Transforms } template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE ScaleT ComputeScale(int num_levels, T max_level, T min_level) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE ScaleT ComputeScale(int num_levels, T max_level, T min_level) { return this->ComputeScale(num_levels, max_level, min_level, ::cuda::std::is_floating_point{}); } #if _CCCL_HAS_NVFP16() - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE ScaleT ComputeScale(int num_levels, __half max_level, __half min_level) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE ScaleT ComputeScale(int num_levels, __half max_level, __half min_level) { // The active scale representation is assigned below. // NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init) @@ -242,7 +242,7 @@ struct Transforms #endif // _CCCL_HAS_NVFP16() #if _CCCL_HAS_NVBF16() - _CCCL_HOST_DEVICE + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE ScaleT ComputeScale(int num_levels, __nv_bfloat16 max_level, __nv_bfloat16 min_level) { // The active scale representation is assigned below. @@ -328,7 +328,7 @@ struct Transforms public: //! @brief Initializes the ScaleTransform for the given parameters - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void Init(int num_levels, LevelT max_level, LevelT min_level) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE void Init(int num_levels, LevelT max_level, LevelT min_level) { m_max = static_cast(max_level); m_min = static_cast(min_level); @@ -339,7 +339,7 @@ struct Transforms // Method for converting samples to bin-ids. The sample type is a template parameter because the // agent also feeds privatized bin indices through this op, which must not round-trip through SampleT. template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(Sample sample, int& bin, bool valid) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE void BinSelect(Sample sample, int& bin, bool valid) const { const CommonT common_sample = static_cast(sample); @@ -361,17 +361,17 @@ struct Transforms // No-op Init for uniformity with ScaleTransform template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void Init(int, T, T) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE void Init(int, T, T) {} // No-op Init for uniformity with SearchTransform template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void Init(T, int) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE void Init(T, int) {} // Method for converting samples to bin-ids template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(Sample sample, int& bin, bool valid) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE void BinSelect(Sample sample, int& bin, bool valid) const { if (valid) { diff --git a/cub/cub/device/dispatch/kernels/kernel_reduce.cuh b/cub/cub/device/dispatch/kernels/kernel_reduce.cuh index 0473575bb156..80da3ee5a363 100644 --- a/cub/cub/device/dispatch/kernels/kernel_reduce.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_reduce.cuh @@ -42,7 +42,7 @@ struct empty_problem_init_t { T init; - _CCCL_HOST_DEVICE operator T() const + _CCCL_HOST_DEVICE_API operator T() const { return init; } @@ -56,20 +56,20 @@ struct no_init_t inline constexpr auto no_init = no_init_t{}; template -_CCCL_HOST_DEVICE _CCCL_FORCEINLINE void handle_empty_problem(OutputIteratorT&& d_out, InitValueT init) +_CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE void handle_empty_problem(OutputIteratorT&& d_out, InitValueT init) { *d_out = init; } template -_CCCL_HOST_DEVICE _CCCL_FORCEINLINE void +_CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE void handle_empty_problem(OutputIteratorT&& d_out, empty_problem_init_t empty_problem_init) { *d_out = empty_problem_init.init; } template -_CCCL_HOST_DEVICE _CCCL_FORCEINLINE void handle_empty_problem(OutputIteratorT&&, no_init_t) +_CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE void handle_empty_problem(OutputIteratorT&&, no_init_t) {} /** @@ -81,7 +81,7 @@ _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void handle_empty_problem(OutputIteratorT&&, * @param block_aggregate Aggregate value computed by the block */ template -_CCCL_HOST_DEVICE void +_CCCL_HOST_DEVICE_API void finalize_and_store_aggregate(OutputIteratorT d_out, ReductionOpT reduction_op, InitValueT init, AccumT block_aggregate) { *d_out = reduction_op(init, block_aggregate); @@ -94,14 +94,14 @@ finalize_and_store_aggregate(OutputIteratorT d_out, ReductionOpT reduction_op, I * @param block_aggregate Aggregate value computed by the block */ template -_CCCL_HOST_DEVICE void finalize_and_store_aggregate( +_CCCL_HOST_DEVICE_API void finalize_and_store_aggregate( OutputIteratorT d_out, ReductionOpT, empty_problem_init_t, AccumT block_aggregate) { *d_out = block_aggregate; } template -_CCCL_HOST_DEVICE void +_CCCL_HOST_DEVICE_API void finalize_and_store_aggregate(OutputIteratorT d_out, ReductionOpT, no_init_t, AccumT block_aggregate) { *d_out = block_aggregate; diff --git a/cub/cub/device/dispatch/kernels/kernel_scan.cuh b/cub/cub/device/dispatch/kernels/kernel_scan.cuh index 486fc6aff097..854d6d09043e 100644 --- a/cub/cub/device/dispatch/kernels/kernel_scan.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_scan.cuh @@ -123,7 +123,7 @@ union tile_state_kernel_arg_t // ScanTileState is not trivially [default|copy]-constructible, so because of // https://eel.is/c++draft/class.union#general-note-3, tile_state_kernel_arg_t's special members are deleted. We work // around it by explicitly defining the ones we need. - _CCCL_HOST_DEVICE tile_state_kernel_arg_t() noexcept {} + _CCCL_HOST_DEVICE_API tile_state_kernel_arg_t() noexcept {} }; /****************************************************************************** diff --git a/cub/cub/device/dispatch/kernels/kernel_segmented_reduce.cuh b/cub/cub/device/dispatch/kernels/kernel_segmented_reduce.cuh index 344c16146ee0..84b622456f8a 100644 --- a/cub/cub/device/dispatch/kernels/kernel_segmented_reduce.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_segmented_reduce.cuh @@ -32,12 +32,12 @@ namespace detail::segmented_reduce { /// Normalize input iterator to segment offset template -_CCCL_DEVICE _CCCL_FORCEINLINE void NormalizeReductionOutput(T& /*val*/, OffsetT /*base_offset*/, IteratorT /*itr*/) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void NormalizeReductionOutput(T& /*val*/, OffsetT /*base_offset*/, IteratorT /*itr*/) {} /// Normalize input iterator to segment offset (specialized for arg-index) template -_CCCL_DEVICE _CCCL_FORCEINLINE void NormalizeReductionOutput( +_CCCL_DEVICE_API _CCCL_FORCEINLINE void NormalizeReductionOutput( KeyValuePairT& val, OffsetT base_offset, ArgIndexInputIterator /*itr*/) { val.key -= base_offset; diff --git a/cub/cub/device/dispatch/kernels/kernel_segmented_scan.cuh b/cub/cub/device/dispatch/kernels/kernel_segmented_scan.cuh index c71b98888915..27340e7c7451 100644 --- a/cub/cub/device/dispatch/kernels/kernel_segmented_scan.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_segmented_scan.cuh @@ -187,7 +187,7 @@ public: // Constructor //--------------------------------------------------------------------- - _CCCL_DEVICE _CCCL_FORCEINLINE agent_segmented_scan( + _CCCL_DEVICE_API _CCCL_FORCEINLINE agent_segmented_scan( TempStorage& temp_storage, InputIteratorT d_in, OutputIteratorT d_out, ScanOpT scan_op, InitValueT initial_value) : temp_storage(temp_storage.Alias()) , d_in(d_in) @@ -207,7 +207,7 @@ public: //! @param output_begin_idx //! Index of start of the segment's prefix scan result in the output array //! - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void scan_one_segment(OffsetT input_begin_idx, OffsetT input_end_idx, OffsetT output_begin_idx) { const OffsetT segment_items = ::cuda::std::max(input_end_idx, input_begin_idx) - input_begin_idx; @@ -279,7 +279,7 @@ public: typename OutputBeginOffsetIteratorT, ::cuda::std::size_t NumSegments = max_segments, ::cuda::std::enable_if_t<(NumSegments > 1), int> = 0> - _CCCL_DEVICE _CCCL_FORCEINLINE void scan_segments( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void scan_segments( InputBeginOffsetIteratorT input_begin_idx_it, InputEndOffsetIteratorT input_end_idx_it, OutputBeginOffsetIteratorT output_begin_idx_it, @@ -416,7 +416,7 @@ public: private: template - _CCCL_DEVICE _CCCL_FORCEINLINE void scan_segments_chunked( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void scan_segments_chunked( const SearcherT& searcher, InputBeginOffsetIteratorT input_begin_idx_it, OutputBeginOffsetIteratorT output_begin_idx_it, @@ -548,7 +548,7 @@ private: typename ScanOpTy, bool IsInclusive = is_inclusive, bool HasInit = has_init> - _CCCL_DEVICE _CCCL_FORCEINLINE void scan_first_tile( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void scan_first_tile( ScannerT& scanner, ItemTy (&items)[items_per_thread], InitValueTy init_value, @@ -576,7 +576,7 @@ private: } template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void scan_later_tile(ScannerT& scanner, ItemTy (&items)[items_per_thread], ScanOpTy scan_op, PrefixCallback& prefix_op) { if constexpr (IsInclusive) @@ -623,7 +623,7 @@ __launch_bounds__(_CUB::detail::current_policy().block.threads_p struct policy_getter { - constexpr auto operator()() const + _CCCL_HOST_DEVICE_API constexpr auto operator()() const { return policy; } diff --git a/cub/cub/device/dispatch/kernels/kernel_segmented_sort.cuh b/cub/cub/device/dispatch/kernels/kernel_segmented_sort.cuh index f58eed6629e2..ae1d97e4e627 100644 --- a/cub/cub/device/dispatch/kernels/kernel_segmented_sort.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_segmented_sort.cuh @@ -44,7 +44,7 @@ struct LargeSegmentsSelectorT global_segment_offset_t base_segment_offset{}; #if !_CCCL_COMPILER(NVRTC) - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE LargeSegmentsSelectorT(OffsetT value, BeginOffsetIteratorT d_offset_begin, EndOffsetIteratorT d_offset_end) : value(value) , d_offset_begin(d_offset_begin) @@ -52,7 +52,7 @@ struct LargeSegmentsSelectorT {} #endif // !_CCCL_COMPILER(NVRTC) - _CCCL_DEVICE _CCCL_FORCEINLINE bool operator()(local_segment_index_t segment_id) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE bool operator()(local_segment_index_t segment_id) const { // NOLINTNEXTLINE(bugprone-misplaced-widening-cast) const OffsetT segment_size = @@ -70,7 +70,7 @@ struct SmallSegmentsSelectorT global_segment_offset_t base_segment_offset{}; #if !_CCCL_COMPILER(NVRTC) - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE SmallSegmentsSelectorT(OffsetT value, BeginOffsetIteratorT d_offset_begin, EndOffsetIteratorT d_offset_end) : value(value) , d_offset_begin(d_offset_begin) @@ -78,7 +78,7 @@ struct SmallSegmentsSelectorT {} #endif // !_CCCL_COMPILER(NVRTC) - _CCCL_DEVICE _CCCL_FORCEINLINE bool operator()(local_segment_index_t segment_id) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE bool operator()(local_segment_index_t segment_id) const { // NOLINTNEXTLINE(bugprone-misplaced-widening-cast) const OffsetT segment_size = diff --git a/cub/cub/device/dispatch/kernels/kernel_three_way_partition.cuh b/cub/cub/device/dispatch/kernels/kernel_three_way_partition.cuh index 27223b735fba..55811831edda 100644 --- a/cub/cub/device/dispatch/kernels/kernel_three_way_partition.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_three_way_partition.cuh @@ -49,14 +49,14 @@ class streaming_context_t TotalNumItemsT* d_num_selected_out = nullptr; public: - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE streaming_context_t(TotalNumItemsT* d_num_selected_in, TotalNumItemsT* d_num_selected_out, bool is_last_partition) : last_partition(is_last_partition) , d_num_selected_in(d_num_selected_in) , d_num_selected_out(d_num_selected_out) {} - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void advance(TotalNumItemsT num_items, bool next_partition_is_the_last) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE void advance(TotalNumItemsT num_items, bool next_partition_is_the_last) { ::cuda::std::swap(d_num_selected_in, d_num_selected_out); first_partition = false; @@ -64,29 +64,29 @@ public: total_previous_num_items += num_items; }; - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE TotalNumItemsT input_offset() const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE TotalNumItemsT input_offset() const { return first_partition ? TotalNumItemsT{0} : total_previous_num_items; }; - _CCCL_DEVICE _CCCL_FORCEINLINE TotalNumItemsT num_previously_selected_first() const + _CCCL_DEVICE_API _CCCL_FORCEINLINE TotalNumItemsT num_previously_selected_first() const { return first_partition ? TotalNumItemsT{0} : d_num_selected_in[0]; }; - _CCCL_DEVICE _CCCL_FORCEINLINE TotalNumItemsT num_previously_selected_second() const + _CCCL_DEVICE_API _CCCL_FORCEINLINE TotalNumItemsT num_previously_selected_second() const { return first_partition ? TotalNumItemsT{0} : d_num_selected_in[1]; }; - _CCCL_DEVICE _CCCL_FORCEINLINE TotalNumItemsT num_previously_rejected() const + _CCCL_DEVICE_API _CCCL_FORCEINLINE TotalNumItemsT num_previously_rejected() const { return first_partition ? TotalNumItemsT{0} : d_num_selected_in[2]; ; }; template - _CCCL_DEVICE _CCCL_FORCEINLINE void update_num_selected( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void update_num_selected( NumSelectedIteratorT user_num_selected_out_it, TotalNumItemsT num_selected_first, TotalNumItemsT num_selected_second, diff --git a/cub/cub/device/dispatch/kernels/kernel_transform.cuh b/cub/cub/device/dispatch/kernels/kernel_transform.cuh index a14ef441eda9..92c3b46bd559 100644 --- a/cub/cub/device/dispatch/kernels/kernel_transform.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_transform.cuh @@ -75,14 +75,14 @@ _CCCL_IKET_CREATE_PUSH_POP_RANGE(ProcessAndStore); // TODO(bgruber): there is also the cp.async.bulk.prefetch instruction available on Hopper. May improve perf a tiny bit // as we need to create less instructions to prefetch the same amount of data. template -_CCCL_DEVICE _CCCL_FORCEINLINE void prefetch(const T* addr) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void prefetch(const T* addr) { // TODO(bgruber): prefetch to L1 may be even better asm volatile("prefetch.global.L2 [%0];" : : "l"(__cvta_generic_to_global(addr)) : "memory"); } template -_CCCL_DEVICE _CCCL_FORCEINLINE void prefetch_tile(It begin, int items) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void prefetch_tile(It begin, int items) { if constexpr (THRUST_NS_QUALIFIER::is_contiguous_iterator_v) { @@ -102,7 +102,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void prefetch_tile(It begin, int items) // _CCCL_PRAGMA_UNROLL(-1), but issues a warning though. Trying to suppress this warning inside the _CCCL_PRAGMA_UNROLL // macro kills MSVC. Also clang in CUDA mode does not support a non-positive unroll factor. template -_CCCL_DEVICE _CCCL_FORCEINLINE void unrolled_for(int count, F&& body) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void unrolled_for(int count, F&& body) { static_assert(UnrollFactor >= 0, "unroll_factor must be zero (ignore unroll hint) or positive (unroll with the given factor)"); @@ -134,7 +134,7 @@ template -_CCCL_DEVICE void transform_kernel_prefetch( +_CCCL_DEVICE_API void transform_kernel_prefetch( Offset num_items, int num_elem_per_thread, Predicate pred, @@ -189,7 +189,7 @@ struct alignas(32) aligned32_t #endif // _CCCL_CTK_BELOW(13, 0) template -_CCCL_HOST_DEVICE _CCCL_CONSTEVAL auto load_store_type() +_CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto load_store_type() { static_assert(::cuda::is_power_of_two(Bytes)); if constexpr (Bytes == 1) @@ -238,7 +238,7 @@ template -_CCCL_DEVICE void transform_kernel_vectorized( +_CCCL_DEVICE_API void transform_kernel_vectorized( Offset num_items, int num_elem_per_thread_prefetch, bool can_vectorize, @@ -409,19 +409,19 @@ struct aligned_base_ptr // alignof(T). E.g.: array of int3 starting at address 4, ptr == 0 int head_padding; // byte offset between ptr and the original pointer. Value inside [0;15] or [0;127]. - _CCCL_HOST_DEVICE const T* ptr_to_elements() const + _CCCL_HOST_DEVICE_API const T* ptr_to_elements() const { return reinterpret_cast(ptr + head_padding); } - _CCCL_HOST_DEVICE friend bool operator==(const aligned_base_ptr& a, const aligned_base_ptr& b) + _CCCL_HOST_DEVICE_API friend bool operator==(const aligned_base_ptr& a, const aligned_base_ptr& b) { return a.ptr == b.ptr && a.head_padding == b.head_padding; } }; template -_CCCL_HOST_DEVICE auto make_aligned_base_ptr(const T* ptr, int alignment) -> aligned_base_ptr +_CCCL_HOST_DEVICE_API auto make_aligned_base_ptr(const T* ptr, int alignment) -> aligned_base_ptr { const auto raw_ptr = reinterpret_cast(ptr); const auto base_ptr = ::cuda::align_down(raw_ptr, alignment); @@ -429,7 +429,7 @@ _CCCL_HOST_DEVICE auto make_aligned_base_ptr(const T* ptr, int alignment) -> ali } template -_CCCL_DEVICE void memcpy_async_aligned(void* dst, const void* src, unsigned int bytes_to_copy) +_CCCL_DEVICE_API void memcpy_async_aligned(void* dst, const void* src, unsigned int bytes_to_copy) { _CCCL_ASSERT(::cuda::std::is_sufficiently_aligned(src), ""); _CCCL_ASSERT(::cuda::std::is_sufficiently_aligned(dst), ""); @@ -456,7 +456,8 @@ _CCCL_DEVICE void memcpy_async_aligned(void* dst, const void* src, unsigned int } template -_CCCL_DEVICE void memcpy_async_maybe_unaligned(void* dst, const void* src, unsigned int bytes_to_copy, int head_padding) +_CCCL_DEVICE_API void +memcpy_async_maybe_unaligned(void* dst, const void* src, unsigned int bytes_to_copy, int head_padding) { // early exiting if (head_padding == 0 && bytes_to_copy % ldgsts_size_and_align == 0) does not yield a benefit @@ -513,7 +514,7 @@ _CCCL_DEVICE void memcpy_async_maybe_unaligned(void* dst, const void* src, unsig // Turning this function into a lambda will make nvcc generate it once for each iterator instead of for each distinct // value type (which may be less). template -_CCCL_DEVICE auto +_CCCL_DEVICE_API auto copy_and_return_smem_dst(AlignedPtr aligned_ptr, int& smem_offset, Offset offset, char* smem, int valid_items) { using T = typename decltype(aligned_ptr)::value_type; @@ -548,7 +549,7 @@ copy_and_return_smem_dst(AlignedPtr aligned_ptr, int& smem_offset, Offset offset } template -_CCCL_DEVICE auto copy_and_return_smem_dst_fallback( +_CCCL_DEVICE_API auto copy_and_return_smem_dst_fallback( AlignedPtr aligned_ptr, int& smem_offset, Offset offset, char* smem, int valid_items, int tile_size) { // TODO(bgruber): drop handling of head bytes and just read OOB, since gmem buffers are always sufficiently aligned @@ -587,7 +588,7 @@ template -_CCCL_DEVICE void transform_kernel_ldgsts( +_CCCL_DEVICE_API void transform_kernel_ldgsts( Offset num_items, int num_elem_per_thread, Predicate pred, @@ -658,7 +659,7 @@ _CCCL_DEVICE void transform_kernel_ldgsts( } template -_CCCL_DEVICE void bulk_copy_maybe_unaligned( +_CCCL_DEVICE_API void bulk_copy_maybe_unaligned( void* dst, const void* src, unsigned int bytes_to_copy, @@ -738,7 +739,7 @@ template -_CCCL_DEVICE void transform_kernel_ublkcp( +_CCCL_DEVICE_API void transform_kernel_ublkcp( Offset num_items, int num_elem_per_thread, Predicate pred, @@ -1199,10 +1200,10 @@ union kernel_arg // thrust::normal_iterator>), so because of // https://eel.is/c++draft/class.union#general-note-3, kernel_args's special members are deleted. We work around it by // explicitly defining them. - _CCCL_HOST_DEVICE kernel_arg() noexcept {} - _CCCL_HOST_DEVICE ~kernel_arg() noexcept {} + _CCCL_HOST_DEVICE_API kernel_arg() noexcept {} + _CCCL_HOST_DEVICE_API ~kernel_arg() noexcept {} - _CCCL_HOST_DEVICE kernel_arg(const kernel_arg& other) + _CCCL_HOST_DEVICE_API kernel_arg(const kernel_arg& other) { // since we use kernel_arg only to pass data to the device, the contained data is semantically trivially copyable, // even if the type system is telling us otherwise. @@ -1211,7 +1212,7 @@ union kernel_arg }; template -_CCCL_HOST_DEVICE auto make_iterator_kernel_arg(It it) -> kernel_arg +_CCCL_HOST_DEVICE_API auto make_iterator_kernel_arg(It it) -> kernel_arg { kernel_arg arg; // since we switch the active member of the union, we must use placement new or construct_at. This also uses the copy @@ -1222,7 +1223,7 @@ _CCCL_HOST_DEVICE auto make_iterator_kernel_arg(It it) -> kernel_arg } template -_CCCL_HOST_DEVICE auto make_aligned_base_ptr_kernel_arg(It ptr, int alignment) -> kernel_arg +_CCCL_HOST_DEVICE_API auto make_aligned_base_ptr_kernel_arg(It ptr, int alignment) -> kernel_arg { kernel_arg arg; arg.aligned_ptr = _CUB::detail::transform::make_aligned_base_ptr(ptr, alignment); diff --git a/cub/cub/device/dispatch/tuning/common.cuh b/cub/cub/device/dispatch/tuning/common.cuh index c0acefcf6f18..751d82d5692d 100644 --- a/cub/cub/device/dispatch/tuning/common.cuh +++ b/cub/cub/device/dispatch/tuning/common.cuh @@ -235,7 +235,7 @@ namespace detail } // namespace detail #if _CCCL_HOSTED() -inline ::std::ostream& operator<<(::std::ostream& os, LookbackDelayAlgorithm algo) +_CCCL_HOST_API inline ::std::ostream& operator<<(::std::ostream& os, LookbackDelayAlgorithm algo) { return os << _CUB::detail::to_string(algo); } @@ -248,7 +248,7 @@ template <::cuda::std::same_as CharT> struct std::formatter<_CUB::LookbackDelayAlgorithm, CharT> : formatter { template - auto format(const _CUB::LookbackDelayAlgorithm& algo, FmtCtx& ctx) const + _CCCL_HOST_API auto format(const _CUB::LookbackDelayAlgorithm& algo, FmtCtx& ctx) const { return formatter::format(_CUB::detail::to_string(algo), ctx); } @@ -277,7 +277,7 @@ struct LookbackDelayPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const LookbackDelayPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const LookbackDelayPolicy& p) { return os << "LookbackDelayPolicy { .kind = " << p.kind << ", .delay = " << p.delay << ", .l2_write_latency = " << p.l2_write_latency << " }"; diff --git a/cub/cub/device/dispatch/tuning/structural_inplace_vector.cuh b/cub/cub/device/dispatch/tuning/structural_inplace_vector.cuh index ecdcdeae139d..076d7f557cd6 100644 --- a/cub/cub/device/dispatch/tuning/structural_inplace_vector.cuh +++ b/cub/cub/device/dispatch/tuning/structural_inplace_vector.cuh @@ -37,7 +37,7 @@ struct structural_inplace_vector T elems[Capacity]{}; size_type count = 0; - constexpr structural_inplace_vector() = default; + _CCCL_HIDE_FROM_ABI constexpr structural_inplace_vector() = default; // A parameter pack rather than an initializer_list constructor: some MSVC toolsets fail to constant-fold this // type through CC dispatch's NTTP-based policy resolution when constructed from an initializer_list. diff --git a/cub/cub/device/dispatch/tuning/tuning_adjacent_difference.cuh b/cub/cub/device/dispatch/tuning/tuning_adjacent_difference.cuh index 824e1536cb98..743ce114cd8f 100644 --- a/cub/cub/device/dispatch/tuning/tuning_adjacent_difference.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_adjacent_difference.cuh @@ -50,7 +50,7 @@ struct AdjacentDifferencePolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const AdjacentDifferencePolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const AdjacentDifferencePolicy& p) { return os << "AdjacentDifferencePolicy { .threads_per_block = " << p.threads_per_block << ", .items_per_thread = " << p.items_per_thread << ", .load_algorithm = " << p.load_algorithm diff --git a/cub/cub/device/dispatch/tuning/tuning_batch_memcpy.cuh b/cub/cub/device/dispatch/tuning/tuning_batch_memcpy.cuh index a8b9ff97d704..c1c96480ae4b 100644 --- a/cub/cub/device/dispatch/tuning/tuning_batch_memcpy.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_batch_memcpy.cuh @@ -60,7 +60,7 @@ struct BatchedCopySmallBufferPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const BatchedCopySmallBufferPolicy& policy) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const BatchedCopySmallBufferPolicy& policy) { return os << "BatchedCopySmallBufferPolicy { .threads_per_block = " << policy.threads_per_block @@ -93,7 +93,7 @@ struct BatchedCopyLargeBufferPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const BatchedCopyLargeBufferPolicy& policy) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const BatchedCopyLargeBufferPolicy& policy) { return os << "BatchedCopyLargeBufferPolicy { .threads_per_block = " << policy.threads_per_block << ", .bytes_per_thread = " << policy.bytes_per_thread << " }"; @@ -120,7 +120,7 @@ struct BatchedCopyLookbackPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const BatchedCopyLookbackPolicy& policy) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const BatchedCopyLookbackPolicy& policy) { return os << "BatchedCopyLookbackPolicy { .small_buffer = " << policy.small_buffer << ", .large_buffer = " << policy.large_buffer << " }"; @@ -148,7 +148,7 @@ namespace detail } } // namespace detail -inline ::std::ostream& operator<<(::std::ostream& os, BatchedCopyAlgorithm algo) +_CCCL_HOST_API inline ::std::ostream& operator<<(::std::ostream& os, BatchedCopyAlgorithm algo) { return os << _CUB::detail::to_string(algo); } @@ -174,7 +174,7 @@ struct BatchedCopyPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const BatchedCopyPolicy& policy) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const BatchedCopyPolicy& policy) { return os << "BatchedCopyPolicy { .algorithm = " << policy.algorithm << ", .lookback = " << policy.lookback << " }"; } diff --git a/cub/cub/device/dispatch/tuning/tuning_batched_topk.cuh b/cub/cub/device/dispatch/tuning/tuning_batched_topk.cuh index b4a20f66f7af..32e3dfb1fb52 100644 --- a/cub/cub/device/dispatch/tuning/tuning_batched_topk.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_batched_topk.cuh @@ -49,7 +49,7 @@ struct epilogue_policy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const epilogue_policy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const epilogue_policy& p) { return os << "epilogue_policy { .items_per_thread = " << p.items_per_thread << ", .load_algorithm = " << p.load_algorithm @@ -82,7 +82,7 @@ struct worker_policy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const worker_policy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const worker_policy& p) { return os << "worker_policy { .threads_per_block = " << p.threads_per_block << ", .items_per_thread = " << p.items_per_thread << ", .load_algorithm = " << p.load_algorithm @@ -109,7 +109,7 @@ struct multi_worker_policy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const multi_worker_policy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const multi_worker_policy& p) { return os << "multi_worker_policy { .threads_per_block = " << p.threads_per_block << ", .items_per_thread = " << p.items_per_thread << " }"; @@ -163,7 +163,7 @@ struct baseline_topk_policy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const baseline_topk_policy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const baseline_topk_policy& p) { os << "baseline_topk_policy { .worker_per_segment_policies = { "; for (::cuda::std::size_t i = 0; i < p.worker_per_segment_policies.size(); ++i) @@ -280,7 +280,7 @@ struct cluster_topk_policy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const cluster_topk_policy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const cluster_topk_policy& p) { return os << "cluster_topk_policy { .threads_per_block = " << p.threads_per_block @@ -2000,7 +2000,7 @@ enum class topk_algorithm }; #if _CCCL_HOSTED() -[[nodiscard]] inline ::std::ostream& operator<<(::std::ostream& os, topk_algorithm backend) +[[nodiscard]] _CCCL_HOST_API inline ::std::ostream& operator<<(::std::ostream& os, topk_algorithm backend) { switch (backend) { @@ -2037,7 +2037,7 @@ struct topk_policy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const topk_policy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const topk_policy& p) { return os << "topk_policy { .backend = " << p.backend << ", .baseline = " << p.baseline << ", .cluster = " << p.cluster << " }"; diff --git a/cub/cub/device/dispatch/tuning/tuning_find.cuh b/cub/cub/device/dispatch/tuning/tuning_find.cuh index 4f77c89f6eb4..b12ae5f564a4 100644 --- a/cub/cub/device/dispatch/tuning/tuning_find.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_find.cuh @@ -46,7 +46,7 @@ struct FindIfPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const FindIfPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const FindIfPolicy& p) { return os << "FindIfPolicy { .threads_per_block = " << p.threads_per_block << ", .items_per_thread = " diff --git a/cub/cub/device/dispatch/tuning/tuning_find_bound_sorted_values.cuh b/cub/cub/device/dispatch/tuning/tuning_find_bound_sorted_values.cuh index 78a7adf52d35..652e31246b22 100644 --- a/cub/cub/device/dispatch/tuning/tuning_find_bound_sorted_values.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_find_bound_sorted_values.cuh @@ -49,7 +49,7 @@ struct FindBoundSortedValuesPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const FindBoundSortedValuesPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const FindBoundSortedValuesPolicy& p) { return os << "FindBoundSortedValuesPolicy { .threads_per_block = " << p.threads_per_block << ", .items_per_thread = " << p.items_per_thread << ", .load_modifier = " << p.load_modifier << " }"; diff --git a/cub/cub/device/dispatch/tuning/tuning_for.cuh b/cub/cub/device/dispatch/tuning/tuning_for.cuh index 4b756366dbb6..a771c9b5eb33 100644 --- a/cub/cub/device/dispatch/tuning/tuning_for.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_for.cuh @@ -41,7 +41,7 @@ struct ForPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const ForPolicy& policy) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const ForPolicy& policy) { return os << "ForPolicy { .threads_per_block = " << policy.threads_per_block << ", .items_per_thread = " << policy.items_per_thread << " }"; diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index 2b220c70ba86..154fd7500962 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -59,7 +59,7 @@ struct HistogramPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const HistogramPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const HistogramPolicy& p) { return os << "HistogramPolicy { .threads_per_block = " << p.threads_per_block << ", .pixels_per_thread = " diff --git a/cub/cub/device/dispatch/tuning/tuning_merge.cuh b/cub/cub/device/dispatch/tuning/tuning_merge.cuh index c0e66bbacd38..ff94aefe43fd 100644 --- a/cub/cub/device/dispatch/tuning/tuning_merge.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_merge.cuh @@ -66,7 +66,7 @@ struct MergePolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const MergePolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const MergePolicy& p) { return os << "MergePolicy { .threads_per_block = " << p.threads_per_block diff --git a/cub/cub/device/dispatch/tuning/tuning_merge_sort.cuh b/cub/cub/device/dispatch/tuning/tuning_merge_sort.cuh index 80bf1aab66d1..d0f43d6c21e4 100644 --- a/cub/cub/device/dispatch/tuning/tuning_merge_sort.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_merge_sort.cuh @@ -85,7 +85,7 @@ struct MergeSortPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const MergeSortPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const MergeSortPolicy& p) { return os << "MergeSortPolicy { .threads_per_block = " << p.threads_per_block << ", .items_per_thread = " diff --git a/cub/cub/device/dispatch/tuning/tuning_radix_sort.cuh b/cub/cub/device/dispatch/tuning/tuning_radix_sort.cuh index 4767fefb5e7e..965d443e608e 100644 --- a/cub/cub/device/dispatch/tuning/tuning_radix_sort.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_radix_sort.cuh @@ -71,7 +71,7 @@ namespace detail #endif // _CCCL_HOSTED() #if _CCCL_HOSTED() -inline ::std::ostream& operator<<(::std::ostream& os, RadixSortAlgorithm algo) +_CCCL_HOST_API inline ::std::ostream& operator<<(::std::ostream& os, RadixSortAlgorithm algo) { return os << _CUB::detail::to_string(algo); } @@ -118,7 +118,7 @@ struct RadixSortHistogramPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const RadixSortHistogramPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const RadixSortHistogramPolicy& p) { return os << "RadixSortHistogramPolicy { .threads_per_block = " << p.threads_per_block << ", .items_per_thread = " << p.items_per_thread << ", .private_partitions = " << p.private_partitions @@ -146,7 +146,7 @@ struct RadixSortExclusiveSumPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const RadixSortExclusiveSumPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const RadixSortExclusiveSumPolicy& p) { return os << "RadixSortExclusiveSumPolicy { .threads_per_block = " << p.threads_per_block << ", .radix_bits = " << p.radix_bits << " }"; @@ -186,7 +186,7 @@ struct RadixSortOnesweepPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const RadixSortOnesweepPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const RadixSortOnesweepPolicy& p) { return os << "RadixSortOnesweepPolicy { .threads_per_block = " << p.threads_per_block @@ -224,7 +224,7 @@ struct RadixSortDownsweepPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const RadixSortDownsweepPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const RadixSortDownsweepPolicy& p) { return os << "RadixSortDownsweepPolicy { .threads_per_block = " << p.threads_per_block @@ -257,7 +257,7 @@ struct RadixSortUpsweepPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const RadixSortUpsweepPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const RadixSortUpsweepPolicy& p) { return os << "RadixSortUpsweepPolicy { .threads_per_block = " << p.threads_per_block << ", .items_per_thread = " @@ -297,7 +297,7 @@ struct RadixSortPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const RadixSortPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const RadixSortPolicy& p) { return os << "RadixSortPolicy { .algorithm = " << p.algorithm << ", .histogram = " << p.histogram @@ -971,7 +971,7 @@ get_sm107_tuning(int key_size, int value_size, int offset_size, type_t key_type) template struct RadixSortPolicyWrapper : PolicyT { - _CCCL_HOST_DEVICE RadixSortPolicyWrapper(PolicyT base) + _CCCL_HOST_DEVICE_API RadixSortPolicyWrapper(PolicyT base) : PolicyT(base) {} }; @@ -992,23 +992,23 @@ struct RadixSortPolicyWrapper< typename StaticPolicyT::SegmentedPolicy, typename StaticPolicyT::AltSegmentedPolicy>> : StaticPolicyT { - _CCCL_HOST_DEVICE RadixSortPolicyWrapper(StaticPolicyT base) + _CCCL_HOST_DEVICE_API RadixSortPolicyWrapper(StaticPolicyT base) : StaticPolicyT(base) {} - _CCCL_HOST_DEVICE static constexpr bool IsOnesweep() + _CCCL_HOST_DEVICE_API static constexpr bool IsOnesweep() { return StaticPolicyT::ONESWEEP; } template - _CCCL_HOST_DEVICE static constexpr int RadixBits(PolicyT /*policy*/) + _CCCL_HOST_DEVICE_API static constexpr int RadixBits(PolicyT /*policy*/) { return PolicyT::RADIX_BITS; } template - _CCCL_HOST_DEVICE static constexpr int ThreadsPerBlock(PolicyT /*policy*/) + _CCCL_HOST_DEVICE_API static constexpr int ThreadsPerBlock(PolicyT /*policy*/) { return PolicyT::BLOCK_THREADS; } @@ -1028,7 +1028,7 @@ struct RadixSortPolicyWrapper< // TODO(bgruber): remove in CCCL 4.0 when we drop the radix sort dispatcher after publishing the tuning API template -_CCCL_HOST_DEVICE RadixSortPolicyWrapper MakeRadixSortPolicyWrapper(PolicyT policy) +_CCCL_HOST_DEVICE_API RadixSortPolicyWrapper MakeRadixSortPolicyWrapper(PolicyT policy) { return RadixSortPolicyWrapper{policy}; } diff --git a/cub/cub/device/dispatch/tuning/tuning_reduce.cuh b/cub/cub/device/dispatch/tuning/tuning_reduce.cuh index 375d52730f87..2618e3e7603c 100644 --- a/cub/cub/device/dispatch/tuning/tuning_reduce.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_reduce.cuh @@ -54,7 +54,7 @@ struct ReducePassPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const ReducePassPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const ReducePassPolicy& p) { return os << "ReducePassPolicy { .threads_per_block = " << p.threads_per_block << ", .items_per_thread = " << p.items_per_thread << ", .vec_size = " << p.vec_size @@ -83,7 +83,7 @@ struct ReducePolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const ReducePolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const ReducePolicy& p) { return os << "ReducePolicy { .multi_tile = " << p.multi_tile << ", .single_tile = " << p.single_tile << " }"; } @@ -112,7 +112,7 @@ concept reduce_policy_selector = policy_selector; template struct ReducePolicyWrapper : PolicyT { - _CCCL_HOST_DEVICE ReducePolicyWrapper(PolicyT base) + _CCCL_HOST_DEVICE_API ReducePolicyWrapper(PolicyT base) : PolicyT(base) {} }; @@ -123,7 +123,7 @@ struct ReducePolicyWrapper< StaticPolicyT, ::cuda::std::void_t> : StaticPolicyT { - _CCCL_HOST_DEVICE ReducePolicyWrapper(StaticPolicyT base) + _CCCL_HOST_DEVICE_API ReducePolicyWrapper(StaticPolicyT base) : StaticPolicyT(base) {} @@ -135,7 +135,7 @@ struct ReducePolicyWrapper< // TODO(bgruber): remove in CCCL 4.0 when we drop the dispatchers template -_CCCL_HOST_DEVICE ReducePolicyWrapper MakeReducePolicyWrapper(PolicyT policy) +_CCCL_HOST_DEVICE_API ReducePolicyWrapper MakeReducePolicyWrapper(PolicyT policy) { return ReducePolicyWrapper{policy}; } @@ -162,7 +162,7 @@ enum class accum_size unknown }; template -_CCCL_HOST_DEVICE constexpr accum_size classify_accum_size() +_CCCL_HOST_DEVICE_API constexpr accum_size classify_accum_size() { return sizeof(AccumT) == 1 ? accum_size::_1 : sizeof(AccumT) == 2 ? accum_size::_2 @@ -173,7 +173,7 @@ _CCCL_HOST_DEVICE constexpr accum_size classify_accum_size() : accum_size::unknown; } template -_CCCL_HOST_DEVICE constexpr offset_size classify_offset_size() +_CCCL_HOST_DEVICE_API constexpr offset_size classify_offset_size() { return sizeof(OffsetT) == 4 ? offset_size::_4 : sizeof(OffsetT) == 8 ? offset_size::_8 : offset_size::unknown; } diff --git a/cub/cub/device/dispatch/tuning/tuning_reduce_by_key.cuh b/cub/cub/device/dispatch/tuning/tuning_reduce_by_key.cuh index eaa6ab61793b..33128ee158eb 100644 --- a/cub/cub/device/dispatch/tuning/tuning_reduce_by_key.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_reduce_by_key.cuh @@ -56,7 +56,7 @@ struct ReduceByKeyLookbackPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const ReduceByKeyLookbackPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const ReduceByKeyLookbackPolicy& p) { return os << "ReduceByKeyLookbackPolicy { .threads_per_block = " << p.threads_per_block << ", .items_per_thread = " @@ -86,7 +86,7 @@ namespace detail } } // namespace detail -inline ::std::ostream& operator<<(::std::ostream& os, ReduceByKeyAlgorithm algo) +_CCCL_HOST_API inline ::std::ostream& operator<<(::std::ostream& os, ReduceByKeyAlgorithm algo) { return os << _CUB::detail::to_string(algo); } @@ -112,7 +112,7 @@ struct ReduceByKeyPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const ReduceByKeyPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const ReduceByKeyPolicy& p) { return os << "ReduceByKeyPolicy { .algorithm = " << p.algorithm << ", .lookback = " << p.lookback << " }"; } diff --git a/cub/cub/device/dispatch/tuning/tuning_rle_encode.cuh b/cub/cub/device/dispatch/tuning/tuning_rle_encode.cuh index 6f53c8159495..9a383f1048ca 100644 --- a/cub/cub/device/dispatch/tuning/tuning_rle_encode.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_rle_encode.cuh @@ -67,7 +67,7 @@ struct RleLookbackPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const RleLookbackPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const RleLookbackPolicy& p) { return os << "RleLookbackPolicy { .threads_per_block = " << p.threads_per_block << ", .items_per_thread = " @@ -100,7 +100,7 @@ namespace detail } } // namespace detail -inline ::std::ostream& operator<<(::std::ostream& os, RleAlgorithm algo) +_CCCL_HOST_API inline ::std::ostream& operator<<(::std::ostream& os, RleAlgorithm algo) { return os << _CUB::detail::to_string(algo); } @@ -212,7 +212,7 @@ struct RleLookaheadPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const RleLookaheadPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const RleLookaheadPolicy& p) { return os << "RleLookaheadPolicy { .items_per_thread = " << p.items_per_thread << ", .compute_warps = " << p.compute_warps @@ -245,7 +245,7 @@ struct RleEncodePolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const RleEncodePolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const RleEncodePolicy& p) { return os << "RleEncodePolicy { .algorithm = " << p.algorithm << ", .lookback = " << p.lookback << ", .lookahead = " << p.lookahead << " }"; diff --git a/cub/cub/device/dispatch/tuning/tuning_rle_non_trivial_runs.cuh b/cub/cub/device/dispatch/tuning/tuning_rle_non_trivial_runs.cuh index 7a4a9db692c2..dad0f40a3597 100644 --- a/cub/cub/device/dispatch/tuning/tuning_rle_non_trivial_runs.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_rle_non_trivial_runs.cuh @@ -61,7 +61,7 @@ struct RleNonTrivialRunsLookbackPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const RleNonTrivialRunsLookbackPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const RleNonTrivialRunsLookbackPolicy& p) { return os << "RleNonTrivialRunsLookbackPolicy { .threads_per_block = " << p.threads_per_block @@ -92,7 +92,7 @@ namespace detail } } // namespace detail -inline ::std::ostream& operator<<(::std::ostream& os, RleNonTrivialRunsAlgorithm algo) +_CCCL_HOST_API inline ::std::ostream& operator<<(::std::ostream& os, RleNonTrivialRunsAlgorithm algo) { return os << _CUB::detail::to_string(algo); } @@ -120,7 +120,7 @@ struct RleNonTrivialRunsPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const RleNonTrivialRunsPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const RleNonTrivialRunsPolicy& p) { return os << "RleNonTrivialRunsPolicy { .algorithm = " << p.algorithm << ", .lookback = " << p.lookback << " }"; } diff --git a/cub/cub/device/dispatch/tuning/tuning_scan.cuh b/cub/cub/device/dispatch/tuning/tuning_scan.cuh index 72fef9cd32ac..536567c70fd8 100644 --- a/cub/cub/device/dispatch/tuning/tuning_scan.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_scan.cuh @@ -76,7 +76,7 @@ namespace detail #endif // _CCCL_HOSTED() #if _CCCL_HOSTED() -inline ::std::ostream& operator<<(::std::ostream& os, ScanAlgorithm algo) +_CCCL_HOST_API inline ::std::ostream& operator<<(::std::ostream& os, ScanAlgorithm algo) { return os << _CUB::detail::to_string(algo); } @@ -89,7 +89,7 @@ template <::cuda::std::same_as CharT> struct std::formatter<_CUB::ScanAlgorithm, CharT> : formatter { template - auto format(const _CUB::ScanAlgorithm& algo, FmtCtx& ctx) const + _CCCL_HOST_API auto format(const _CUB::ScanAlgorithm& algo, FmtCtx& ctx) const { const auto str = _CUB::detail::to_string(algo); return formatter::format(str, ctx); @@ -132,7 +132,7 @@ struct ScanLookbackPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const ScanLookbackPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const ScanLookbackPolicy& p) { return os << "ScanLookbackPolicy { .threads_per_block = " << p.threads_per_block << ", .items_per_thread = " @@ -187,7 +187,7 @@ struct ScanLookaheadPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const ScanLookaheadPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const ScanLookaheadPolicy& p) { return os << "ScanLookaheadPolicy { .reduce_and_scan_warps = " << p.reduce_and_scan_warps << ", .items_per_thread = " @@ -219,7 +219,7 @@ struct ScanPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const ScanPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const ScanPolicy& p) { return os << "ScanPolicy { .algorithm = " << p.algorithm << ", .lookback = " << p.lookback << ", .lookahead = " << p.lookahead << " }"; @@ -277,21 +277,21 @@ enum class accum_size // TODO(bgruber): remove this in CCCL 4.0 when we remove the public scan dispatcher template -constexpr _CCCL_HOST_DEVICE primitive_accum is_primitive_accum() +_CCCL_HOST_DEVICE_API constexpr primitive_accum is_primitive_accum() { return is_primitive::value ? primitive_accum::yes : primitive_accum::no; } // TODO(bgruber): remove this in CCCL 4.0 when we remove the public scan dispatcher template -constexpr _CCCL_HOST_DEVICE primitive_op is_primitive_op() +_CCCL_HOST_DEVICE_API constexpr primitive_op is_primitive_op() { return basic_binary_op_v ? primitive_op::yes : primitive_op::no; } // TODO(bgruber): remove this in CCCL 4.0 when we remove the public scan dispatcher template -constexpr _CCCL_HOST_DEVICE value_size classify_value_size() +_CCCL_HOST_DEVICE_API constexpr value_size classify_value_size() { return sizeof(ValueT) == 1 ? value_size::_1 : sizeof(ValueT) == 2 ? value_size::_2 @@ -304,7 +304,7 @@ constexpr _CCCL_HOST_DEVICE value_size classify_value_size() // TODO(bgruber): remove this in CCCL 4.0 when we remove the public scan dispatcher template -constexpr _CCCL_HOST_DEVICE accum_size classify_accum_size() +_CCCL_HOST_DEVICE_API constexpr accum_size classify_accum_size() { return sizeof(AccumT) == 1 ? accum_size::_1 : sizeof(AccumT) == 2 ? accum_size::_2 @@ -317,7 +317,7 @@ constexpr _CCCL_HOST_DEVICE accum_size classify_accum_size() // TODO(bgruber): remove this in CCCL 4.0 when we remove the public scan dispatcher template -constexpr _CCCL_HOST_DEVICE offset_size classify_offset_size() +_CCCL_HOST_DEVICE_API constexpr offset_size classify_offset_size() { return sizeof(OffsetT) == 4 ? offset_size::_4 : sizeof(OffsetT) == 8 ? offset_size::_8 : offset_size::unknown; } diff --git a/cub/cub/device/dispatch/tuning/tuning_scan_by_key.cuh b/cub/cub/device/dispatch/tuning/tuning_scan_by_key.cuh index 0f7352a0384c..5f44c5a026de 100644 --- a/cub/cub/device/dispatch/tuning/tuning_scan_by_key.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_scan_by_key.cuh @@ -63,7 +63,7 @@ struct ScanByKeyLookbackPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const ScanByKeyLookbackPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const ScanByKeyLookbackPolicy& p) { return os << "ScanByKeyLookbackPolicy { .threads_per_block = " << p.threads_per_block @@ -94,7 +94,7 @@ namespace detail } } // namespace detail -inline ::std::ostream& operator<<(::std::ostream& os, ScanByKeyAlgorithm algo) +_CCCL_HOST_API inline ::std::ostream& operator<<(::std::ostream& os, ScanByKeyAlgorithm algo) { return os << _CUB::detail::to_string(algo); } @@ -107,7 +107,7 @@ template <::cuda::std::same_as CharT> struct std::formatter<_CUB::ScanByKeyAlgorithm, CharT> : formatter { template - auto format(const _CUB::ScanByKeyAlgorithm& algo, FmtCtx& ctx) const + _CCCL_HOST_API auto format(const _CUB::ScanByKeyAlgorithm& algo, FmtCtx& ctx) const { return formatter::format(_CUB::detail::to_string(algo), ctx); } @@ -136,7 +136,7 @@ struct ScanByKeyPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const ScanByKeyPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const ScanByKeyPolicy& p) { return os << "ScanByKeyPolicy { .algorithm = " << p.algorithm << ", .lookback = " << p.lookback << " }"; } @@ -175,19 +175,19 @@ enum class key_size }; template -constexpr primitive_accum is_primitive_accum() +_CCCL_HOST_API constexpr primitive_accum is_primitive_accum() { return is_primitive::value ? primitive_accum::yes : primitive_accum::no; } template -constexpr primitive_op is_primitive_op() +_CCCL_HOST_API constexpr primitive_op is_primitive_op() { return basic_binary_op_v ? primitive_op::yes : primitive_op::no; } template -constexpr val_size classify_val_size() +_CCCL_HOST_API constexpr val_size classify_val_size() { return sizeof(ValueT) == 1 ? val_size::_1 : sizeof(ValueT) == 2 ? val_size::_2 @@ -199,7 +199,7 @@ constexpr val_size classify_val_size() } template -constexpr key_size classify_key_size() +_CCCL_HOST_API constexpr key_size classify_key_size() { return sizeof(KeyT) == 1 ? key_size::_1 : sizeof(KeyT) == 2 ? key_size::_2 diff --git a/cub/cub/device/dispatch/tuning/tuning_segmented_radix_sort.cuh b/cub/cub/device/dispatch/tuning/tuning_segmented_radix_sort.cuh index e37a60677f99..b27011e0a2cc 100644 --- a/cub/cub/device/dispatch/tuning/tuning_segmented_radix_sort.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_segmented_radix_sort.cuh @@ -50,7 +50,7 @@ struct SegmentedRadixSortPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const SegmentedRadixSortPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const SegmentedRadixSortPolicy& p) { return os << "SegmentedRadixSortPolicy { .regular_pass = " << p.regular_pass << ", .alternate_pass = " << p.alternate_pass << " }"; diff --git a/cub/cub/device/dispatch/tuning/tuning_segmented_reduce.cuh b/cub/cub/device/dispatch/tuning/tuning_segmented_reduce.cuh index f69cc167985c..f5105b91b44c 100644 --- a/cub/cub/device/dispatch/tuning/tuning_segmented_reduce.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_segmented_reduce.cuh @@ -59,7 +59,7 @@ struct SegmentedReduceWarpReducePolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const SegmentedReduceWarpReducePolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const SegmentedReduceWarpReducePolicy& p) { return os << "SegmentedReduceWarpReducePolicy { .threads_per_block = " << p.threads_per_block << ", .threads_per_warp = " << p.threads_per_warp << ", .items_per_thread = " << p.items_per_thread @@ -89,7 +89,7 @@ struct SegmentedReducePolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const SegmentedReducePolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const SegmentedReducePolicy& p) { return os << "SegmentedReducePolicy { .large_reduce = " << p.large_reduce << ", .medium_reduce = " << p.medium_reduce << ", .small_reduce = " << p.small_reduce << " }"; diff --git a/cub/cub/device/dispatch/tuning/tuning_segmented_scan.cuh b/cub/cub/device/dispatch/tuning/tuning_segmented_scan.cuh index ab7abc8a68d5..0751c0c2ef4e 100644 --- a/cub/cub/device/dispatch/tuning/tuning_segmented_scan.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_segmented_scan.cuh @@ -56,7 +56,7 @@ struct SegmentedScanBlockPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const SegmentedScanBlockPolicy& policy) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const SegmentedScanBlockPolicy& policy) { return os << "SegmentedScanBlockPolicy { .threads_per_block = " << policy.threads_per_block @@ -86,7 +86,7 @@ struct SegmentedScanPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const SegmentedScanPolicy& policy) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const SegmentedScanPolicy& policy) { return os << "SegmentedScanPolicy { .block = " << policy.block << " }"; } diff --git a/cub/cub/device/dispatch/tuning/tuning_segmented_sort.cuh b/cub/cub/device/dispatch/tuning/tuning_segmented_sort.cuh index 58e1f023a5ae..b3f7e9de7a22 100644 --- a/cub/cub/device/dispatch/tuning/tuning_segmented_sort.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_segmented_sort.cuh @@ -61,7 +61,7 @@ struct SegmentedSortRadixSortPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const SegmentedSortRadixSortPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const SegmentedSortRadixSortPolicy& p) { return os << "SegmentedSortRadixSortPolicy { .threads_per_block = " << p.threads_per_block @@ -107,7 +107,7 @@ struct SegmentedSortSubWarpMergeSortPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const SegmentedSortSubWarpMergeSortPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const SegmentedSortSubWarpMergeSortPolicy& p) { return os << "SegmentedSortSubWarpMergeSortPolicy { .threads_per_block = " << p.threads_per_block @@ -141,7 +141,7 @@ struct SegmentedSortPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const SegmentedSortPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const SegmentedSortPolicy& p) { return os << "SegmentedSortPolicy { .large_segment = " << p.large_segment << ", .medium_segment = " << p.medium_segment << ", .small_segment = " << p.small_segment @@ -308,7 +308,7 @@ struct policy_selector_from_types template struct SegmentedSortPolicyWrapper : PolicyT { - _CCCL_HOST_DEVICE SegmentedSortPolicyWrapper(PolicyT base) + _CCCL_HOST_DEVICE_API SegmentedSortPolicyWrapper(PolicyT base) : PolicyT(base) {} }; @@ -320,81 +320,81 @@ struct SegmentedSortPolicyWrapper> : StaticPolicyT { - _CCCL_HOST_DEVICE SegmentedSortPolicyWrapper(StaticPolicyT base) + _CCCL_HOST_DEVICE_API SegmentedSortPolicyWrapper(StaticPolicyT base) : StaticPolicyT(base) {} - _CCCL_HOST_DEVICE static constexpr int PartitioningThreshold() + _CCCL_HOST_DEVICE_API static constexpr int PartitioningThreshold() { return StaticPolicyT::PARTITIONING_THRESHOLD; } - _CCCL_HOST_DEVICE static constexpr int LargeSegmentRadixBits() + _CCCL_HOST_DEVICE_API static constexpr int LargeSegmentRadixBits() { return StaticPolicyT::LargeSegmentPolicy::RADIX_BITS; } - _CCCL_HOST_DEVICE static constexpr int LargeSegmentThreadsPerBlock() + _CCCL_HOST_DEVICE_API static constexpr int LargeSegmentThreadsPerBlock() { return StaticPolicyT::LargeSegmentPolicy::BLOCK_THREADS; } - _CCCL_HOST_DEVICE static constexpr int LargeSegmentItemsPerThread() + _CCCL_HOST_DEVICE_API static constexpr int LargeSegmentItemsPerThread() { return StaticPolicyT::LargeSegmentPolicy::ITEMS_PER_THREAD; } - _CCCL_HOST_DEVICE static constexpr int SmallSegmentThreadsPerBlock() + _CCCL_HOST_DEVICE_API static constexpr int SmallSegmentThreadsPerBlock() { return StaticPolicyT::SmallSegmentPolicy::BLOCK_THREADS; } - _CCCL_HOST_DEVICE static constexpr int SegmentsPerSmallBlock() + _CCCL_HOST_DEVICE_API static constexpr int SegmentsPerSmallBlock() { return StaticPolicyT::SmallSegmentPolicy::SEGMENTS_PER_BLOCK; } - _CCCL_HOST_DEVICE static constexpr int SegmentsPerMediumBlock() + _CCCL_HOST_DEVICE_API static constexpr int SegmentsPerMediumBlock() { return StaticPolicyT::MediumSegmentPolicy::SEGMENTS_PER_BLOCK; } - _CCCL_HOST_DEVICE static constexpr int SmallPolicyItemsPerTile() + _CCCL_HOST_DEVICE_API static constexpr int SmallPolicyItemsPerTile() { return StaticPolicyT::SmallSegmentPolicy::ITEMS_PER_TILE; } - _CCCL_HOST_DEVICE static constexpr int MediumPolicyItemsPerTile() + _CCCL_HOST_DEVICE_API static constexpr int MediumPolicyItemsPerTile() { return StaticPolicyT::MediumSegmentPolicy::ITEMS_PER_TILE; } - _CCCL_HOST_DEVICE static constexpr CacheLoadModifier LargeSegmentLoadModifier() + _CCCL_HOST_DEVICE_API static constexpr CacheLoadModifier LargeSegmentLoadModifier() { return StaticPolicyT::LargeSegmentPolicy::LOAD_MODIFIER; } - _CCCL_HOST_DEVICE static constexpr BlockLoadAlgorithm LargeSegmentLoadAlgorithm() + _CCCL_HOST_DEVICE_API static constexpr BlockLoadAlgorithm LargeSegmentLoadAlgorithm() { return StaticPolicyT::LargeSegmentPolicy::LOAD_ALGORITHM; } - _CCCL_HOST_DEVICE static constexpr WarpLoadAlgorithm MediumSegmentLoadAlgorithm() + _CCCL_HOST_DEVICE_API static constexpr WarpLoadAlgorithm MediumSegmentLoadAlgorithm() { return StaticPolicyT::MediumSegmentPolicy::LOAD_ALGORITHM; } - _CCCL_HOST_DEVICE static constexpr WarpLoadAlgorithm SmallSegmentLoadAlgorithm() + _CCCL_HOST_DEVICE_API static constexpr WarpLoadAlgorithm SmallSegmentLoadAlgorithm() { return StaticPolicyT::SmallSegmentPolicy::LOAD_ALGORITHM; } - _CCCL_HOST_DEVICE static constexpr WarpStoreAlgorithm MediumSegmentStoreAlgorithm() + _CCCL_HOST_DEVICE_API static constexpr WarpStoreAlgorithm MediumSegmentStoreAlgorithm() { return StaticPolicyT::MediumSegmentPolicy::STORE_ALGORITHM; } - _CCCL_HOST_DEVICE static constexpr WarpStoreAlgorithm SmallSegmentStoreAlgorithm() + _CCCL_HOST_DEVICE_API static constexpr WarpStoreAlgorithm SmallSegmentStoreAlgorithm() { return StaticPolicyT::SmallSegmentPolicy::STORE_ALGORITHM; } @@ -402,7 +402,7 @@ struct SegmentedSortPolicyWrapper -_CCCL_HOST_DEVICE SegmentedSortPolicyWrapper MakeSegmentedSortPolicyWrapper(PolicyT policy) +_CCCL_HOST_DEVICE_API SegmentedSortPolicyWrapper MakeSegmentedSortPolicyWrapper(PolicyT policy) { return SegmentedSortPolicyWrapper{policy}; } diff --git a/cub/cub/device/dispatch/tuning/tuning_select_if.cuh b/cub/cub/device/dispatch/tuning/tuning_select_if.cuh index 53b2d92df233..8f11af8bea0a 100644 --- a/cub/cub/device/dispatch/tuning/tuning_select_if.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_select_if.cuh @@ -68,7 +68,7 @@ struct SelectLookbackPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const SelectLookbackPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const SelectLookbackPolicy& p) { return os << "SelectLookbackPolicy { .threads_per_block = " << p.threads_per_block @@ -99,7 +99,7 @@ namespace detail } } // namespace detail -inline ::std::ostream& operator<<(::std::ostream& os, SelectAlgorithm algo) +_CCCL_HOST_API inline ::std::ostream& operator<<(::std::ostream& os, SelectAlgorithm algo) { return os << _CUB::detail::to_string(algo); } @@ -126,7 +126,7 @@ struct SelectPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const SelectPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const SelectPolicy& p) { return os << "SelectPolicy { .algorithm = " << p.algorithm << ", .lookback = " << p.lookback << " }"; } @@ -153,7 +153,7 @@ namespace detail } } // namespace detail -inline ::std::ostream& operator<<(::std::ostream& os, PartitionAlgorithm algo) +_CCCL_HOST_API inline ::std::ostream& operator<<(::std::ostream& os, PartitionAlgorithm algo) { return os << _CUB::detail::to_string(algo); } @@ -186,7 +186,7 @@ struct PartitionLookbackPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const PartitionLookbackPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const PartitionLookbackPolicy& p) { return os << "PartitionLookbackPolicy { .threads_per_block = " << p.threads_per_block << ", .items_per_thread = " @@ -217,7 +217,7 @@ struct PartitionPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const PartitionPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const PartitionPolicy& p) { return os << "PartitionPolicy { .algorithm = " << p.algorithm << ", .lookback = " << p.lookback << " }"; } @@ -1628,28 +1628,28 @@ struct sm100_tuning -constexpr primitive is_primitive() +_CCCL_HOST_API constexpr primitive is_primitive() { return detail::is_primitive_v ? primitive::yes : primitive::no; } // TODO(bgruber): drop in CCCL 4.0 template -constexpr flagged is_flagged() +_CCCL_HOST_API constexpr flagged is_flagged() { return ::cuda::std::is_same_v ? flagged::no : flagged::yes; } // TODO(bgruber): drop in CCCL 4.0 template -constexpr keep_rejects are_rejects_kept() +_CCCL_HOST_API constexpr keep_rejects are_rejects_kept() { return KeepRejects ? keep_rejects::yes : keep_rejects::no; } // TODO(bgruber): drop in CCCL 4.0 template -constexpr input_size classify_input_size() +_CCCL_HOST_API constexpr input_size classify_input_size() { return sizeof(InputT) == 1 ? input_size::_1 : sizeof(InputT) == 2 ? input_size::_2 @@ -1662,21 +1662,21 @@ constexpr input_size classify_input_size() // TODO(bgruber): drop in CCCL 4.0 template -constexpr offset_size classify_offset_size() +_CCCL_HOST_API constexpr offset_size classify_offset_size() { return sizeof(OffsetT) == 4 ? offset_size::_4 : sizeof(OffsetT) == 8 ? offset_size::_8 : offset_size::unknown; } // TODO(bgruber): drop in CCCL 4.0 template -constexpr may_alias should_alias() +_CCCL_HOST_API constexpr may_alias should_alias() { return Alias ? may_alias::yes : may_alias::no; } // TODO(bgruber): drop in CCCL 4.0 template -constexpr distinct_partitions is_distinct_partitions() +_CCCL_HOST_API constexpr distinct_partitions is_distinct_partitions() { return DistinctPartitions ? distinct_partitions::yes : distinct_partitions::no; } diff --git a/cub/cub/device/dispatch/tuning/tuning_set_operations.cuh b/cub/cub/device/dispatch/tuning/tuning_set_operations.cuh index feff7fbb75da..f87328e721ae 100644 --- a/cub/cub/device/dispatch/tuning/tuning_set_operations.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_set_operations.cuh @@ -51,7 +51,7 @@ struct SetOpsPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const SetOpsPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const SetOpsPolicy& p) { return os << "SetOpsPolicy { .threads_per_block = " << p.threads_per_block << ", .items_per_thread = " << p.items_per_thread << ", .load_modifier = " << p.load_modifier diff --git a/cub/cub/device/dispatch/tuning/tuning_three_way_partition.cuh b/cub/cub/device/dispatch/tuning/tuning_three_way_partition.cuh index e5e9a22b285d..2ffc08e99782 100644 --- a/cub/cub/device/dispatch/tuning/tuning_three_way_partition.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_three_way_partition.cuh @@ -54,7 +54,7 @@ struct ThreeWayPartitionLookbackPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const ThreeWayPartitionLookbackPolicy& policy) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const ThreeWayPartitionLookbackPolicy& policy) { return os << "ThreeWayPartitionLookbackPolicy { .threads_per_block = " << policy.threads_per_block @@ -85,7 +85,7 @@ namespace detail } } // namespace detail -inline ::std::ostream& operator<<(::std::ostream& os, ThreeWayPartitionAlgorithm algo) +_CCCL_HOST_API inline ::std::ostream& operator<<(::std::ostream& os, ThreeWayPartitionAlgorithm algo) { return os << _CUB::detail::to_string(algo); } @@ -111,7 +111,7 @@ struct ThreeWayPartitionPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const ThreeWayPartitionPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const ThreeWayPartitionPolicy& p) { return os << "ThreeWayPartitionPolicy { .algorithm = " << p.algorithm << ", .lookback = " << p.lookback << " }"; } @@ -124,7 +124,7 @@ namespace detail::three_way_partition template struct ThreeWayPartitionPolicyWrapper : PolicyT { - _CCCL_HOST_DEVICE ThreeWayPartitionPolicyWrapper(PolicyT base) + _CCCL_HOST_DEVICE_API ThreeWayPartitionPolicyWrapper(PolicyT base) : PolicyT(base) {} }; @@ -134,7 +134,7 @@ template struct ThreeWayPartitionPolicyWrapper> : StaticPolicyT { - _CCCL_HOST_DEVICE ThreeWayPartitionPolicyWrapper(StaticPolicyT base) + _CCCL_HOST_DEVICE_API ThreeWayPartitionPolicyWrapper(StaticPolicyT base) : StaticPolicyT(base) {} @@ -143,7 +143,7 @@ struct ThreeWayPartitionPolicyWrapper -_CCCL_HOST_DEVICE ThreeWayPartitionPolicyWrapper MakeThreeWayPartitionPolicyWrapper(PolicyT policy) +_CCCL_HOST_DEVICE_API ThreeWayPartitionPolicyWrapper MakeThreeWayPartitionPolicyWrapper(PolicyT policy) { return ThreeWayPartitionPolicyWrapper{policy}; } @@ -166,7 +166,7 @@ enum class offset_size }; template -_CCCL_HOST_DEVICE constexpr input_size classify_input_size() +_CCCL_HOST_DEVICE_API constexpr input_size classify_input_size() { return sizeof(InputT) == 1 ? input_size::_1 : sizeof(InputT) == 2 ? input_size::_2 @@ -178,7 +178,7 @@ _CCCL_HOST_DEVICE constexpr input_size classify_input_size() } template -_CCCL_HOST_DEVICE constexpr offset_size classify_offset_size() +_CCCL_HOST_DEVICE_API constexpr offset_size classify_offset_size() { return sizeof(OffsetT) == 4 ? offset_size::_4 : sizeof(OffsetT) == 8 ? offset_size::_8 : offset_size::unknown; } diff --git a/cub/cub/device/dispatch/tuning/tuning_topk.cuh b/cub/cub/device/dispatch/tuning/tuning_topk.cuh index 3e86f3eac518..bb2ec1bdc1b1 100644 --- a/cub/cub/device/dispatch/tuning/tuning_topk.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_topk.cuh @@ -72,7 +72,7 @@ struct topk_policy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const topk_policy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const topk_policy& p) { return os << "topk_policy { .threads_per_block = " << p.threads_per_block << ", .items_per_thread = " << p.items_per_thread << ", .load_algorithm = " << p.load_algorithm diff --git a/cub/cub/device/dispatch/tuning/tuning_transform.cuh b/cub/cub/device/dispatch/tuning/tuning_transform.cuh index 29fdad5fb42e..62f4f970d69b 100644 --- a/cub/cub/device/dispatch/tuning/tuning_transform.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_transform.cuh @@ -70,7 +70,7 @@ namespace detail #endif // _CCCL_HOSTED() #if _CCCL_HOSTED() -inline ::std::ostream& operator<<(::std::ostream& os, const TransformAlgorithm& algo) +_CCCL_HOST_API inline ::std::ostream& operator<<(::std::ostream& os, const TransformAlgorithm& algo) { return os << _CUB::detail::to_string(algo); } @@ -83,7 +83,7 @@ template <::cuda::std::same_as CharT> struct std::formatter<_CUB::TransformAlgorithm, CharT> : formatter { template - auto format(const _CUB::TransformAlgorithm& algo, FmtCtx& ctx) const + _CCCL_HOST_API auto format(const _CUB::TransformAlgorithm& algo, FmtCtx& ctx) const { return formatter::format(_CUB::detail::to_string(algo), ctx); } @@ -125,7 +125,7 @@ struct TransformPrefetchPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const TransformPrefetchPolicy& policy) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const TransformPrefetchPolicy& policy) { return os << "TransformPrefetchPolicy { .threads_per_block = " << policy.threads_per_block @@ -158,7 +158,7 @@ struct TransformVectorizedPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const TransformVectorizedPolicy& policy) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const TransformVectorizedPolicy& policy) { return os << "TransformVectorizedPolicy { .threads_per_block = " << policy.threads_per_block << ", .items_per_thread = " << policy.items_per_thread << ", .vec_size = " << policy.vec_size << " }"; @@ -199,7 +199,7 @@ struct TransformAsyncCopyPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const TransformAsyncCopyPolicy& policy) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const TransformAsyncCopyPolicy& policy) { return os << "TransformAsyncCopyPolicy { .threads_per_block = " << policy.threads_per_block @@ -240,7 +240,7 @@ struct TransformPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const TransformPolicy& policy) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const TransformPolicy& policy) { return os << "TransformPolicy { .min_bytes_in_flight = " << policy.min_bytes_in_flight << ", .algorithm = " << policy.algorithm << ", .prefetch = " << policy.prefetch @@ -257,7 +257,7 @@ concept transform_policy_selector = policy_selector; #endif // _CCCL_HAS_CONCEPTS() template -_CCCL_HOST_DEVICE constexpr auto loaded_bytes_per_iteration() -> int +_CCCL_HOST_DEVICE_API constexpr auto loaded_bytes_per_iteration() -> int { return (int{sizeof(it_value_t)} + ... + 0); } @@ -265,7 +265,7 @@ _CCCL_HOST_DEVICE constexpr auto loaded_bytes_per_iteration() -> int inline constexpr int ldgsts_size_and_align = 16; template -_CCCL_HOST_DEVICE constexpr auto memcpy_async_dyn_smem_for_tile_size( +_CCCL_HOST_DEVICE_API constexpr auto memcpy_async_dyn_smem_for_tile_size( const ::cuda::std::array& inputs, int tile_size, int copy_alignment = ldgsts_size_and_align) -> int @@ -285,13 +285,13 @@ _CCCL_HOST_DEVICE constexpr auto memcpy_async_dyn_smem_for_tile_size( inline constexpr int bulk_copy_size_multiple = 16; -_CCCL_HOST_DEVICE constexpr auto bulk_copy_alignment(::cuda::compute_capability cc) -> int +_CCCL_HOST_DEVICE_API constexpr auto bulk_copy_alignment(::cuda::compute_capability cc) -> int { return (cc < ::cuda::compute_capability{10, 0}) ? 128 : 16; } template -_CCCL_HOST_DEVICE constexpr auto bulk_copy_dyn_smem_for_tile_size( +_CCCL_HOST_DEVICE_API constexpr auto bulk_copy_dyn_smem_for_tile_size( const ::cuda::std::array& inputs, int tile_size, int bulk_copy_align) -> int { // we rely on the tile_size being a multiple of alignments, so shifting offsets/pointers by it retains alignments diff --git a/cub/cub/device/dispatch/tuning/tuning_unique_by_key.cuh b/cub/cub/device/dispatch/tuning/tuning_unique_by_key.cuh index 1987f6d16a58..548a88f0aa5a 100644 --- a/cub/cub/device/dispatch/tuning/tuning_unique_by_key.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_unique_by_key.cuh @@ -55,7 +55,7 @@ struct UniqueByKeyPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const UniqueByKeyPolicy& p) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const UniqueByKeyPolicy& p) { return os << "UniqueByKeyPolicy { .threads_per_block = " << p.threads_per_block << ", .items_per_thread = " @@ -97,19 +97,19 @@ enum class val_size }; template -_CCCL_HOST_DEVICE constexpr primitive_key is_primitive_key() +_CCCL_HOST_DEVICE_API constexpr primitive_key is_primitive_key() { return is_primitive::value ? primitive_key::yes : primitive_key::no; } template -_CCCL_HOST_DEVICE constexpr primitive_val is_primitive_val() +_CCCL_HOST_DEVICE_API constexpr primitive_val is_primitive_val() { return is_primitive::value ? primitive_val::yes : primitive_val::no; } template -_CCCL_HOST_DEVICE constexpr key_size classify_key_size() +_CCCL_HOST_DEVICE_API constexpr key_size classify_key_size() { return sizeof(KeyT) == 1 ? key_size::_1 : sizeof(KeyT) == 2 ? key_size::_2 @@ -121,7 +121,7 @@ _CCCL_HOST_DEVICE constexpr key_size classify_key_size() } template -_CCCL_HOST_DEVICE constexpr val_size classify_val_size() +_CCCL_HOST_DEVICE_API constexpr val_size classify_val_size() { return sizeof(ValueT) == 1 ? val_size::_1 : sizeof(ValueT) == 2 ? val_size::_2 diff --git a/cub/cub/grid/grid_even_share.cuh b/cub/cub/grid/grid_even_share.cuh index c64679954eca..9055e74e8754 100644 --- a/cub/cub/grid/grid_even_share.cuh +++ b/cub/cub/grid/grid_even_share.cuh @@ -83,7 +83,7 @@ public: /** * \brief Constructor. */ - _CCCL_FORCEINLINE GridEvenShare() = default; + _CCCL_HIDE_FROM_ABI GridEvenShare() = default; /** * @brief Dispatch initializer. To be called prior to kernel launch. @@ -98,7 +98,7 @@ public: * @param tile_items * Number of data items per input tile */ - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void DispatchInit(OffsetT num_items_, int max_grid_size, int tile_items) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE void DispatchInit(OffsetT num_items_, int max_grid_size, int tile_items) { if (num_items_ <= 0 || max_grid_size <= 0 || tile_items <= 0) { @@ -129,7 +129,7 @@ public: * consecutive sequence of input tiles. */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void BlockInit(int block_id, detail::constant_t /*strategy_tag*/) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void BlockInit(int block_id, detail::constant_t /*strategy_tag*/) { block_stride = TileItems; if (block_id < big_shares) @@ -154,7 +154,7 @@ public: * of input tiles. */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void BlockInit(int block_id, detail::constant_t /*strategy_tag*/) { block_stride = grid_size * OffsetT{TileItems}; @@ -168,7 +168,7 @@ public: * separated by a stride equal to the the extent of the grid. */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void BlockInit() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void BlockInit() { BlockInit(blockIdx.x, detail::constant_v); } @@ -185,7 +185,7 @@ public: * Threadblock end offset (exclusive) */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void BlockInit(OffsetT1 block_offset, OffsetT1 block_end) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void BlockInit(OffsetT1 block_offset, OffsetT1 block_end) { this->block_offset = block_offset; this->block_end = block_end; diff --git a/cub/cub/grid/grid_queue.cuh b/cub/cub/grid/grid_queue.cuh index 1cebf23a3844..6a43154b2c3e 100644 --- a/cub/cub/grid/grid_queue.cuh +++ b/cub/cub/grid/grid_queue.cuh @@ -73,13 +73,13 @@ class GridQueue public: /// Returns the device allocation size in bytes needed to construct a GridQueue instance - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE static size_t AllocationSize() + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE static size_t AllocationSize() { return sizeof(OffsetT) * 2; } /// Constructs an invalid GridQueue descriptor - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE GridQueue() + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE GridQueue() : d_counters(nullptr) {} @@ -90,14 +90,14 @@ public: * Device allocation to back the GridQueue. Must be at least as big as * AllocationSize(). */ - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE GridQueue(void* d_storage) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE GridQueue(void* d_storage) : d_counters((OffsetT*) d_storage) {} /// This operation sets the fill-size and resets the drain counter, preparing the GridQueue for /// draining in the next kernel instance. To be called by the host or by a kernel prior to the one /// which will be draining. - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE cudaError_t + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE cudaError_t FillAndResetDrain(OffsetT fill_size, [[maybe_unused]] cudaStream_t stream = nullptr) { cudaError_t result = cudaErrorUnknown; @@ -121,7 +121,7 @@ public: /// This operation resets the drain so that it may advance to meet the existing fill-size. /// To be called by the host or by a kernel prior to the one which will be draining. - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE cudaError_t ResetDrain([[maybe_unused]] cudaStream_t stream = nullptr) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE cudaError_t ResetDrain([[maybe_unused]] cudaStream_t stream = nullptr) { cudaError_t result = cudaErrorUnknown; @@ -137,7 +137,7 @@ public: /// This operation resets the fill counter. /// To be called by the host or by a kernel prior to the one which will be filling. - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE cudaError_t ResetFill([[maybe_unused]] cudaStream_t stream = nullptr) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE cudaError_t ResetFill([[maybe_unused]] cudaStream_t stream = nullptr) { cudaError_t result = cudaErrorUnknown; @@ -152,7 +152,7 @@ public: } /// Returns the fill-size established by the parent or by the previous kernel. - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE cudaError_t + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE cudaError_t FillSize(OffsetT& fill_size, [[maybe_unused]] cudaStream_t stream = nullptr) { cudaError_t result = cudaErrorUnknown; @@ -173,14 +173,14 @@ public: /// Drain @p num_items from the queue. Returns offset from which to read items. /// To be called from CUDA kernel. - _CCCL_DEVICE _CCCL_FORCEINLINE OffsetT Drain(OffsetT num_items) + _CCCL_DEVICE_API _CCCL_FORCEINLINE OffsetT Drain(OffsetT num_items) { return atomicAdd(d_counters + DRAIN, num_items); } /// Fill @p num_items into the queue. Returns offset from which to write items. /// To be called from CUDA kernel. - _CCCL_DEVICE _CCCL_FORCEINLINE OffsetT Fill(OffsetT num_items) + _CCCL_DEVICE_API _CCCL_FORCEINLINE OffsetT Fill(OffsetT num_items) { return atomicAdd(d_counters + FILL, num_items); } diff --git a/cub/cub/iterator/arg_index_input_iterator.cuh b/cub/cub/iterator/arg_index_input_iterator.cuh index 3ba342e51757..1caf31ee1c5e 100644 --- a/cub/cub/iterator/arg_index_input_iterator.cuh +++ b/cub/cub/iterator/arg_index_input_iterator.cuh @@ -126,13 +126,13 @@ public: * @param offset * OffsetT (in items) from @p itr denoting the position of the iterator */ - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE ArgIndexInputIterator(InputIteratorT itr, difference_type offset = 0) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE ArgIndexInputIterator(InputIteratorT itr, difference_type offset = 0) : itr(itr) , offset(offset) {} /// Postfix increment - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE self_type operator++(int) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE self_type operator++(int) { self_type retval = *this; offset++; @@ -140,14 +140,14 @@ public: } /// Prefix increment - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE self_type operator++() + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE self_type operator++() { offset++; return *this; } /// Indirection - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE reference operator*() const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE reference operator*() const { value_type retval; retval.value = itr[offset]; @@ -157,7 +157,7 @@ public: /// Addition template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE self_type operator+(Distance n) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE self_type operator+(Distance n) const { self_type retval(itr, offset + n); return retval; @@ -165,7 +165,7 @@ public: /// Addition assignment template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE self_type& operator+=(Distance n) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE self_type& operator+=(Distance n) { offset += n; return *this; @@ -173,7 +173,7 @@ public: /// Subtraction template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE self_type operator-(Distance n) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE self_type operator-(Distance n) const { self_type retval(itr, offset - n); return retval; @@ -181,53 +181,53 @@ public: /// Subtraction assignment template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE self_type& operator-=(Distance n) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE self_type& operator-=(Distance n) { offset -= n; return *this; } /// Distance - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE difference_type operator-(self_type other) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE difference_type operator-(self_type other) const { return offset - other.offset; } /// Array subscript template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE reference operator[](Distance n) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE reference operator[](Distance n) const { const self_type offset = (*this) + n; return *offset; } /// Structure dereference - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE pointer operator->() + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE pointer operator->() { return &(*(*this)); } /// Equal to - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE bool operator==(const self_type& rhs) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE bool operator==(const self_type& rhs) { return ((itr == rhs.itr) && (offset == rhs.offset)); } /// Not equal to - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE bool operator!=(const self_type& rhs) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE bool operator!=(const self_type& rhs) { return ((itr != rhs.itr) || (offset != rhs.offset)); } /// Normalize - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void normalize() + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE void normalize() { itr += offset; offset = 0; } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const self_type& /*itr*/) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const self_type& /*itr*/) { return os; } diff --git a/cub/cub/iterator/cache_modified_input_iterator.cuh b/cub/cub/iterator/cache_modified_input_iterator.cuh index a5de3741d57e..a401e773a0da 100644 --- a/cub/cub/iterator/cache_modified_input_iterator.cuh +++ b/cub/cub/iterator/cache_modified_input_iterator.cuh @@ -124,12 +124,13 @@ public: /// Constructor template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE CacheModifiedInputIterator(QualifiedValueType* ptr) ///< Native pointer to wrap + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE CacheModifiedInputIterator(QualifiedValueType* ptr) ///< Native pointer to + ///< wrap : ptr(const_cast<::cuda::std::remove_cv_t*>(ptr)) {} /// Postfix increment - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE self_type operator++(int) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE self_type operator++(int) { self_type retval = *this; ptr++; @@ -137,21 +138,21 @@ public: } /// Prefix increment - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE self_type operator++() + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE self_type operator++() { ptr++; return *this; } /// Indirection - _CCCL_DEVICE _CCCL_FORCEINLINE reference operator*() const + _CCCL_DEVICE_API _CCCL_FORCEINLINE reference operator*() const { return _CUB::ThreadLoad(ptr); } /// Addition template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE self_type operator+(Distance n) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE self_type operator+(Distance n) const { self_type retval(ptr + n); return retval; @@ -159,7 +160,7 @@ public: /// Addition assignment template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE self_type& operator+=(Distance n) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE self_type& operator+=(Distance n) { ptr += n; return *this; @@ -167,7 +168,7 @@ public: /// Subtraction template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE self_type operator-(Distance n) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE self_type operator-(Distance n) const { self_type retval(ptr - n); return retval; @@ -175,46 +176,46 @@ public: /// Subtraction assignment template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE self_type& operator-=(Distance n) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE self_type& operator-=(Distance n) { ptr -= n; return *this; } /// Distance - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE difference_type operator-(self_type other) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE difference_type operator-(self_type other) const { return ptr - other.ptr; } /// Array subscript template - _CCCL_DEVICE _CCCL_FORCEINLINE reference operator[](Distance n) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE reference operator[](Distance n) const { return _CUB::ThreadLoad(ptr + n); } /// Structure dereference - _CCCL_DEVICE _CCCL_FORCEINLINE pointer operator->() + _CCCL_DEVICE_API _CCCL_FORCEINLINE pointer operator->() { return &_CUB::ThreadLoad(ptr); } /// Equal to - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE bool operator==(const self_type& rhs) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE bool operator==(const self_type& rhs) const { return (ptr == rhs.ptr); } /// Not equal to - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE bool operator!=(const self_type& rhs) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE bool operator!=(const self_type& rhs) const { return (ptr != rhs.ptr); } /// ostream operator #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const self_type& /*itr*/) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const self_type& /*itr*/) { return os; } @@ -230,7 +231,7 @@ template inline constexpr bool is_CacheModifiedInputIterator> = true; template -_CCCL_HOST_DEVICE _CCCL_FORCEINLINE auto try_make_cache_modified_iterator(Iterator it) +_CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE auto try_make_cache_modified_iterator(Iterator it) { if constexpr (THRUST_NS_QUALIFIER::is_contiguous_iterator_v) { diff --git a/cub/cub/iterator/cache_modified_output_iterator.cuh b/cub/cub/iterator/cache_modified_output_iterator.cuh index 8f194d5a8db2..ee2d4275eab3 100644 --- a/cub/cub/iterator/cache_modified_output_iterator.cuh +++ b/cub/cub/iterator/cache_modified_output_iterator.cuh @@ -91,12 +91,12 @@ class CacheModifiedOutputIterator ValueType* ptr; /// Constructor - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE Reference(ValueType* ptr) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE Reference(ValueType* ptr) : ptr(ptr) {} /// Assignment - _CCCL_DEVICE _CCCL_FORCEINLINE ValueType operator=(ValueType val) + _CCCL_DEVICE_API _CCCL_FORCEINLINE ValueType operator=(ValueType val) { _CUB::ThreadStore(ptr, val); return val; @@ -135,12 +135,12 @@ public: * Native pointer to wrap */ template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE CacheModifiedOutputIterator(QualifiedValueType* ptr) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE CacheModifiedOutputIterator(QualifiedValueType* ptr) : ptr(const_cast<::cuda::std::remove_cv_t*>(ptr)) {} /// Postfix increment - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE self_type operator++(int) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE self_type operator++(int) { self_type retval = *this; ptr++; @@ -148,21 +148,21 @@ public: } /// Prefix increment - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE self_type operator++() + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE self_type operator++() { ptr++; return *this; } /// Indirection - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE reference operator*() const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE reference operator*() const { return Reference(ptr); } /// Addition template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE self_type operator+(Distance n) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE self_type operator+(Distance n) const { self_type retval(ptr + n); return retval; @@ -170,7 +170,7 @@ public: /// Addition assignment template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE self_type& operator+=(Distance n) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE self_type& operator+=(Distance n) { ptr += n; return *this; @@ -178,7 +178,7 @@ public: /// Subtraction template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE self_type operator-(Distance n) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE self_type operator-(Distance n) const { self_type retval(ptr - n); return retval; @@ -186,40 +186,40 @@ public: /// Subtraction assignment template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE self_type& operator-=(Distance n) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE self_type& operator-=(Distance n) { ptr -= n; return *this; } /// Distance - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE difference_type operator-(self_type other) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE difference_type operator-(self_type other) const { return ptr - other.ptr; } /// Array subscript template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE reference operator[](Distance n) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE reference operator[](Distance n) const { return Reference(ptr + n); } /// Equal to - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE bool operator==(const self_type& rhs) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE bool operator==(const self_type& rhs) { return (ptr == rhs.ptr); } /// Not equal to - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE bool operator!=(const self_type& rhs) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE bool operator!=(const self_type& rhs) { return (ptr != rhs.ptr); } #if _CCCL_HOSTED() /// ostream operator - friend ::std::ostream& operator<<(::std::ostream& os, const self_type& itr) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const self_type& itr) { return os; } diff --git a/cub/cub/iterator/tex_obj_input_iterator.cuh b/cub/cub/iterator/tex_obj_input_iterator.cuh index 8c7866de3dbe..b608e9c03aa5 100644 --- a/cub/cub/iterator/tex_obj_input_iterator.cuh +++ b/cub/cub/iterator/tex_obj_input_iterator.cuh @@ -130,7 +130,7 @@ private: public: /// Constructor - _CCCL_FORCEINLINE TexObjInputIterator() = default; + _CCCL_HIDE_FROM_ABI TexObjInputIterator() = default; #if _CCCL_HOSTED() /** @@ -146,7 +146,7 @@ public: * OffsetT (in items) from @p ptr denoting the position of the iterator */ template - cudaError_t BindTexture(QualifiedT* ptr, size_t bytes, size_t tex_offset = 0) + _CCCL_HOST_API cudaError_t BindTexture(QualifiedT* ptr, size_t bytes, size_t tex_offset = 0) { this->ptr = const_cast<::cuda::std::remove_cv_t*>(ptr); this->tex_offset = static_cast(tex_offset); @@ -163,14 +163,14 @@ public: } /// Unbind this iterator from its texture reference - cudaError_t UnbindTexture() + _CCCL_HOST_API cudaError_t UnbindTexture() { return CubDebug(cudaDestroyTextureObject(tex_obj)); } #endif // _CCCL_HOSTED() /// Postfix increment - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE self_type operator++(int) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE self_type operator++(int) { self_type retval = *this; tex_offset++; @@ -178,21 +178,21 @@ public: } /// Prefix increment - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE self_type operator++() + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE self_type operator++() { tex_offset++; return *this; } /// Indirection - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE reference operator*() const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE reference operator*() const { NV_IF_ELSE_TARGET(NV_IS_HOST, (return ptr[tex_offset];), (return this->device_deref();)); } /// Addition template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE self_type operator+(Distance n) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE self_type operator+(Distance n) const { self_type retval; retval.ptr = ptr; @@ -203,7 +203,7 @@ public: /// Addition assignment template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE self_type& operator+=(Distance n) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE self_type& operator+=(Distance n) { tex_offset += n; return *this; @@ -211,7 +211,7 @@ public: /// Subtraction template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE self_type operator-(Distance n) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE self_type operator-(Distance n) const { self_type retval; retval.ptr = ptr; @@ -222,47 +222,47 @@ public: /// Subtraction assignment template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE self_type& operator-=(Distance n) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE self_type& operator-=(Distance n) { tex_offset -= n; return *this; } /// Distance - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE difference_type operator-(self_type other) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE difference_type operator-(self_type other) const { return tex_offset - other.tex_offset; } /// Array subscript template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE reference operator[](Distance n) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE reference operator[](Distance n) const { const self_type offset = (*this) + n; return *offset; } /// Structure dereference - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE pointer operator->() + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE pointer operator->() { return &(*(*this)); } /// Equal to - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE bool operator==(const self_type& rhs) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE bool operator==(const self_type& rhs) const { return ((ptr == rhs.ptr) && (tex_offset == rhs.tex_offset) && (tex_obj == rhs.tex_obj)); } /// Not equal to - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE bool operator!=(const self_type& rhs) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE bool operator!=(const self_type& rhs) const { return ((ptr != rhs.ptr) || (tex_offset != rhs.tex_offset) || (tex_obj != rhs.tex_obj)); } #if _CCCL_HOSTED() /// ostream operator - friend ::std::ostream& operator<<(::std::ostream& os, const self_type& itr) + _CCCL_HOST_API friend ::std::ostream& operator<<(::std::ostream& os, const self_type& itr) { os << "cub::TexObjInputIterator( ptr=" << itr.ptr << ", offset=" << itr.tex_offset << ", tex_obj=" << itr.tex_obj << " )"; @@ -273,7 +273,7 @@ public: private: // This is hoisted out of operator* because #pragma can't be used inside of // NV_IF_TARGET - _CCCL_DEVICE _CCCL_FORCEINLINE reference device_deref() const + _CCCL_DEVICE_API _CCCL_FORCEINLINE reference device_deref() const { // Move array of uninitialized words, then alias and assign to return // value diff --git a/cub/cub/thread/thread_load.cuh b/cub/cub/thread/thread_load.cuh index 5ecdec5b30db..11f6cf85ec52 100644 --- a/cub/cub/thread/thread_load.cuh +++ b/cub/cub/thread/thread_load.cuh @@ -85,7 +85,7 @@ namespace detail #endif // _CCCL_HOSTED() #if _CCCL_HOSTED() && !defined(_CCCL_DOXYGEN_INVOKED) -inline ::std::ostream& operator<<(::std::ostream& os, CacheLoadModifier modifier) +_CCCL_HOST_API inline ::std::ostream& operator<<(::std::ostream& os, CacheLoadModifier modifier) { return os << _CUB::detail::to_string(modifier); } @@ -98,7 +98,7 @@ template <::cuda::std::same_as CharT> struct std::formatter<_CUB::CacheLoadModifier, CharT> : formatter { template - auto format(const _CUB::CacheLoadModifier& modifier, FmtCtx& ctx) const + _CCCL_HOST_API auto format(const _CUB::CacheLoadModifier& modifier, FmtCtx& ctx) const { return formatter::format(_CUB::detail::to_string(modifier), ctx); } @@ -143,7 +143,7 @@ CUB_NAMESPACE_BEGIN //! @tparam RandomAccessIterator //! [inferred] The input's iterator type \iterator template -_CCCL_DEVICE _CCCL_FORCEINLINE cub::detail::it_value_t ThreadLoad(RandomAccessIterator itr); +_CCCL_DEVICE_API _CCCL_FORCEINLINE cub::detail::it_value_t ThreadLoad(RandomAccessIterator itr); //@} @@ -152,14 +152,14 @@ _CCCL_DEVICE _CCCL_FORCEINLINE cub::detail::it_value_t Thr namespace detail { template -_CCCL_DEVICE _CCCL_FORCEINLINE void +_CCCL_DEVICE_API _CCCL_FORCEINLINE void UnrolledThreadLoadImpl(T const* src, T* dst, ::cuda::std::integer_sequence) { ((dst[Is] = _CUB::ThreadLoad(src + Is)), ...); } template -_CCCL_DEVICE _CCCL_FORCEINLINE void +_CCCL_DEVICE_API _CCCL_FORCEINLINE void UnrolledCopyImpl(RandomAccessIterator src, T* dst, ::cuda::std::integer_sequence) { ((dst[Is] = src[Is]), ...); @@ -167,40 +167,40 @@ UnrolledCopyImpl(RandomAccessIterator src, T* dst, ::cuda::std::integer_sequence } // namespace detail template -_CCCL_DEVICE _CCCL_FORCEINLINE void UnrolledThreadLoad(T const* src, T* dst) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void UnrolledThreadLoad(T const* src, T* dst) { _CUB::detail::UnrolledThreadLoadImpl(src, dst, ::cuda::std::make_integer_sequence{}); } template -_CCCL_DEVICE _CCCL_FORCEINLINE void UnrolledCopy(RandomAccessIterator src, T* dst) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void UnrolledCopy(RandomAccessIterator src, T* dst) { _CUB::detail::UnrolledCopyImpl(src, dst, ::cuda::std::make_integer_sequence{}); } //! Define a uint4 (16B) ThreadLoad specialization for the given Cache load modifier -# define _CUB_LOAD_16(cub_modifier, ptx_modifier) \ - template <> \ - _CCCL_DEVICE _CCCL_FORCEINLINE uint4 ThreadLoad(uint4 const* ptr) \ - { \ - uint4 retval; \ - asm volatile("ld." #ptx_modifier ".v4.u32 {%0, %1, %2, %3}, [%4];" \ - : "=r"(retval.x), "=r"(retval.y), "=r"(retval.z), "=r"(retval.w) \ - : "l"(ptr)); \ - return retval; \ - } \ - template <> \ - _CCCL_DEVICE _CCCL_FORCEINLINE ulonglong2 ThreadLoad(ulonglong2 const* ptr) \ - { \ - ulonglong2 retval; \ - asm volatile("ld." #ptx_modifier ".v2.u64 {%0, %1}, [%2];" : "=l"(retval.x), "=l"(retval.y) : "l"(ptr)); \ - return retval; \ +# define _CUB_LOAD_16(cub_modifier, ptx_modifier) \ + template <> \ + _CCCL_DEVICE_API _CCCL_FORCEINLINE uint4 ThreadLoad(uint4 const* ptr) \ + { \ + uint4 retval; \ + asm volatile("ld." #ptx_modifier ".v4.u32 {%0, %1, %2, %3}, [%4];" \ + : "=r"(retval.x), "=r"(retval.y), "=r"(retval.z), "=r"(retval.w) \ + : "l"(ptr)); \ + return retval; \ + } \ + template <> \ + _CCCL_DEVICE_API _CCCL_FORCEINLINE ulonglong2 ThreadLoad(ulonglong2 const* ptr) \ + { \ + ulonglong2 retval; \ + asm volatile("ld." #ptx_modifier ".v2.u64 {%0, %1}, [%2];" : "=l"(retval.x), "=l"(retval.y) : "l"(ptr)); \ + return retval; \ } //! Define a uint2 (8B) ThreadLoad specialization for the given Cache load modifier # define _CUB_LOAD_8(cub_modifier, ptx_modifier) \ template <> \ - _CCCL_DEVICE _CCCL_FORCEINLINE ushort4 ThreadLoad(ushort4 const* ptr) \ + _CCCL_DEVICE_API _CCCL_FORCEINLINE ushort4 ThreadLoad(ushort4 const* ptr) \ { \ ushort4 retval; \ asm volatile("ld." #ptx_modifier ".v4.u16 {%0, %1, %2, %3}, [%4];" \ @@ -209,14 +209,14 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void UnrolledCopy(RandomAccessIterator src, T* ds return retval; \ } \ template <> \ - _CCCL_DEVICE _CCCL_FORCEINLINE uint2 ThreadLoad(uint2 const* ptr) \ + _CCCL_DEVICE_API _CCCL_FORCEINLINE uint2 ThreadLoad(uint2 const* ptr) \ { \ uint2 retval; \ asm volatile("ld." #ptx_modifier ".v2.u32 {%0, %1}, [%2];" : "=r"(retval.x), "=r"(retval.y) : "l"(ptr)); \ return retval; \ } \ template <> \ - _CCCL_DEVICE _CCCL_FORCEINLINE unsigned long long ThreadLoad( \ + _CCCL_DEVICE_API _CCCL_FORCEINLINE unsigned long long ThreadLoad( \ unsigned long long const* ptr) \ { \ unsigned long long retval; \ @@ -225,42 +225,43 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void UnrolledCopy(RandomAccessIterator src, T* ds } //! Define a uint (4B) ThreadLoad specialization for the given Cache load modifier -# define _CUB_LOAD_4(cub_modifier, ptx_modifier) \ - template <> \ - _CCCL_DEVICE _CCCL_FORCEINLINE unsigned int ThreadLoad(unsigned int const* ptr) \ - { \ - unsigned int retval; \ - asm volatile("ld." #ptx_modifier ".u32 %0, [%1];" : "=r"(retval) : "l"(ptr)); \ - return retval; \ - } - -//! Define a unsigned short (2B) ThreadLoad specialization for the given Cache load modifier -# define _CUB_LOAD_2(cub_modifier, ptx_modifier) \ +# define _CUB_LOAD_4(cub_modifier, ptx_modifier) \ template <> \ - _CCCL_DEVICE _CCCL_FORCEINLINE unsigned short ThreadLoad( \ - unsigned short const* ptr) \ + _CCCL_DEVICE_API _CCCL_FORCEINLINE unsigned int ThreadLoad( \ + unsigned int const* ptr) \ { \ - unsigned short retval; \ - asm volatile("ld." #ptx_modifier ".u16 %0, [%1];" : "=h"(retval) : "l"(ptr)); \ + unsigned int retval; \ + asm volatile("ld." #ptx_modifier ".u32 %0, [%1];" : "=r"(retval) : "l"(ptr)); \ return retval; \ } +//! Define a unsigned short (2B) ThreadLoad specialization for the given Cache load modifier +# define _CUB_LOAD_2(cub_modifier, ptx_modifier) \ + template <> \ + _CCCL_DEVICE_API _CCCL_FORCEINLINE unsigned short ThreadLoad( \ + unsigned short const* ptr) \ + { \ + unsigned short retval; \ + asm volatile("ld." #ptx_modifier ".u16 %0, [%1];" : "=h"(retval) : "l"(ptr)); \ + return retval; \ + } + //! Define an unsigned char (1B) ThreadLoad specialization for the given Cache load modifier -# define _CUB_LOAD_1(cub_modifier, ptx_modifier) \ - template <> \ - _CCCL_DEVICE _CCCL_FORCEINLINE unsigned char ThreadLoad( \ - unsigned char const* ptr) \ - { \ - unsigned short retval; \ - asm volatile( \ - "{" \ - " .reg .u8 datum;" \ - " ld." #ptx_modifier ".u8 datum, [%1];" \ - " cvt.u16.u8 %0, datum;" \ - "}" \ - : "=h"(retval) \ - : "l"(ptr)); \ - return (unsigned char) retval; \ +# define _CUB_LOAD_1(cub_modifier, ptx_modifier) \ + template <> \ + _CCCL_DEVICE_API _CCCL_FORCEINLINE unsigned char ThreadLoad( \ + unsigned char const* ptr) \ + { \ + unsigned short retval; \ + asm volatile( \ + "{" \ + " .reg .u8 datum;" \ + " ld." #ptx_modifier ".u8 datum, [%1];" \ + " cvt.u16.u8 %0, datum;" \ + "}" \ + : "=h"(retval) \ + : "l"(ptr)); \ + return (unsigned char) retval; \ } //! Define powers-of-two ThreadLoad specializations for the given Cache load modifier @@ -322,7 +323,8 @@ using thread_load_word_t = typename UnitWord::DeviceWord; //! ThreadLoad definition for LOAD_DEFAULT modifier on iterator types //! deprecated [Since 3.3] template -CCCL_DEPRECATED_BECAUSE("Use *itr instead") _CCCL_DEVICE _CCCL_FORCEINLINE cub::detail::it_value_t +CCCL_DEPRECATED_BECAUSE("Use *itr instead") _CCCL_DEVICE_API +_CCCL_FORCEINLINE cub::detail::it_value_t ThreadLoad(RandomAccessIterator itr, detail::constant_t /*modifier*/, ::cuda::std::false_type /*is_pointer*/) @@ -333,7 +335,7 @@ ThreadLoad(RandomAccessIterator itr, //! ThreadLoad definition for LOAD_DEFAULT modifier on pointer types //! deprecated [Since 3.3] template -CCCL_DEPRECATED_BECAUSE("Use *itr instead") _CCCL_DEVICE _CCCL_FORCEINLINE T +CCCL_DEPRECATED_BECAUSE("Use *itr instead") _CCCL_DEVICE_API _CCCL_FORCEINLINE T ThreadLoad(const T* ptr, detail::constant_t /*modifier*/, ::cuda::std::true_type /*is_pointer*/) { return *ptr; @@ -342,7 +344,7 @@ ThreadLoad(const T* ptr, detail::constant_t /*modifier*/, ::cuda:: //! ThreadLoad definition for LOAD_VOLATILE modifier on primitive pointer types //! deprecated [Since 3.3] template -CCCL_DEPRECATED_BECAUSE("Use ThreadLoad(ptr) instead") _CCCL_DEVICE _CCCL_FORCEINLINE T +CCCL_DEPRECATED_BECAUSE("Use ThreadLoad(ptr) instead") _CCCL_DEVICE_API _CCCL_FORCEINLINE T ThreadLoadVolatilePointer(const T* ptr, ::cuda::std::true_type /*is_primitive*/) { T retval = *reinterpret_cast(ptr); @@ -352,7 +354,7 @@ ThreadLoadVolatilePointer(const T* ptr, ::cuda::std::true_type /*is_primitive*/) //! ThreadLoad definition for LOAD_VOLATILE modifier on non-primitive pointer types //! deprecated [Since 3.3] template -CCCL_DEPRECATED_BECAUSE("Use ThreadLoad(ptr) instead") _CCCL_DEVICE _CCCL_FORCEINLINE T +CCCL_DEPRECATED_BECAUSE("Use ThreadLoad(ptr) instead") _CCCL_DEVICE_API _CCCL_FORCEINLINE T ThreadLoadVolatilePointer(const T* ptr, ::cuda::std::false_type /*is_primitive*/) { // Word type for memcpying @@ -368,7 +370,7 @@ ThreadLoadVolatilePointer(const T* ptr, ::cuda::std::false_type /*is_primitive*/ //! ThreadLoad definition for LOAD_VOLATILE modifier on pointer types //! deprecated [Since 3.3] template -CCCL_DEPRECATED_BECAUSE("Use ThreadLoad(ptr) instead") _CCCL_DEVICE _CCCL_FORCEINLINE T +CCCL_DEPRECATED_BECAUSE("Use ThreadLoad(ptr) instead") _CCCL_DEVICE_API _CCCL_FORCEINLINE T ThreadLoad(const T* ptr, detail::constant_t /*modifier*/, ::cuda::std::true_type /*is_pointer*/) { return _CUB::ThreadLoadVolatilePointer(ptr, detail::bool_constant_v>); @@ -377,7 +379,7 @@ ThreadLoad(const T* ptr, detail::constant_t /*modifier*/, ::cuda: //! ThreadLoad definition for generic modifiers on pointer types //! deprecated [Since 3.3] template -CCCL_DEPRECATED_BECAUSE("Use ThreadLoad(ptr) instead") _CCCL_DEVICE _CCCL_FORCEINLINE T +CCCL_DEPRECATED_BECAUSE("Use ThreadLoad(ptr) instead") _CCCL_DEVICE_API _CCCL_FORCEINLINE T ThreadLoad(T const* ptr, detail::constant_t /*modifier*/, ::cuda::std::true_type /*is_pointer*/) { using DeviceWord = typename UnitWord::DeviceWord; @@ -390,7 +392,7 @@ ThreadLoad(T const* ptr, detail::constant_t /*modifier*/, ::cuda::std: } template -_CCCL_DEVICE _CCCL_FORCEINLINE detail::it_value_t ThreadLoad(RandomAccessIterator itr) +_CCCL_DEVICE_API _CCCL_FORCEINLINE detail::it_value_t ThreadLoad(RandomAccessIterator itr) { using T = detail::it_value_t; if constexpr (!THRUST_NS_QUALIFIER::is_contiguous_iterator_v || MODIFIER == LOAD_DEFAULT) diff --git a/cub/cub/thread/thread_operators.cuh b/cub/cub/thread/thread_operators.cuh index 6d34442f9bde..4a1d8b471a59 100644 --- a/cub/cub/thread/thread_operators.cuh +++ b/cub/cub/thread/thread_operators.cuh @@ -42,13 +42,13 @@ struct InequalityWrapper EqualityOp op; /// Constructor - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE InequalityWrapper(EqualityOp op) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE InequalityWrapper(EqualityOp op) : op(op) {} /// Boolean inequality operator, returns `t != u` template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE bool operator()(T&& t, U&& u) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE bool operator()(T&& t, U&& u) { return !op(::cuda::std::forward(t), ::cuda::std::forward(u)); } @@ -61,7 +61,7 @@ struct ArgMax /// Boolean max operator, preferring the item having the smaller offset in /// case of ties template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE KeyValuePair + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE KeyValuePair operator()(const KeyValuePair& a, const KeyValuePair& b) const { // Mooch BUG (device reduce argmax gk110 3.2 million random fp32) @@ -85,7 +85,7 @@ struct ArgMin /// Boolean min operator, preferring the item having the smaller offset in /// case of ties template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE KeyValuePair + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE KeyValuePair operator()(const KeyValuePair& a, const KeyValuePair& b) const { // Mooch BUG (device reduce argmax gk110 3.2 million random fp32) @@ -110,7 +110,7 @@ template > struct arg_reduce_op : ValueLessThen { template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE ::cuda::std::pair + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE ::cuda::std::pair operator()(const ::cuda::std::pair& a, const ::cuda::std::pair& b) const { const auto& less = static_cast(*this); @@ -123,7 +123,7 @@ struct arg_reduce_op : ValueLessThen } template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE KeyValuePair + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE KeyValuePair operator()(const KeyValuePair& a, const KeyValuePair& b) const { const auto& less = static_cast(*this); @@ -176,7 +176,7 @@ template , bool LastMax = false> struct arg_minmax_reduce_op : CompareOpT { template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE argminmax_accum_t + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE argminmax_accum_t operator()(const argminmax_accum_t& a, const argminmax_accum_t& b) const { const auto& less = static_cast(*this); @@ -208,10 +208,10 @@ struct ScanBySegmentOp ScanOpT op; /// Constructor - _CCCL_FORCEINLINE ScanBySegmentOp() = default; + _CCCL_HIDE_FROM_ABI ScanBySegmentOp() = default; /// Constructor - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE ScanBySegmentOp(ScanOpT op) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE ScanBySegmentOp(ScanOpT op) : op(op) {} @@ -228,7 +228,8 @@ struct ScanBySegmentOp * Second partial reduction */ template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE KeyValuePairT operator()(const KeyValuePairT& first, const KeyValuePairT& second) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE KeyValuePairT + operator()(const KeyValuePairT& first, const KeyValuePairT& second) { // Both branches assign key and value before returning. // NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init) @@ -272,7 +273,7 @@ struct CastOp { /// Cast operator, returns `(B) a` template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE B operator()(A&& a) const + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE B operator()(A&& a) const { return (B)::cuda::std::forward(a); } @@ -287,13 +288,13 @@ class SwizzleScanOp public: /// Constructor - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE SwizzleScanOp(ScanOp scan_op) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE SwizzleScanOp(ScanOp scan_op) : scan_op(scan_op) {} /// Switch the scan arguments template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE T operator()(const T& a, const T& b) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE T operator()(const T& a, const T& b) { return scan_op(b, a); } @@ -323,10 +324,10 @@ struct ReduceBySegmentOp ReductionOpT op; /// Constructor - _CCCL_FORCEINLINE ReduceBySegmentOp() = default; + _CCCL_HIDE_FROM_ABI ReduceBySegmentOp() = default; /// Constructor - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE ReduceBySegmentOp(ReductionOpT op) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE ReduceBySegmentOp(ReductionOpT op) : op(op) {} @@ -343,7 +344,8 @@ struct ReduceBySegmentOp * Second partial reduction */ template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE KeyValuePairT operator()(const KeyValuePairT& first, const KeyValuePairT& second) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE KeyValuePairT + operator()(const KeyValuePairT& first, const KeyValuePairT& second) { // Both branches assign key and value before returning. // NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init) @@ -386,10 +388,10 @@ struct ReduceByKeyOp ReductionOpT op; /// Constructor - _CCCL_FORCEINLINE ReduceByKeyOp() = default; + _CCCL_HIDE_FROM_ABI ReduceByKeyOp() = default; /// Constructor - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE ReduceByKeyOp(ReductionOpT op) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE ReduceByKeyOp(ReductionOpT op) : op(op) {} @@ -400,7 +402,8 @@ struct ReduceByKeyOp * @param[in] second Second partial reduction */ template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE KeyValuePairT operator()(const KeyValuePairT& first, const KeyValuePairT& second) + _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE KeyValuePairT + operator()(const KeyValuePairT& first, const KeyValuePairT& second) { KeyValuePairT retval = second; @@ -472,7 +475,7 @@ template using generalize_operator_t = typename GeneralizeOperator::type; template -[[nodiscard]] constexpr _CCCL_DEVICE _CCCL_FORCEINLINE auto generalize_operator(Operator op) +[[nodiscard]] _CCCL_DEVICE_API constexpr _CCCL_FORCEINLINE auto generalize_operator(Operator op) { if constexpr (::cuda::__is_cuda_std_logical_or_v || ::cuda::__is_cuda_std_logical_and_v || is_cuda_minimum_maximum_v || is_cuda_std_plus_mul_v diff --git a/cub/cub/thread/thread_reduce.cuh b/cub/cub/thread/thread_reduce.cuh index 513630ce4dde..08a4a5897e99 100644 --- a/cub/cub/thread/thread_reduce.cuh +++ b/cub/cub/thread/thread_reduce.cuh @@ -147,7 +147,7 @@ template , typename AccumT = ::cuda::std::__accumulator_t> -[[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE AccumT ThreadReduce(const Input& input, ReductionOp reduction_op); +[[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE AccumT ThreadReduce(const Input& input, ReductionOp reduction_op); // forward declaration /*********************************************************************************************************************** @@ -277,7 +277,8 @@ inline constexpr bool enable_ternary_reduction_sm50_v = **********************************************************************************************************************/ template -[[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE AccumT ThreadReduceSequential(const Input& input, ReductionOp reduction_op) +[[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE AccumT +ThreadReduceSequential(const Input& input, ReductionOp reduction_op) { auto retval = static_cast(input[0]); _CCCL_PRAGMA_UNROLL_FULL() @@ -289,7 +290,8 @@ template } template -[[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE AccumT ThreadReduceBinaryTree(const Input& input, ReductionOp reduction_op) +[[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE AccumT +ThreadReduceBinaryTree(const Input& input, ReductionOp reduction_op) { constexpr auto length = static_size_v; auto array = _CUB::detail::to_array(input); @@ -306,7 +308,8 @@ template } template -[[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE AccumT ThreadReduceTernaryTree(const Input& input, ReductionOp reduction_op) +[[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE AccumT +ThreadReduceTernaryTree(const Input& input, ReductionOp reduction_op) { constexpr auto length = static_size_v; auto array = _CUB::detail::to_array(input); @@ -324,7 +327,7 @@ template } template -[[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE AccumT +[[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE AccumT ThreadReduceSequentialPartial(const Input& input, ReductionOp reduction_op, int valid_items) { auto retval = static_cast(input[0]); @@ -344,7 +347,7 @@ ThreadReduceSequentialPartial(const Input& input, ReductionOp reduction_op, int **********************************************************************************************************************/ template -_CCCL_DEVICE _CCCL_FORCEINLINE auto ThreadReduceSimd(const Input& input, ReductionOp) +_CCCL_DEVICE_API _CCCL_FORCEINLINE auto ThreadReduceSimd(const Input& input, ReductionOp) { using _CUB::detail::unsafe_bitcast; using T = ::cuda::std::iter_value_t; @@ -388,7 +391,7 @@ template , typename AccumT = ::cuda::std::__accumulator_t> -[[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE AccumT +[[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE AccumT ThreadReducePartial(const Input& input, ReductionOp reduction_op, int valid_items) { static_assert(is_fixed_size_random_access_range_v, @@ -413,7 +416,7 @@ ThreadReducePartial(const Input& input, ReductionOp reduction_op, int valid_item **********************************************************************************************************************/ template -[[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE AccumT ThreadReduce(const Input& input, ReductionOp reduction_op) +[[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE AccumT ThreadReduce(const Input& input, ReductionOp reduction_op) { using namespace _CUB::detail; static_assert(is_fixed_size_random_access_range_v, @@ -509,7 +512,7 @@ template , typename AccumT = ::cuda::std::__accumulator_t> -[[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE AccumT +[[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE AccumT ThreadReduce(const Input& input, ReductionOp reduction_op, PrefixT prefix) { using namespace _CUB::detail; diff --git a/cub/cub/thread/thread_scan.cuh b/cub/cub/thread/thread_scan.cuh index afd24503bb1d..7a6e0855cdd7 100644 --- a/cub/cub/thread/thread_scan.cuh +++ b/cub/cub/thread/thread_scan.cuh @@ -47,7 +47,7 @@ namespace detail * Binary scan operator */ template -_CCCL_DEVICE _CCCL_FORCEINLINE T ThreadScanExclusive( +_CCCL_DEVICE_API _CCCL_FORCEINLINE T ThreadScanExclusive( T inclusive, T exclusive, T* input, T* output, ScanOp scan_op, detail::constant_t /*length*/) { _CCCL_PRAGMA_UNROLL_FULL() @@ -93,7 +93,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE T ThreadScanExclusive( * (Handy for preventing thread-0 from applying a prefix.) */ template -_CCCL_DEVICE _CCCL_FORCEINLINE T +_CCCL_DEVICE_API _CCCL_FORCEINLINE T ThreadScanExclusive(T* input, T* output, ScanOp scan_op, T prefix, bool apply_prefix = true) { T inclusive = input[0]; @@ -139,7 +139,7 @@ ThreadScanExclusive(T* input, T* output, ScanOp scan_op, T prefix, bool apply_pr * (Handy for preventing thread-0 from applying a prefix.) */ template -_CCCL_DEVICE _CCCL_FORCEINLINE T +_CCCL_DEVICE_API _CCCL_FORCEINLINE T ThreadScanExclusive(T (&input)[LENGTH], T (&output)[LENGTH], ScanOp scan_op, T prefix, bool apply_prefix = true) { return _CUB::detail::ThreadScanExclusive((T*) input, (T*) output, scan_op, prefix, apply_prefix); @@ -190,7 +190,7 @@ template , typename AccumT = ::cuda::std::__accumulator_t> -_CCCL_DEVICE _CCCL_FORCEINLINE void ThreadScanExclusivePartial( +_CCCL_DEVICE_API _CCCL_FORCEINLINE void ThreadScanExclusivePartial( Input& input, Output& output, ScanOp scan_op, int valid_items, PrefixT prefix, bool apply_prefix = true) { static_assert(is_fixed_size_random_access_range_v, @@ -236,7 +236,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void ThreadScanExclusivePartial( * Binary scan operator */ template -_CCCL_DEVICE _CCCL_FORCEINLINE T +_CCCL_DEVICE_API _CCCL_FORCEINLINE T ThreadScanInclusive(T inclusive, T* input, T* output, ScanOp scan_op, detail::constant_t /*length*/) { _CCCL_PRAGMA_UNROLL_FULL() @@ -273,7 +273,7 @@ ThreadScanInclusive(T inclusive, T* input, T* output, ScanOp scan_op, detail::co * Binary scan operator */ template -_CCCL_DEVICE _CCCL_FORCEINLINE T ThreadScanInclusive(T* input, T* output, ScanOp scan_op) +_CCCL_DEVICE_API _CCCL_FORCEINLINE T ThreadScanInclusive(T* input, T* output, ScanOp scan_op) { T inclusive = input[0]; output[0] = inclusive; @@ -306,7 +306,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE T ThreadScanInclusive(T* input, T* output, ScanOp * Binary scan operator */ template -_CCCL_DEVICE _CCCL_FORCEINLINE T ThreadScanInclusive(T (&input)[LENGTH], T (&output)[LENGTH], ScanOp scan_op) +_CCCL_DEVICE_API _CCCL_FORCEINLINE T ThreadScanInclusive(T (&input)[LENGTH], T (&output)[LENGTH], ScanOp scan_op) { return _CUB::detail::ThreadScanInclusive((T*) input, (T*) output, scan_op); } @@ -343,7 +343,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE T ThreadScanInclusive(T (&input)[LENGTH], T (&out * (Handy for preventing thread-0 from applying a prefix.) */ template -_CCCL_DEVICE _CCCL_FORCEINLINE T +_CCCL_DEVICE_API _CCCL_FORCEINLINE T ThreadScanInclusive(T* input, T* output, ScanOp scan_op, T prefix, bool apply_prefix = true) { T inclusive = input[0]; @@ -389,7 +389,7 @@ ThreadScanInclusive(T* input, T* output, ScanOp scan_op, T prefix, bool apply_pr * (Handy for preventing thread-0 from applying a prefix.) */ template -_CCCL_DEVICE _CCCL_FORCEINLINE T +_CCCL_DEVICE_API _CCCL_FORCEINLINE T ThreadScanInclusive(T (&input)[LENGTH], T (&output)[LENGTH], ScanOp scan_op, T prefix, bool apply_prefix = true) { return _CUB::detail::ThreadScanInclusive((T*) input, (T*) output, scan_op, prefix, apply_prefix); @@ -440,7 +440,7 @@ template , typename AccumT = ::cuda::std::__accumulator_t> -_CCCL_DEVICE _CCCL_FORCEINLINE void ThreadScanInclusivePartial( +_CCCL_DEVICE_API _CCCL_FORCEINLINE void ThreadScanInclusivePartial( Input& input, Output& output, ScanOp scan_op, int valid_items, PrefixT prefix, bool apply_prefix = true) { static_assert(is_fixed_size_random_access_range_v, diff --git a/cub/cub/thread/thread_search.cuh b/cub/cub/thread/thread_search.cuh index 291360c00fad..eb3d3fe38a4f 100644 --- a/cub/cub/thread/thread_search.cuh +++ b/cub/cub/thread/thread_search.cuh @@ -37,7 +37,7 @@ CUB_NAMESPACE_BEGIN * Deprecated [Since 3.0] */ template -CCCL_DEPRECATED _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void MergePathSearch( +CCCL_DEPRECATED _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE void MergePathSearch( OffsetT diagonal, AIteratorT a, BIteratorT b, OffsetT a_len, OffsetT b_len, CoordinateT& path_coordinate) { /// The value type of the input iterator @@ -80,7 +80,7 @@ CCCL_DEPRECATED _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void MergePathSearch( */ // TODO(bgruber): deprecate once ::cuda::std::lower_bound is made public template -_CCCL_DEVICE _CCCL_FORCEINLINE OffsetT LowerBound(InputIteratorT input, OffsetT num_items, T val) +_CCCL_DEVICE_API _CCCL_FORCEINLINE OffsetT LowerBound(InputIteratorT input, OffsetT num_items, T val) { OffsetT retval = 0; while (num_items > 0) @@ -115,7 +115,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE OffsetT LowerBound(InputIteratorT input, OffsetT */ // TODO(bgruber): deprecate once ::cuda::std::upper_bound is made public template -_CCCL_DEVICE _CCCL_FORCEINLINE OffsetT UpperBound(InputIteratorT input, OffsetT num_items, T val) +_CCCL_DEVICE_API _CCCL_FORCEINLINE OffsetT UpperBound(InputIteratorT input, OffsetT num_items, T val) { OffsetT retval = 0; while (num_items > 0) diff --git a/cub/cub/thread/thread_simd.cuh b/cub/cub/thread/thread_simd.cuh index f8e7dae060bf..4b5763a5b1f0 100644 --- a/cub/cub/thread/thread_simd.cuh +++ b/cub/cub/thread/thread_simd.cuh @@ -60,7 +60,7 @@ struct SimdMin template <> struct SimdMin { - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE short2 operator()(short2 a, short2 b) const + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE short2 operator()(short2 a, short2 b) const { NV_IF_ELSE_TARGET( NV_PROVIDES_SM_90, @@ -74,7 +74,7 @@ struct SimdMin template <> struct SimdMin { - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE ushort2 operator()(ushort2 a, ushort2 b) const + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE ushort2 operator()(ushort2 a, ushort2 b) const { NV_IF_ELSE_TARGET( NV_PROVIDES_SM_90, @@ -90,7 +90,7 @@ struct SimdMin template <> struct SimdMin<__half2> { - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE __half2 operator()(__half2 a, __half2 b) const + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE __half2 operator()(__half2 a, __half2 b) const { return ::__hmin2(a, b); } @@ -103,7 +103,7 @@ struct SimdMin<__half2> template <> struct SimdMin<__nv_bfloat162> { - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE __nv_bfloat162 operator()(__nv_bfloat162 a, __nv_bfloat162 b) const + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE __nv_bfloat162 operator()(__nv_bfloat162 a, __nv_bfloat162 b) const { NV_IF_ELSE_TARGET(NV_PROVIDES_SM_80, (return ::__hmin2(a, b);), @@ -124,7 +124,7 @@ struct SimdMax template <> struct SimdMax { - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE short2 operator()(short2 a, short2 b) const + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE short2 operator()(short2 a, short2 b) const { NV_IF_ELSE_TARGET( NV_PROVIDES_SM_90, @@ -138,7 +138,7 @@ struct SimdMax template <> struct SimdMax { - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE ushort2 operator()(ushort2 a, ushort2 b) const + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE ushort2 operator()(ushort2 a, ushort2 b) const { NV_IF_ELSE_TARGET( NV_PROVIDES_SM_90, @@ -154,7 +154,7 @@ struct SimdMax template <> struct SimdMax<__half2> { - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE __half2 operator()(__half2 a, __half2 b) const + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE __half2 operator()(__half2 a, __half2 b) const { return ::__hmax2(a, b); } @@ -167,7 +167,7 @@ struct SimdMax<__half2> template <> struct SimdMax<__nv_bfloat162> { - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE __nv_bfloat162 operator()(__nv_bfloat162 a, __nv_bfloat162 b) const + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE __nv_bfloat162 operator()(__nv_bfloat162 a, __nv_bfloat162 b) const { NV_IF_ELSE_TARGET(NV_PROVIDES_SM_80, (return ::__hmax2(a, b);), // @@ -190,7 +190,7 @@ struct SimdSum template <> struct SimdSum<__half2> { - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE __half2 operator()(__half2 a, __half2 b) const + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE __half2 operator()(__half2 a, __half2 b) const { NV_IF_ELSE_TARGET(NV_PROVIDES_SM_53, (return ::__hadd2(a, b);), // @@ -205,7 +205,7 @@ struct SimdSum<__half2> template <> struct SimdSum<__nv_bfloat162> { - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE __nv_bfloat162 operator()(__nv_bfloat162 a, __nv_bfloat162 b) const + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE __nv_bfloat162 operator()(__nv_bfloat162 a, __nv_bfloat162 b) const { NV_IF_ELSE_TARGET(NV_PROVIDES_SM_80, (return ::__hadd2(a, b);), // @@ -218,7 +218,7 @@ struct SimdSum<__nv_bfloat162> template <> struct SimdSum { - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE ushort2 operator()(ushort2 a, ushort2 b) const + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE ushort2 operator()(ushort2 a, ushort2 b) const { // clang-format off NV_IF_ELSE_TARGET(NV_PROVIDES_SM_90, @@ -236,7 +236,7 @@ struct SimdSum template <> struct SimdSum { - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE short2 operator()(short2 a, short2 b) const + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE short2 operator()(short2 a, short2 b) const { // clang-format off NV_IF_ELSE_TARGET(NV_PROVIDES_SM_90, @@ -254,7 +254,7 @@ struct SimdSum template <> struct SimdSum { - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE float2 operator()(float2 a, float2 b) const + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE float2 operator()(float2 a, float2 b) const { // clang-format off NV_IF_ELSE_TARGET(NV_PROVIDES_SM_100, @@ -282,7 +282,7 @@ struct SimdMul template <> struct SimdMul<__half2> { - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE __half2 operator()(__half2 a, __half2 b) const + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE __half2 operator()(__half2 a, __half2 b) const { NV_IF_ELSE_TARGET(NV_PROVIDES_SM_53, (return ::__hmul2(a, b);), // @@ -297,7 +297,7 @@ struct SimdMul<__half2> template <> struct SimdMul<__nv_bfloat162> { - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE __nv_bfloat162 operator()(__nv_bfloat162 a, __nv_bfloat162 b) const + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE __nv_bfloat162 operator()(__nv_bfloat162 a, __nv_bfloat162 b) const { NV_IF_ELSE_TARGET(NV_PROVIDES_SM_80, (return ::__hmul2(a, b);), @@ -431,7 +431,7 @@ using cuda_operator_to_simd_x2_t = typename CudaOperatorToSimdX2::t //---------------------------------------------------------------------------------------------------------------------- template -_CCCL_DEVICE _CCCL_FORCEINLINE auto try_simd_operator(Op op) +_CCCL_DEVICE_API _CCCL_FORCEINLINE auto try_simd_operator(Op op) { using ::cuda::std::is_same_v; constexpr bool is_supported_vector_type = is_any_short2_v || is_bfloat162_v || is_half2_v; diff --git a/cub/cub/thread/thread_store.cuh b/cub/cub/thread/thread_store.cuh index 86f0c1a13a0d..b8ae8a3e7e21 100644 --- a/cub/cub/thread/thread_store.cuh +++ b/cub/cub/thread/thread_store.cuh @@ -87,7 +87,7 @@ enum CacheStoreModifier // NOLINT(cppcoreguidelines-use-enum-class) //! @tparam T //! [inferred] Data type of output value template -_CCCL_DEVICE _CCCL_FORCEINLINE void ThreadStore(OutputIteratorT itr, T val); +_CCCL_DEVICE_API _CCCL_FORCEINLINE void ThreadStore(OutputIteratorT itr, T val); //@} @@ -100,14 +100,14 @@ template struct iterate_thread_store { template - static _CCCL_DEVICE _CCCL_FORCEINLINE void Store(T* ptr, T* vals) + _CCCL_DEVICE_API static _CCCL_FORCEINLINE void Store(T* ptr, T* vals) { _CUB::ThreadStore(ptr + COUNT, vals[COUNT]); iterate_thread_store::template Store(ptr, vals); } template - static _CCCL_DEVICE _CCCL_FORCEINLINE void Dereference(OutputIteratorT ptr, T* vals) + _CCCL_DEVICE_API static _CCCL_FORCEINLINE void Dereference(OutputIteratorT ptr, T* vals) { ptr[COUNT] = vals[COUNT]; iterate_thread_store::Dereference(ptr, vals); @@ -119,96 +119,96 @@ template struct iterate_thread_store { template - static _CCCL_DEVICE _CCCL_FORCEINLINE void Store(T* /*ptr*/, T* /*vals*/) + _CCCL_DEVICE_API static _CCCL_FORCEINLINE void Store(T* /*ptr*/, T* /*vals*/) {} template - static _CCCL_DEVICE _CCCL_FORCEINLINE void Dereference(OutputIteratorT /*ptr*/, T* /*vals*/) + _CCCL_DEVICE_API static _CCCL_FORCEINLINE void Dereference(OutputIteratorT /*ptr*/, T* /*vals*/) {} }; template -_CCCL_DEVICE _CCCL_FORCEINLINE void store_helper(T* ptr, T* vals, ::cuda::std::index_sequence) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void store_helper(T* ptr, T* vals, ::cuda::std::index_sequence) { (_CUB::ThreadStore(ptr + Is, vals[Is]), ...); } template -_CCCL_DEVICE _CCCL_FORCEINLINE void dereference_helper(volatile T* ptr, T* vals, ::cuda::std::index_sequence) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void dereference_helper(volatile T* ptr, T* vals, ::cuda::std::index_sequence) { ((ptr[Is] = vals[Is]), ...); } } // namespace detail //! Define a uint4 (16B) ThreadStore specialization for the given Cache load modifier -# define _CUB_STORE_16(cub_modifier, ptx_modifier) \ - template <> \ - _CCCL_DEVICE _CCCL_FORCEINLINE void ThreadStore(uint4 * ptr, uint4 val) \ - { \ - asm volatile("st." #ptx_modifier ".v4.u32 [%0], {%1, %2, %3, %4};" \ - : \ - : "l"(ptr), "r"(val.x), "r"(val.y), "r"(val.z), "r"(val.w)); \ - } \ - template <> \ - _CCCL_DEVICE _CCCL_FORCEINLINE void ThreadStore( \ - ulonglong2 * ptr, ulonglong2 val) \ - { \ - asm volatile("st." #ptx_modifier ".v2.u64 [%0], {%1, %2};" : : "l"(ptr), "l"(val.x), "l"(val.y)); \ +# define _CUB_STORE_16(cub_modifier, ptx_modifier) \ + template <> \ + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ThreadStore(uint4 * ptr, uint4 val) \ + { \ + asm volatile("st." #ptx_modifier ".v4.u32 [%0], {%1, %2, %3, %4};" \ + : \ + : "l"(ptr), "r"(val.x), "r"(val.y), "r"(val.z), "r"(val.w)); \ + } \ + template <> \ + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ThreadStore( \ + ulonglong2 * ptr, ulonglong2 val) \ + { \ + asm volatile("st." #ptx_modifier ".v2.u64 [%0], {%1, %2};" : : "l"(ptr), "l"(val.x), "l"(val.y)); \ } //! Define a uint2 (8B) ThreadStore specialization for the given Cache load modifier -# define _CUB_STORE_8(cub_modifier, ptx_modifier) \ - template <> \ - _CCCL_DEVICE _CCCL_FORCEINLINE void ThreadStore(ushort4 * ptr, ushort4 val) \ - { \ - asm volatile("st." #ptx_modifier ".v4.u16 [%0], {%1, %2, %3, %4};" \ - : \ - : "l"(ptr), "h"(val.x), "h"(val.y), "h"(val.z), "h"(val.w)); \ - } \ - template <> \ - _CCCL_DEVICE _CCCL_FORCEINLINE void ThreadStore(uint2 * ptr, uint2 val) \ - { \ - asm volatile("st." #ptx_modifier ".v2.u32 [%0], {%1, %2};" : : "l"(ptr), "r"(val.x), "r"(val.y)); \ - } \ - template <> \ - _CCCL_DEVICE _CCCL_FORCEINLINE void ThreadStore( \ - unsigned long long* ptr, unsigned long long val) \ - { \ - asm volatile("st." #ptx_modifier ".u64 [%0], %1;" : : "l"(ptr), "l"(val)); \ +# define _CUB_STORE_8(cub_modifier, ptx_modifier) \ + template <> \ + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ThreadStore(ushort4 * ptr, ushort4 val) \ + { \ + asm volatile("st." #ptx_modifier ".v4.u16 [%0], {%1, %2, %3, %4};" \ + : \ + : "l"(ptr), "h"(val.x), "h"(val.y), "h"(val.z), "h"(val.w)); \ + } \ + template <> \ + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ThreadStore(uint2 * ptr, uint2 val) \ + { \ + asm volatile("st." #ptx_modifier ".v2.u32 [%0], {%1, %2};" : : "l"(ptr), "r"(val.x), "r"(val.y)); \ + } \ + template <> \ + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ThreadStore( \ + unsigned long long* ptr, unsigned long long val) \ + { \ + asm volatile("st." #ptx_modifier ".u64 [%0], %1;" : : "l"(ptr), "l"(val)); \ } //! Define a unsigned int (4B) ThreadStore specialization for the given Cache load modifier -# define _CUB_STORE_4(cub_modifier, ptx_modifier) \ - template <> \ - _CCCL_DEVICE _CCCL_FORCEINLINE void ThreadStore( \ - unsigned int* ptr, unsigned int val) \ - { \ - asm volatile("st." #ptx_modifier ".u32 [%0], %1;" : : "l"(ptr), "r"(val)); \ +# define _CUB_STORE_4(cub_modifier, ptx_modifier) \ + template <> \ + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ThreadStore( \ + unsigned int* ptr, unsigned int val) \ + { \ + asm volatile("st." #ptx_modifier ".u32 [%0], %1;" : : "l"(ptr), "r"(val)); \ } //! Define a unsigned short (2B) ThreadStore specialization for the given Cache load modifier -# define _CUB_STORE_2(cub_modifier, ptx_modifier) \ - template <> \ - _CCCL_DEVICE _CCCL_FORCEINLINE void ThreadStore( \ - unsigned short* ptr, unsigned short val) \ - { \ - asm volatile("st." #ptx_modifier ".u16 [%0], %1;" : : "l"(ptr), "h"(val)); \ +# define _CUB_STORE_2(cub_modifier, ptx_modifier) \ + template <> \ + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ThreadStore( \ + unsigned short* ptr, unsigned short val) \ + { \ + asm volatile("st." #ptx_modifier ".u16 [%0], %1;" : : "l"(ptr), "h"(val)); \ } //! Define a unsigned char (1B) ThreadStore specialization for the given Cache load modifier -# define _CUB_STORE_1(cub_modifier, ptx_modifier) \ - template <> \ - _CCCL_DEVICE _CCCL_FORCEINLINE void ThreadStore( \ - unsigned char* ptr, unsigned char val) \ - { \ - asm volatile( \ - "{" \ - " .reg .u8 datum;" \ - " cvt.u8.u16 datum, %1;" \ - " st." #ptx_modifier ".u8 [%0], datum;" \ - "}" \ - : \ - : "l"(ptr), "h"((unsigned short) val)); \ +# define _CUB_STORE_1(cub_modifier, ptx_modifier) \ + template <> \ + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ThreadStore( \ + unsigned char* ptr, unsigned char val) \ + { \ + asm volatile( \ + "{" \ + " .reg .u8 datum;" \ + " cvt.u8.u16 datum, %1;" \ + " st." #ptx_modifier ".u8 [%0], datum;" \ + "}" \ + : \ + : "l"(ptr), "h"((unsigned short) val)); \ } //! Define powers-of-two ThreadStore specializations for the given Cache load modifier @@ -238,7 +238,7 @@ _CUB_STORE_ALL(STORE_WT, wt) //! ThreadStore definition for STORE_DEFAULT modifier on iterator types //! deprecated [Since 3.3] template -CCCL_DEPRECATED_BECAUSE("Use *itr = val instead") _CCCL_DEVICE _CCCL_FORCEINLINE void ThreadStore( +CCCL_DEPRECATED_BECAUSE("Use *itr = val instead") _CCCL_DEVICE_API _CCCL_FORCEINLINE void ThreadStore( OutputIteratorT itr, T val, detail::constant_t /*modifier*/, ::cuda::std::false_type /*is_pointer*/) { *itr = val; @@ -247,7 +247,7 @@ CCCL_DEPRECATED_BECAUSE("Use *itr = val instead") _CCCL_DEVICE _CCCL_FORCEINLINE //! ThreadStore definition for STORE_DEFAULT modifier on pointer types //! deprecated [Since 3.3] template -CCCL_DEPRECATED_BECAUSE("Use *itr = val instead") _CCCL_DEVICE _CCCL_FORCEINLINE void +CCCL_DEPRECATED_BECAUSE("Use *itr = val instead") _CCCL_DEVICE_API _CCCL_FORCEINLINE void ThreadStore(T* ptr, T val, detail::constant_t /*modifier*/, ::cuda::std::true_type /*is_pointer*/) { *ptr = val; @@ -256,7 +256,7 @@ ThreadStore(T* ptr, T val, detail::constant_t /*modifier*/, ::cud //! ThreadStore definition for STORE_VOLATILE modifier on primitive pointer types //! deprecated [Since 3.3] template -CCCL_DEPRECATED_BECAUSE("Use ThreadStore(ptr, val) instead") _CCCL_DEVICE _CCCL_FORCEINLINE void +CCCL_DEPRECATED_BECAUSE("Use ThreadStore(ptr, val) instead") _CCCL_DEVICE_API _CCCL_FORCEINLINE void ThreadStoreVolatilePtr(T* ptr, T val, ::cuda::std::true_type /*is_primitive*/) { *reinterpret_cast(ptr) = val; @@ -265,7 +265,7 @@ ThreadStoreVolatilePtr(T* ptr, T val, ::cuda::std::true_type /*is_primitive*/) //! ThreadStore definition for STORE_VOLATILE modifier on non-primitive pointer types //! deprecated [Since 3.3] template -CCCL_DEPRECATED_BECAUSE("Use ThreadStore(ptr, val) instead") _CCCL_DEVICE _CCCL_FORCEINLINE void +CCCL_DEPRECATED_BECAUSE("Use ThreadStore(ptr, val) instead") _CCCL_DEVICE_API _CCCL_FORCEINLINE void ThreadStoreVolatilePtr(T* ptr, T val, ::cuda::std::false_type /*is_primitive*/) { // Create a temporary using shuffle-words, then store using volatile-words @@ -290,7 +290,7 @@ ThreadStoreVolatilePtr(T* ptr, T val, ::cuda::std::false_type /*is_primitive*/) //! ThreadStore definition for STORE_VOLATILE modifier on pointer types //! deprecated [Since 3.3] template -CCCL_DEPRECATED_BECAUSE("Use ThreadStore(ptr, val) instead") _CCCL_DEVICE _CCCL_FORCEINLINE void +CCCL_DEPRECATED_BECAUSE("Use ThreadStore(ptr, val) instead") _CCCL_DEVICE_API _CCCL_FORCEINLINE void ThreadStore(T* ptr, T val, detail::constant_t /*modifier*/, ::cuda::std::true_type /*is_pointer*/) { _CUB::ThreadStoreVolatilePtr(ptr, val, detail::bool_constant_v::value>); @@ -299,7 +299,7 @@ ThreadStore(T* ptr, T val, detail::constant_t /*modifier*/, ::cu //! ThreadStore definition for generic modifiers on pointer types //! deprecated [Since 3.3] template -CCCL_DEPRECATED_BECAUSE("Use ThreadStore(ptr, val) instead") _CCCL_DEVICE _CCCL_FORCEINLINE void +CCCL_DEPRECATED_BECAUSE("Use ThreadStore(ptr, val) instead") _CCCL_DEVICE_API _CCCL_FORCEINLINE void ThreadStore(T* ptr, T val, detail::constant_t /*modifier*/, ::cuda::std::true_type /*is_pointer*/) { // Create a temporary using shuffle-words, then store using device-words @@ -322,7 +322,7 @@ ThreadStore(T* ptr, T val, detail::constant_t /*modifier*/, ::cuda::st } template -_CCCL_DEVICE _CCCL_FORCEINLINE void ThreadStore(OutputIteratorT itr, T val) +_CCCL_DEVICE_API _CCCL_FORCEINLINE void ThreadStore(OutputIteratorT itr, T val) { if constexpr (!THRUST_NS_QUALIFIER::is_contiguous_iterator_v || MODIFIER == STORE_DEFAULT) { diff --git a/cub/cub/util_allocator.cuh b/cub/cub/util_allocator.cuh index cba619cf4848..802293315e9f 100644 --- a/cub/cub/util_allocator.cuh +++ b/cub/cub/util_allocator.cuh @@ -131,7 +131,7 @@ struct CCCL_DEPRECATED_BECAUSE("cub::CachingDeviceAllocator is deprecated; use c // Constructor (suitable for searching maps for a specific block, given its pointer and // device) - BlockDescriptor(void* d_ptr, int device) + _CCCL_HOST_API BlockDescriptor(void* d_ptr, int device) : d_ptr(d_ptr) , bytes(0) , bin(INVALID_BIN) @@ -141,7 +141,7 @@ struct CCCL_DEPRECATED_BECAUSE("cub::CachingDeviceAllocator is deprecated; use c {} // Constructor (suitable for searching maps for a range of suitable blocks, given a device) - BlockDescriptor(int device) + _CCCL_HOST_API BlockDescriptor(int device) : d_ptr(nullptr) , bytes(0) , bin(INVALID_BIN) @@ -151,7 +151,7 @@ struct CCCL_DEPRECATED_BECAUSE("cub::CachingDeviceAllocator is deprecated; use c {} // Comparison functor for comparing device pointers - static bool PtrCompare(const BlockDescriptor& a, const BlockDescriptor& b) + _CCCL_HOST_API static bool PtrCompare(const BlockDescriptor& a, const BlockDescriptor& b) { if (a.device == b.device) { @@ -162,7 +162,7 @@ struct CCCL_DEPRECATED_BECAUSE("cub::CachingDeviceAllocator is deprecated; use c } // Comparison functor for comparing allocation sizes - static bool SizeCompare(const BlockDescriptor& a, const BlockDescriptor& b) + _CCCL_HOST_API static bool SizeCompare(const BlockDescriptor& a, const BlockDescriptor& b) { if (a.device == b.device) { @@ -181,7 +181,7 @@ struct CCCL_DEPRECATED_BECAUSE("cub::CachingDeviceAllocator is deprecated; use c public: size_t free; size_t live; - TotalBytes() + _CCCL_HOST_API TotalBytes() { free = live = 0; } @@ -203,7 +203,7 @@ struct CCCL_DEPRECATED_BECAUSE("cub::CachingDeviceAllocator is deprecated; use c /** * Integer pow function for unsigned base and exponent */ - static unsigned int IntPow(unsigned int base, unsigned int exp) + _CCCL_HOST_API static unsigned int IntPow(unsigned int base, unsigned int exp) { unsigned int retval = 1; while (exp > 0) @@ -221,7 +221,7 @@ struct CCCL_DEPRECATED_BECAUSE("cub::CachingDeviceAllocator is deprecated; use c /** * Round up to the nearest power-of */ - void NearestPowerOf(unsigned int& power, size_t& rounded_bytes, unsigned int base, size_t value) + _CCCL_HOST_API void NearestPowerOf(unsigned int& power, size_t& rounded_bytes, unsigned int base, size_t value) { power = 0; rounded_bytes = 1; @@ -307,7 +307,7 @@ struct CCCL_DEPRECATED_BECAUSE("cub::CachingDeviceAllocator is deprecated; use c * Whether or not to skip a call to @p FreeAllCached() when the destructor is called (default * is to deallocate) */ - CachingDeviceAllocator( + _CCCL_HOST_API CachingDeviceAllocator( unsigned int bin_growth, unsigned int min_bin = 1, unsigned int max_bin = INVALID_BIN, @@ -338,7 +338,7 @@ struct CCCL_DEPRECATED_BECAUSE("cub::CachingDeviceAllocator is deprecated; use c * which delineates five bin-sizes: 512B, 4KB, 32KB, 256KB, and 2MB and * sets a maximum of 6,291,455 cached bytes per device */ - CachingDeviceAllocator(bool skip_cleanup = false, bool debug = false) + _CCCL_HOST_API CachingDeviceAllocator(bool skip_cleanup = false, bool debug = false) : bin_growth(8) , min_bin(3) , max_bin(7) @@ -357,7 +357,7 @@ struct CCCL_DEPRECATED_BECAUSE("cub::CachingDeviceAllocator is deprecated; use c * Changing the ceiling of cached bytes does not cause any allocations (in-use or * cached-in-reserve) to be freed. See \p FreeAllCached(). */ - cudaError_t SetMaxCachedBytes(size_t max_cached_bytes_) + _CCCL_HOST_API cudaError_t SetMaxCachedBytes(size_t max_cached_bytes_) { // Lock mutex.lock(); @@ -396,7 +396,7 @@ struct CCCL_DEPRECATED_BECAUSE("cub::CachingDeviceAllocator is deprecated; use c * @param[in] active_stream * The stream to be associated with this allocation */ - cudaError_t DeviceAllocate(int device, void** d_ptr, size_t bytes, cudaStream_t active_stream = nullptr) + _CCCL_HOST_API cudaError_t DeviceAllocate(int device, void** d_ptr, size_t bytes, cudaStream_t active_stream = nullptr) { *d_ptr = nullptr; int entrypoint_device = INVALID_DEVICE_ORDINAL; @@ -657,7 +657,7 @@ struct CCCL_DEPRECATED_BECAUSE("cub::CachingDeviceAllocator is deprecated; use c * @param[in] active_stream * The stream to be associated with this allocation */ - cudaError_t DeviceAllocate(void** d_ptr, size_t bytes, cudaStream_t active_stream = nullptr) + _CCCL_HOST_API cudaError_t DeviceAllocate(void** d_ptr, size_t bytes, cudaStream_t active_stream = nullptr) { return DeviceAllocate(INVALID_DEVICE_ORDINAL, d_ptr, bytes, active_stream); } @@ -671,7 +671,7 @@ struct CCCL_DEPRECATED_BECAUSE("cub::CachingDeviceAllocator is deprecated; use c * available for reuse within other streams when all prior work submitted to @p active_stream * has completed. */ - cudaError_t DeviceFree(int device, void* d_ptr) + _CCCL_HOST_API cudaError_t DeviceFree(int device, void* d_ptr) { int entrypoint_device = INVALID_DEVICE_ORDINAL; cudaError_t error = cudaSuccess; @@ -809,7 +809,7 @@ struct CCCL_DEPRECATED_BECAUSE("cub::CachingDeviceAllocator is deprecated; use c * for reuse within other streams when all prior work submitted to @p active_stream has * completed. */ - cudaError_t DeviceFree(void* d_ptr) + _CCCL_HOST_API cudaError_t DeviceFree(void* d_ptr) { return DeviceFree(INVALID_DEVICE_ORDINAL, d_ptr); } @@ -817,7 +817,7 @@ struct CCCL_DEPRECATED_BECAUSE("cub::CachingDeviceAllocator is deprecated; use c /** * @brief Frees all cached device allocations on all devices */ - cudaError_t FreeAllCached() + _CCCL_HOST_API cudaError_t FreeAllCached() { cudaError_t error = cudaSuccess; int entrypoint_device = INVALID_DEVICE_ORDINAL; @@ -899,7 +899,7 @@ struct CCCL_DEPRECATED_BECAUSE("cub::CachingDeviceAllocator is deprecated; use c /** * @brief Destructor */ - virtual ~CachingDeviceAllocator() + _CCCL_HOST_API virtual ~CachingDeviceAllocator() { if (!skip_cleanup) { diff --git a/cub/cub/util_debug.cuh b/cub/cub/util_debug.cuh index 363336e6e763..dc1c5cdc7a26 100644 --- a/cub/cub/util_debug.cuh +++ b/cub/cub/util_debug.cuh @@ -108,7 +108,7 @@ CUB_NAMESPACE_BEGIN * * \return The CUDA error. */ -_CCCL_HOST_DEVICE _CCCL_FORCEINLINE cudaError_t +_CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE cudaError_t Debug(cudaError_t error, [[maybe_unused]] const char* filename, [[maybe_unused]] int line) { // Clear the global CUDA error state which may have been set by the last diff --git a/cub/cub/util_device.cuh b/cub/cub/util_device.cuh index 9557b8cca661..432838f23389 100644 --- a/cub/cub/util_device.cuh +++ b/cub/cub/util_device.cuh @@ -93,7 +93,7 @@ class [[maybe_unused]] SwitchDevice public: //! @brief Queries the current device and if that is different than @p target_device sets the current device to //! @p target_device - SwitchDevice(const int target_device) + _CCCL_HOST_API SwitchDevice(const int target_device) : target_device_(target_device) , original_device_(target_device) { @@ -106,7 +106,7 @@ public: //! @brief If the @p original_device was not equal to @p target_device sets the current device back to //! @p original_device - ~SwitchDevice() + _CCCL_HOST_API ~SwitchDevice() { if (original_device_ != target_device_) { @@ -134,7 +134,7 @@ CUB_RUNTIME_FUNCTION inline int device_count_uncached() } // TODO(bgruber): remove in CCCL 4.0 -_CCCL_HOST inline int device_count_cached_value() +_CCCL_HOST_API inline int device_count_cached_value() { static const int count = _CUB::detail::device_count_uncached(); return count; @@ -166,7 +166,7 @@ CCCL_DEPRECATED_BECAUSE("Use cuda::devices.size() instead") CUB_RUNTIME_FUNCTION // TODO(bgruber): remove in CCCL 4.0 // Host code. This is a separate function to avoid defining a local static in a host/device function. -CCCL_DEPRECATED_BECAUSE("Use cuda::devices.size() instead") _CCCL_HOST inline int DeviceCountCachedValue() +CCCL_DEPRECATED_BECAUSE("Use cuda::devices.size() instead") _CCCL_HOST_API inline int DeviceCountCachedValue() { return _CUB::detail::device_count_cached_value(); } @@ -224,7 +224,7 @@ public: /** * \brief Construct the cache. */ - _CCCL_HOST PerDeviceAttributeCache() + _CCCL_HOST_API PerDeviceAttributeCache() : entries_() { _CCCL_ASSERT(_CUB::detail::device_count() <= detail::max_devices, ""); @@ -237,7 +237,7 @@ public: * this function has undefined behavior. */ template - _CCCL_HOST DevicePayload operator()(Invocable&& f, int device) + _CCCL_HOST_API DevicePayload operator()(Invocable&& f, int device) { if (device >= _CUB::detail::device_count() || device < 0) { @@ -333,7 +333,7 @@ CUB_RUNTIME_FUNCTION cudaError_t PtxVersionUncached(int& ptx_version) * \brief Retrieves the PTX version that will be used on \p device (major * 100 + minor * 10). */ template -_CCCL_HOST cudaError_t PtxVersionUncached(int& ptx_version, int device) +_CCCL_HOST_API cudaError_t PtxVersionUncached(int& ptx_version, int device) { const SwitchDevice sd(device); return _CUB::PtxVersionUncached(ptx_version); @@ -341,7 +341,7 @@ _CCCL_HOST cudaError_t PtxVersionUncached(int& ptx_version, int device) # if _CCCL_HOSTED() template -_CCCL_HOST inline PerDeviceAttributeCache& GetPerDeviceAttributeCache() +_CCCL_HOST_API inline PerDeviceAttributeCache& GetPerDeviceAttributeCache() { static PerDeviceAttributeCache cache; return cache; @@ -362,7 +362,7 @@ struct SmVersionCacheTag * \note This function is thread safe. */ template -_CCCL_HOST cudaError_t PtxVersion(int& ptx_version, int device) +_CCCL_HOST_API cudaError_t PtxVersion(int& ptx_version, int device) { // Note: the ChainedPolicy pruning (i.e., invoke_static) requites that there's an exact match between one of the // architectures in __CUDA_ARCH__ and the runtime queried ptx version. @@ -653,7 +653,7 @@ inline constexpr int bulk_copy_min_align = 16; //! @tparam T //! Value type to be loaded. template -_CCCL_HOST_DEVICE constexpr int LoadToSharedBufferAlignBytes() +_CCCL_HOST_DEVICE_API constexpr int LoadToSharedBufferAlignBytes() { return (::cuda::std::max) (int{alignof(T)}, detail::bulk_copy_min_align); } @@ -666,7 +666,7 @@ _CCCL_HOST_DEVICE constexpr int LoadToSharedBufferAlignBytes() //! @param[in] num_items //! Size of the source range in global memory template -_CCCL_HOST_DEVICE constexpr int LoadToSharedBufferSizeBytes(::cuda::std::size_t num_items) +_CCCL_HOST_DEVICE_API constexpr int LoadToSharedBufferSizeBytes(::cuda::std::size_t num_items) { static_assert(::cuda::__is_valid_alignment(GmemAlign)); _CCCL_ASSERT(num_items <= ::cuda::std::size_t{::cuda::std::numeric_limits::max()}, @@ -694,10 +694,10 @@ _CCCL_HOST_DEVICE constexpr int LoadToSharedBufferSizeBytes(::cuda::std::size_t #define CUB_DETAIL_POLICY_WRAPPER_REFINE_CONCEPT(concept) concept&& // TODO(bgruber): drop in CCCL 4.0 when we drop the dispatchers -#define CUB_DETAIL_POLICY_WRAPPER_ACCESSOR(field) \ - __host__ __device__ static constexpr auto _CCCL_PP_SECOND field() \ - { \ - return StaticPolicyT::_CCCL_PP_FIRST field; \ +#define CUB_DETAIL_POLICY_WRAPPER_ACCESSOR(field) \ + _CCCL_HOST_DEVICE_API static constexpr auto _CCCL_PP_SECOND field() \ + { \ + return StaticPolicyT::_CCCL_PP_FIRST field; \ } template @@ -711,14 +711,14 @@ _CCCL_CONCEPT always_true = true; template \ struct concept_name##Wrapper : StaticPolicyT \ { \ - __host__ __device__ constexpr concept_name##Wrapper(StaticPolicyT base) \ + _CCCL_HOST_DEVICE_API constexpr concept_name##Wrapper(StaticPolicyT base) \ : StaticPolicyT(base) \ {} \ _CCCL_PP_FOR_EACH(CUB_DETAIL_POLICY_WRAPPER_ACCESSOR, __VA_ARGS__) \ }; \ _CCCL_TEMPLATE(typename StaticPolicyT) \ _CCCL_REQUIRES(concept_name) \ - __host__ __device__ constexpr concept_name##Wrapper MakePolicyWrapper(StaticPolicyT policy) \ + _CCCL_HOST_DEVICE_API constexpr concept_name##Wrapper MakePolicyWrapper(StaticPolicyT policy) \ { \ return concept_name##Wrapper{policy}; \ } @@ -733,7 +733,7 @@ _CCCL_TEMPLATE(typename PolicyT) #if _CCCL_STD_VER < 2020 _CCCL_REQUIRES((!GenericAgentPolicy) ) // in C++20+ we get this by preferring constrained functions #endif -__host__ __device__ constexpr PolicyT MakePolicyWrapper(PolicyT policy) +_CCCL_HOST_DEVICE_API constexpr PolicyT MakePolicyWrapper(PolicyT policy) { return policy; } diff --git a/cub/cub/util_macro.cuh b/cub/cub/util_macro.cuh index 024ed72af5e7..a75a7ef31fbd 100644 --- a/cub/cub/util_macro.cuh +++ b/cub/cub/util_macro.cuh @@ -50,7 +50,7 @@ _CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage) // TODO(bgruber): drop in CCCL 4.0 when we drop the public dispatchers #ifndef CUB_DEFINE_SUB_POLICY_GETTER # define CUB_DEFINE_SUB_POLICY_GETTER(name) \ - _CCCL_HOST_DEVICE static constexpr auto name() \ + _CCCL_HOST_DEVICE_API static constexpr auto name() \ { \ return _CUB::detail::MakePolicyWrapper(typename StaticPolicyT::name##Policy()); \ } diff --git a/cub/cub/util_math.cuh b/cub/cub/util_math.cuh index eb9c8094584c..fba9ecebd60d 100644 --- a/cub/cub/util_math.cuh +++ b/cub/cub/util_math.cuh @@ -42,7 +42,7 @@ using is_integral_or_enum = ::cuda::std::bool_constant<::cuda::std::is_integral_ * where `(lhs + rhs)` would overflow. */ template -_CCCL_HOST_DEVICE _CCCL_FORCEINLINE OffsetT safe_add_bound_to_max(OffsetT lhs, OffsetT rhs) +_CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE OffsetT safe_add_bound_to_max(OffsetT lhs, OffsetT rhs) { static_assert(::cuda::std::is_integral_v, "OffsetT must be an integral type"); static_assert(sizeof(OffsetT) >= 4, "OffsetT must be at least 32 bits in size"); @@ -50,12 +50,12 @@ _CCCL_HOST_DEVICE _CCCL_FORCEINLINE OffsetT safe_add_bound_to_max(OffsetT lhs, O return lhs + capped_operand_rhs; } -constexpr _CCCL_HOST_DEVICE int nominal_4B_items_to_items(int nominal_items_per_thread, int item_size) +_CCCL_HOST_DEVICE_API constexpr int nominal_4B_items_to_items(int nominal_items_per_thread, int item_size) { return ::cuda::std::clamp(nominal_items_per_thread * 4 / item_size, 1, nominal_items_per_thread); } -constexpr _CCCL_HOST_DEVICE int nominal_8B_items_to_items(int nominal_items_per_thread, int item_size) +_CCCL_HOST_DEVICE_API constexpr int nominal_8B_items_to_items(int nominal_items_per_thread, int item_size) { if (item_size <= 8) { @@ -65,20 +65,20 @@ constexpr _CCCL_HOST_DEVICE int nominal_8B_items_to_items(int nominal_items_per_ } } // namespace detail -constexpr _CCCL_HOST_DEVICE int Nominal4BItemsToItemsCombined(int nominal_items_per_thread, int combined_bytes) +_CCCL_HOST_DEVICE_API constexpr int Nominal4BItemsToItemsCombined(int nominal_items_per_thread, int combined_bytes) { return ( ::cuda::std::min) (nominal_items_per_thread, (::cuda::std::max) (1, nominal_items_per_thread * 8 / combined_bytes)); } template -constexpr _CCCL_HOST_DEVICE int Nominal4BItemsToItems(int nominal_items_per_thread) +_CCCL_HOST_DEVICE_API constexpr int Nominal4BItemsToItems(int nominal_items_per_thread) { return _CUB::detail::nominal_4B_items_to_items(nominal_items_per_thread, int{sizeof(T)}); } template -constexpr _CCCL_HOST_DEVICE int Nominal8BItemsToItems(int nominal_items_per_thread) +_CCCL_HOST_DEVICE_API constexpr int Nominal8BItemsToItems(int nominal_items_per_thread) { return _CUB::detail::nominal_8B_items_to_items(nominal_items_per_thread, int{sizeof(ItemT)}); } @@ -91,7 +91,7 @@ constexpr _CCCL_HOST_DEVICE int Nominal8BItemsToItems(int nominal_items_per_thre * \return Half the sum of \p begin and \p end */ template -constexpr _CCCL_HOST_DEVICE T MidPoint(T begin, T end) +_CCCL_HOST_DEVICE_API constexpr T MidPoint(T begin, T end) { return begin + ((end - begin) / 2); } diff --git a/cub/cub/util_ptx.cuh b/cub/cub/util_ptx.cuh index 1ff2bc729bc1..2f562a92051f 100644 --- a/cub/cub/util_ptx.cuh +++ b/cub/cub/util_ptx.cuh @@ -40,7 +40,7 @@ CUB_NAMESPACE_BEGIN */ template //! deprecated [Since 3.0] -CCCL_DEPRECATED_BECAUSE("Use cuda::bitfield_extract()") _CCCL_DEVICE _CCCL_FORCEINLINE unsigned int +CCCL_DEPRECATED_BECAUSE("Use cuda::bitfield_extract()") _CCCL_DEVICE_API _CCCL_FORCEINLINE unsigned int BFE(UnsignedBits source, unsigned int bit_start, unsigned int num_bits, detail::constant_t /*byte_len*/) { unsigned int bits; @@ -53,7 +53,7 @@ BFE(UnsignedBits source, unsigned int bit_start, unsigned int num_bits, detail:: */ template //! deprecated [Since 3.0] -CCCL_DEPRECATED_BECAUSE("Use cuda::bitfield_extract()") _CCCL_DEVICE _CCCL_FORCEINLINE unsigned int +CCCL_DEPRECATED_BECAUSE("Use cuda::bitfield_extract()") _CCCL_DEVICE_API _CCCL_FORCEINLINE unsigned int BFE(UnsignedBits source, unsigned int bit_start, unsigned int num_bits, detail::constant_t<8> /*byte_len*/) { const unsigned long long MASK = (1ull << num_bits) - 1; @@ -66,7 +66,7 @@ BFE(UnsignedBits source, unsigned int bit_start, unsigned int num_bits, detail:: */ template //! deprecated [Since 3.0] -CCCL_DEPRECATED_BECAUSE("Use cuda::bitfield_extract()") _CCCL_DEVICE _CCCL_FORCEINLINE unsigned int +CCCL_DEPRECATED_BECAUSE("Use cuda::bitfield_extract()") _CCCL_DEVICE_API _CCCL_FORCEINLINE unsigned int BFE(UnsignedBits source, unsigned int bit_start, unsigned int num_bits, detail::constant_t<16> /*byte_len*/) { const __uint128_t MASK = (__uint128_t{1} << num_bits) - 1; @@ -82,7 +82,7 @@ BFE(UnsignedBits source, unsigned int bit_start, unsigned int num_bits, detail:: */ template //! deprecated [Since 3.0] -CCCL_DEPRECATED_BECAUSE("Use cuda::bitfield_extract()") _CCCL_DEVICE +CCCL_DEPRECATED_BECAUSE("Use cuda::bitfield_extract()") _CCCL_DEVICE_API _CCCL_FORCEINLINE unsigned int BFE(UnsignedBits source, unsigned int bit_start, unsigned int num_bits) { return _CUB::BFE(source, bit_start, num_bits, detail::constant_v); @@ -93,7 +93,7 @@ _CCCL_FORCEINLINE unsigned int BFE(UnsignedBits source, unsigned int bit_start, /** * Warp synchronous shfl_up */ -_CCCL_DEVICE _CCCL_FORCEINLINE unsigned int +_CCCL_DEVICE_API _CCCL_FORCEINLINE unsigned int SHFL_UP_SYNC(unsigned int word, int src_offset, int flags, unsigned int member_mask) { asm volatile("shfl.sync.up.b32 %0, %1, %2, %3, %4;" @@ -105,7 +105,7 @@ SHFL_UP_SYNC(unsigned int word, int src_offset, int flags, unsigned int member_m /** * Warp synchronous shfl_down */ -_CCCL_DEVICE _CCCL_FORCEINLINE unsigned int +_CCCL_DEVICE_API _CCCL_FORCEINLINE unsigned int SHFL_DOWN_SYNC(unsigned int word, int src_offset, int flags, unsigned int member_mask) { asm volatile("shfl.sync.down.b32 %0, %1, %2, %3, %4;" @@ -119,7 +119,7 @@ SHFL_DOWN_SYNC(unsigned int word, int src_offset, int flags, unsigned int member /** * \brief Terminates the calling thread */ -_CCCL_DEVICE _CCCL_FORCEINLINE void ThreadExit() +_CCCL_DEVICE_API _CCCL_FORCEINLINE void ThreadExit() { asm volatile("exit;"); } @@ -127,7 +127,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE void ThreadExit() /** * \brief Returns the row-major linear thread identifier for a multidimensional thread block */ -_CCCL_DEVICE _CCCL_FORCEINLINE int RowMajorTid(int block_dim_x, int block_dim_y, int block_dim_z) +_CCCL_DEVICE_API _CCCL_FORCEINLINE int RowMajorTid(int block_dim_x, int block_dim_y, int block_dim_z) { return static_cast(((block_dim_z == 1) ? 0 : (threadIdx.z * block_dim_x * block_dim_y)) + ((block_dim_y == 1) ? 0 : (threadIdx.y * block_dim_x)) + threadIdx.x); @@ -219,7 +219,7 @@ template * 32-bit mask of participating warp lanes */ template -_CCCL_DEVICE _CCCL_FORCEINLINE T ShuffleUp(T input, int src_offset, int first_thread, unsigned int member_mask) +_CCCL_DEVICE_API _CCCL_FORCEINLINE T ShuffleUp(T input, int src_offset, int first_thread, unsigned int member_mask) { /// The 5-bit SHFL mask for logically splitting warps into sub-segments starts 8-bits up constexpr int SHFL_C = (32 - LogicalWarpThreads) << 8; @@ -299,7 +299,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE T ShuffleUp(T input, int src_offset, int first_th * 32-bit mask of participating warp lanes */ template -_CCCL_DEVICE _CCCL_FORCEINLINE T ShuffleDown(T input, int src_offset, int last_thread, unsigned int member_mask) +_CCCL_DEVICE_API _CCCL_FORCEINLINE T ShuffleDown(T input, int src_offset, int last_thread, unsigned int member_mask) { /// The 5-bit SHFL mask for logically splitting warps into sub-segments starts 8-bits up static constexpr int SHFL_C = (32 - LogicalWarpThreads) << 8; @@ -379,7 +379,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE T ShuffleDown(T input, int src_offset, int last_t * 32-bit mask of participating warp lanes */ template -_CCCL_DEVICE _CCCL_FORCEINLINE T ShuffleIndex(T input, int src_lane, unsigned int member_mask) +_CCCL_DEVICE_API _CCCL_FORCEINLINE T ShuffleIndex(T input, int src_lane, unsigned int member_mask) { using ShuffleWord = typename UnitWord::ShuffleWord; @@ -423,7 +423,7 @@ namespace detail template struct warp_matcher_t { - static _CCCL_DEVICE unsigned int match_any(unsigned int label) + _CCCL_DEVICE_API static unsigned int match_any(unsigned int label) { return warp_matcher_t::match_any(label) & ~(~0 << WarpActiveThreads); } @@ -434,7 +434,7 @@ struct warp_matcher_t { // match.any.sync.b32 is slower when matching a few bits // using a ballot loop instead - static _CCCL_DEVICE unsigned int match_any(unsigned int label) + _CCCL_DEVICE_API static unsigned int match_any(unsigned int label) { unsigned int retval; @@ -466,7 +466,7 @@ struct warp_matcher_t * @brief Shifts @p val left by the amount specified by unsigned 32-bit value in @p num_bits. If @p * num_bits is larger than 32 bits, @p num_bits is clamped to 32. */ -_CCCL_DEVICE _CCCL_FORCEINLINE uint32_t LogicShiftLeft(uint32_t val, uint32_t num_bits) +_CCCL_DEVICE_API _CCCL_FORCEINLINE uint32_t LogicShiftLeft(uint32_t val, uint32_t num_bits) { uint32_t ret{}; // NOLINT(misc-const-correctness) asm("shl.b32 %0, %1, %2;" : "=r"(ret) : "r"(val), "r"(num_bits)); @@ -477,7 +477,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE uint32_t LogicShiftLeft(uint32_t val, uint32_t nu * @brief Shifts @p val right by the amount specified by unsigned 32-bit value in @p num_bits. If @p * num_bits is larger than 32 bits, @p num_bits is clamped to 32. */ -_CCCL_DEVICE _CCCL_FORCEINLINE uint32_t LogicShiftRight(uint32_t val, uint32_t num_bits) +_CCCL_DEVICE_API _CCCL_FORCEINLINE uint32_t LogicShiftRight(uint32_t val, uint32_t num_bits) { uint32_t ret{}; // NOLINT(misc-const-correctness) asm("shr.b32 %0, %1, %2;" : "=r"(ret) : "r"(val), "r"(num_bits)); @@ -491,7 +491,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE uint32_t LogicShiftRight(uint32_t val, uint32_t n * LabelBits of \p label as the calling thread. */ template -inline _CCCL_DEVICE unsigned int MatchAny(unsigned int label) +_CCCL_DEVICE_API inline unsigned int MatchAny(unsigned int label) { return detail::warp_matcher_t::match_any(label); } diff --git a/cub/cub/util_temporary_storage.cuh b/cub/cub/util_temporary_storage.cuh index fc8d3a7f1623..f41f171fc3d8 100644 --- a/cub/cub/util_temporary_storage.cuh +++ b/cub/cub/util_temporary_storage.cuh @@ -47,7 +47,7 @@ namespace detail //! @param[in] allocation_sizes //! Sizes in bytes of device allocations needed template -[[nodiscard]] _CCCL_HOST_DEVICE _CCCL_FORCEINLINE cudaError_t alias_temporaries( +[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_FORCEINLINE cudaError_t alias_temporaries( void* d_temp_storage, size_t& temp_storage_bytes, void* (&allocations)[NumAllocations], diff --git a/cub/cub/util_vsmem.cuh b/cub/cub/util_vsmem.cuh index 1498a1926cd3..268440d8957b 100644 --- a/cub/cub/util_vsmem.cuh +++ b/cub/cub/util_vsmem.cuh @@ -77,7 +77,7 @@ public: * passed to the agent, specialized for the case when we can use native shared memory as temporary * storage. */ - static _CCCL_DEVICE _CCCL_FORCEINLINE typename AgentT::TempStorage& + _CCCL_DEVICE_API static _CCCL_FORCEINLINE typename AgentT::TempStorage& get_temp_storage(typename AgentT::TempStorage& static_temp_storage, vsmem_t&) { return static_temp_storage; @@ -88,7 +88,7 @@ public: * passed to the agent, specialized for the case when we can use native shared memory as temporary * storage and taking a linear block id. */ - static _CCCL_DEVICE _CCCL_FORCEINLINE typename AgentT::TempStorage& + _CCCL_DEVICE_API static _CCCL_FORCEINLINE typename AgentT::TempStorage& get_temp_storage(typename AgentT::TempStorage& static_temp_storage, vsmem_t&, ::cuda::std::size_t) { return static_temp_storage; @@ -99,7 +99,7 @@ public: * passed to the agent, specialized for the case when we have to use global memory-backed * virtual shared memory as temporary storage. */ - static _CCCL_DEVICE _CCCL_FORCEINLINE typename AgentT::TempStorage& + _CCCL_DEVICE_API static _CCCL_FORCEINLINE typename AgentT::TempStorage& get_temp_storage(cub::NullType& static_temp_storage, vsmem_t& vsmem) { return *reinterpret_cast( @@ -111,7 +111,7 @@ public: * passed to the agent, specialized for the case when we have to use global memory-backed * virtual shared memory as temporary storage and taking a linear block id. */ - static _CCCL_DEVICE _CCCL_FORCEINLINE typename AgentT::TempStorage& + _CCCL_DEVICE_API static _CCCL_FORCEINLINE typename AgentT::TempStorage& get_temp_storage(cub::NullType& static_temp_storage, vsmem_t& vsmem, ::cuda::std::size_t linear_block_id) { return *reinterpret_cast( @@ -126,7 +126,7 @@ public: * supposed to be reused after this function call. */ template = 0> - static _CCCL_DEVICE _CCCL_FORCEINLINE bool discard_temp_storage(typename AgentT::TempStorage& temp_storage) + _CCCL_DEVICE_API static _CCCL_FORCEINLINE bool discard_temp_storage(typename AgentT::TempStorage& temp_storage) { return false; } @@ -139,7 +139,7 @@ public: * supposed to be reused after this function call. */ template = 0> - static _CCCL_DEVICE _CCCL_FORCEINLINE bool discard_temp_storage(typename AgentT::TempStorage& temp_storage) + _CCCL_DEVICE_API static _CCCL_FORCEINLINE bool discard_temp_storage(typename AgentT::TempStorage& temp_storage) { // Ensure all threads finished using temporary storage __syncthreads(); @@ -160,7 +160,7 @@ public: }; template -_CCCL_HOST_DEVICE constexpr bool use_fallback_agent() +_CCCL_HOST_DEVICE_API constexpr bool use_fallback_agent() { return (sizeof(typename DefaultAgentT::TempStorage) > max_smem_per_block) && (sizeof(typename FallbackAgentT::TempStorage) <= max_smem_per_block); diff --git a/cub/cub/warp/specializations/warp_exchange_shfl.cuh b/cub/cub/warp/specializations/warp_exchange_shfl.cuh index 1dfa937b5b5e..a68c433ae366 100644 --- a/cub/cub/warp/specializations/warp_exchange_shfl.cuh +++ b/cub/cub/warp/specializations/warp_exchange_shfl.cuh @@ -35,7 +35,7 @@ class WarpExchangeShfl "LogicalWarpThreads"); template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void transpose_foreach(InputT (&vals)[ItemsPerThread], const bool xor_bit_set, const unsigned mask) { // The implementation here is a recursive divide-and-conquer approach @@ -189,7 +189,7 @@ class WarpExchangeShfl } template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void transpose(InputT (&vals)[ItemsPerThread], const unsigned int lane_id, const unsigned int mask) { if constexpr (NumEntries != 0) @@ -202,7 +202,7 @@ class WarpExchangeShfl } template - _CCCL_DEVICE void copy_into( + _CCCL_DEVICE_API void copy_into( InputT (&vals)[ItemsPerThread], const InputT (&input_items)[ItemsPerThread], ::cuda::std::index_sequence) { // A #pragma unroll'ed for-loop is not sufficient, it produces worse SASS @@ -210,7 +210,7 @@ class WarpExchangeShfl } template - _CCCL_DEVICE void copy_out( + _CCCL_DEVICE_API void copy_out( OutputT (&output_items)[ItemsPerThread], const InputT (&vals)[ItemsPerThread], ::cuda::std::index_sequence) { // A #pragma unroll'ed for-loop is not sufficient, it produces worse SASS @@ -225,10 +225,10 @@ public: WarpExchangeShfl() = delete; - explicit _CCCL_DEVICE _CCCL_FORCEINLINE WarpExchangeShfl(TempStorage&) {} + explicit _CCCL_DEVICE_API _CCCL_FORCEINLINE WarpExchangeShfl(TempStorage&) {} template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void BlockedToStriped(const InputT (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread]) { InputT vals[ItemsPerThread]; @@ -240,7 +240,7 @@ public: } template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void StripedToBlocked(const InputT (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread]) { BlockedToStriped(input_items, output_items); @@ -255,7 +255,7 @@ public: }; template - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterToStriped(InputT (&)[ItemsPerThread], OffsetT (&)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterToStriped(InputT (&)[ItemsPerThread], OffsetT (&)[ItemsPerThread]) { static_assert(dependent_false::value, "Shuffle specialization of warp exchange does not support\n" @@ -264,7 +264,7 @@ public: } template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterToStriped(const InputT (&)[ItemsPerThread], OutputT (&)[ItemsPerThread], OffsetT (&)[ItemsPerThread]) { static_assert(dependent_false::value, diff --git a/cub/cub/warp/specializations/warp_exchange_smem.cuh b/cub/cub/warp/specializations/warp_exchange_smem.cuh index e424d229c8c4..29e05917a64b 100644 --- a/cub/cub/warp/specializations/warp_exchange_smem.cuh +++ b/cub/cub/warp/specializations/warp_exchange_smem.cuh @@ -60,12 +60,12 @@ public: WarpExchangeSmem() = delete; - explicit _CCCL_DEVICE _CCCL_FORCEINLINE WarpExchangeSmem(TempStorage& temp_storage) + explicit _CCCL_DEVICE_API _CCCL_FORCEINLINE WarpExchangeSmem(TempStorage& temp_storage) : temp_storage(temp_storage.Alias()) {} template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void BlockedToStriped(const InputT (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread]) { for (int item = 0; item < ItemsPerThread; item++) @@ -83,7 +83,7 @@ public: } template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void StripedToBlocked(const InputT (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread]) { for (int item = 0; item < ItemsPerThread; item++) @@ -101,13 +101,14 @@ public: } template - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterToStriped(InputT (&items)[ItemsPerThread], OffsetT (&ranks)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void + ScatterToStriped(InputT (&items)[ItemsPerThread], OffsetT (&ranks)[ItemsPerThread]) { ScatterToStriped(items, items, ranks); } template - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterToStriped( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterToStriped( const InputT (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread], OffsetT (&ranks)[ItemsPerThread]) diff --git a/cub/cub/warp/specializations/warp_reduce_smem.cuh b/cub/cub/warp/specializations/warp_reduce_smem.cuh new file mode 100644 index 000000000000..356f029d8302 --- /dev/null +++ b/cub/cub/warp/specializations/warp_reduce_smem.cuh @@ -0,0 +1,375 @@ +// SPDX-FileCopyrightText: Copyright (c) 2011, Duane Merrill. All rights reserved. +// SPDX-FileCopyrightText: Copyright (c) 2011-2025, NVIDIA CORPORATION. All rights reserved. +// SPDX-License-Identifier: BSD-3 + +/** + * @file + * cub::WarpReduceSmem provides smem-based variants of parallel reduction of items partitioned + * across a CUDA thread warp. + */ + +#pragma once + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +CUB_NAMESPACE_BEGIN + +namespace detail +{ +/** + * @brief WarpReduceSmem provides smem-based variants of parallel reduction of items partitioned + * across a CUDA thread warp. + * + * @tparam T + * Data type being reduced + * + * @tparam LogicalWarpThreads + * Number of threads per logical warp + */ +template +struct WarpReduceSmem +{ + /****************************************************************************** + * Constants and type definitions + ******************************************************************************/ + + /// Whether the logical warp size and the PTX warp size coincide + static constexpr bool IS_ARCH_WARP = (LogicalWarpThreads == warp_threads); + + /// Whether the logical warp size is a power-of-two + static constexpr bool IS_POW_OF_TWO = ::cuda::is_power_of_two(LogicalWarpThreads); + + /// The number of warp reduction steps + static constexpr int STEPS = Log2::VALUE; + + /// The number of threads in half a warp + static constexpr int HALF_WARP_THREADS = 1 << (STEPS - 1); + + /// The number of shared memory elements per warp + static constexpr int WARP_SMEM_ELEMENTS = LogicalWarpThreads + HALF_WARP_THREADS; + + /// FlagT status (when not using ballot) + static constexpr auto UNSET = 0x0; // Is initially unset + static constexpr auto SET = 0x1; // Is initially set + static constexpr auto SEEN = 0x2; // Has seen another head flag from a successor peer + + /// Shared memory flag type + using SmemFlag = unsigned char; + + /// Shared memory storage layout type (1.5 warps-worth of elements for each warp) + struct _TempStorage + { + T reduce[WARP_SMEM_ELEMENTS]; + SmemFlag flags[WARP_SMEM_ELEMENTS]; + }; + + // Alias wrapper allowing storage to be unioned + using TempStorage = Uninitialized<_TempStorage>; + + /****************************************************************************** + * Thread fields + ******************************************************************************/ + + _TempStorage& temp_storage; + unsigned int lane_id = CUB_NS_QUALIFIER::detail::logical_lane_id(); + unsigned int member_mask = CUB_NS_QUALIFIER::detail::warp_mask(); + + /****************************************************************************** + * Construction + ******************************************************************************/ + + /// Constructor + explicit _CCCL_DEVICE_API _CCCL_FORCEINLINE WarpReduceSmem(TempStorage& temp_storage) + : temp_storage(temp_storage.Alias()) + {} + + /****************************************************************************** + * Utility methods + ******************************************************************************/ + + //--------------------------------------------------------------------- + // Regular reduction + //--------------------------------------------------------------------- + + /** + * @brief Reduction step + * + * @tparam AllLanesValid + * Whether all lanes in each warp are contributing a valid fold of items + * + * @param[in] input + * Calling thread's input + * + * @param[in] valid_items + * Total number of valid items across the logical warp + * + * @param[in] reduction_op + * Reduction operator + */ + template + _CCCL_DEVICE_API _CCCL_FORCEINLINE T + ReduceStep(T input, int valid_items, ReductionOp reduction_op, constant_t /*step*/) + { + constexpr int OFFSET = 1 << STEP; + // Share input through buffer + temp_storage.reduce[lane_id] = input; + __syncwarp(member_mask); + // Update input if peer_addend is in range + if ((AllLanesValid && IS_POW_OF_TWO) || ((lane_id + OFFSET) < valid_items)) + { + T peer_addend = temp_storage.reduce[lane_id + OFFSET]; + input = reduction_op(input, peer_addend); + } + __syncwarp(member_mask); + return ReduceStep(input, valid_items, reduction_op, constant_v); + } + + /** + * @brief Reduction step (terminate) + * + * @tparam AllLanesValid + * Whether all lanes in each warp are contributing a valid fold of items + * + * @param[in] input + * Calling thread's input + * + * @param[in] valid_items + * Total number of valid items across the logical warp + * + * @param[in] reduction_op + * Reduction operator + */ + template + _CCCL_DEVICE_API _CCCL_FORCEINLINE T + ReduceStep(T input, int valid_items, ReductionOp /*reduction_op*/, constant_t /*step*/) + { + return input; + } + + //--------------------------------------------------------------------- + // Segmented reduction + //--------------------------------------------------------------------- + + /** + * @brief Ballot-based segmented reduce + * + * @tparam HeadSegmented + * Whether flags indicate a segment-head or a segment-tail + * + * @param[in] input + * Calling thread's input + * + * @param[in] flag + * Whether or not the current lane is a segment head/tail + * + * @param[in] reduction_op + * Reduction operator + * + * @param[in] has_ballot + * Marker type for whether the target arch has ballot functionality + */ + template + _CCCL_DEVICE_API _CCCL_FORCEINLINE T + SegmentedReduce(T input, FlagT flag, ReductionOp reduction_op, ::cuda::std::true_type /*has_ballot*/) + { + // Get the start flags for each thread in the warp. + unsigned warp_flags = __ballot_sync(member_mask, flag); + + if (!HeadSegmented) + { + warp_flags <<= 1; + } + + // Keep bits above the current thread. + warp_flags &= ::cuda::ptx::get_sreg_lanemask_gt(); + + // Accommodate packing of multiple logical warps in a single physical warp + warp_flags >>= CUB_NS_QUALIFIER::detail::logical_warp_base_id(); + + // Find next flag + int next_flag = ::cuda::std::countr_zero(warp_flags); + + // Clip the next segment at the warp boundary if necessary + if (LogicalWarpThreads != 32) + { + next_flag = ::cuda::std::min(next_flag, LogicalWarpThreads); + } + + _CCCL_PRAGMA_UNROLL_FULL() + for (int STEP = 0; STEP < STEPS; STEP++) + { + const int OFFSET = 1 << STEP; + + // Share input into buffer + temp_storage.reduce[lane_id] = input; + + __syncwarp(member_mask); + + // Update input if peer_addend is in range + if (OFFSET + lane_id < next_flag) + { + T peer_addend = temp_storage.reduce[lane_id + OFFSET]; + input = reduction_op(input, peer_addend); + } + + __syncwarp(member_mask); + } + + return input; + } + + /** + * @brief Smem-based segmented reduce + * + * @tparam HeadSegmented + * Whether flags indicate a segment-head or a segment-tail + * + * @param[in] input + * Calling thread's input + * + * @param[in] flag + * Whether or not the current lane is a segment head/tail + * + * @param[in] reduction_op + * Reduction operator + * + * @param[in] has_ballot + * Marker type for whether the target arch has ballot functionality + */ + template + _CCCL_DEVICE_API _CCCL_FORCEINLINE T + SegmentedReduce(T input, FlagT flag, ReductionOp reduction_op, ::cuda::std::false_type /*has_ballot*/) + { + // Alias flags onto shared data storage + SmemFlag* flag_storage = temp_storage.flags; + + SmemFlag flag_status = (flag) ? SET : UNSET; + + for (int STEP = 0; STEP < STEPS; STEP++) + { + const int OFFSET = 1 << STEP; + + // Share input through buffer + temp_storage.reduce[lane_id] = input; + + __syncwarp(member_mask); + + // Get peer from buffer + T peer_addend = temp_storage.reduce[lane_id + OFFSET]; + + __syncwarp(member_mask); + + // Share flag through buffer + flag_storage[lane_id] = flag_status; + + // Get peer flag from buffer + SmemFlag peer_flag_status = flag_storage[lane_id + OFFSET]; + + // Update input if peer was in range + if (lane_id < LogicalWarpThreads - OFFSET) + { + if (HeadSegmented) + { + // Head-segmented + if ((flag_status & SEEN) == 0) + { + // Has not seen a more distant head flag + if (peer_flag_status & SET) + { + // Has now seen a head flag + flag_status |= SEEN; + } + else + { + // Peer is not a head flag: grab its count + input = reduction_op(input, peer_addend); + } + + // Update seen status to include that of peer + flag_status |= (peer_flag_status & SEEN); + } + } + else + { + // Tail-segmented. Simply propagate flag status + if (!flag_status) + { + input = reduction_op(input, peer_addend); + flag_status |= peer_flag_status; + } + } + } + } + + return input; + } + + /****************************************************************************** + * Interface + ******************************************************************************/ + + /** + * @brief Reduction + * + * @tparam AllLanesValid + * Whether all lanes in each warp are contributing a valid fold of items + * + * @param[in] input + * Calling thread's input + * + * @param[in] valid_items + * Total number of valid items across the logical warp + * + * @param[in] reduction_op + * Reduction operator + */ + template + _CCCL_DEVICE_API _CCCL_FORCEINLINE T Reduce(T input, int valid_items, ReductionOp reduction_op) + { + return ReduceStep(input, valid_items, reduction_op, constant_v<0>); + } + + /** + * @brief Segmented reduction + * + * @tparam HeadSegmented + * Whether flags indicate a segment-head or a segment-tail + * + * @param[in] input + * Calling thread's input + * + * @param[in] flag + * Whether or not the current lane is a segment head/tail + * + * @param[in] reduction_op + * Reduction operator + */ + template + _CCCL_DEVICE_API _CCCL_FORCEINLINE T SegmentedReduce(T input, FlagT flag, ReductionOp reduction_op) + { + return SegmentedReduce(input, flag, reduction_op, ::cuda::std::true_type()); + } +}; +} // namespace detail + +CUB_NAMESPACE_END diff --git a/cub/cub/warp/specializations/warp_scan_shfl.cuh b/cub/cub/warp/specializations/warp_scan_shfl.cuh index a04af650e491..536dc5c5a1a9 100644 --- a/cub/cub/warp/specializations/warp_scan_shfl.cuh +++ b/cub/cub/warp/specializations/warp_scan_shfl.cuh @@ -92,7 +92,7 @@ struct WarpScanShfl //--------------------------------------------------------------------- /// Constructor - explicit _CCCL_DEVICE _CCCL_FORCEINLINE WarpScanShfl(TempStorage& /*temp_storage*/) {} + explicit _CCCL_DEVICE_API _CCCL_FORCEINLINE WarpScanShfl(TempStorage& /*temp_storage*/) {} //--------------------------------------------------------------------- // Inclusive scan steps @@ -113,7 +113,7 @@ struct WarpScanShfl * @param[in] offset * Up-offset to pull from */ - _CCCL_DEVICE _CCCL_FORCEINLINE int + _CCCL_DEVICE_API _CCCL_FORCEINLINE int InclusiveScanStep(int input, ::cuda::std::plus<> /*scan_op*/, int first_lane, int offset) { int output; @@ -149,7 +149,7 @@ struct WarpScanShfl * @param[in] offset * Up-offset to pull from */ - _CCCL_DEVICE _CCCL_FORCEINLINE unsigned int + _CCCL_DEVICE_API _CCCL_FORCEINLINE unsigned int InclusiveScanStep(unsigned int input, ::cuda::std::plus<> /*scan_op*/, int first_lane, int offset) { unsigned int output; @@ -185,7 +185,7 @@ struct WarpScanShfl * @param[in] offset * Up-offset to pull from */ - _CCCL_DEVICE _CCCL_FORCEINLINE float + _CCCL_DEVICE_API _CCCL_FORCEINLINE float InclusiveScanStep(float input, ::cuda::std::plus<> /*scan_op*/, int first_lane, int offset) { float output; @@ -221,7 +221,7 @@ struct WarpScanShfl * @param[in] offset * Up-offset to pull from */ - _CCCL_DEVICE _CCCL_FORCEINLINE unsigned long long + _CCCL_DEVICE_API _CCCL_FORCEINLINE unsigned long long InclusiveScanStep(unsigned long long input, ::cuda::std::plus<> /*scan_op*/, int first_lane, int offset) { unsigned long long output; @@ -262,7 +262,7 @@ struct WarpScanShfl * @param[in] offset * Up-offset to pull from */ - _CCCL_DEVICE _CCCL_FORCEINLINE long long + _CCCL_DEVICE_API _CCCL_FORCEINLINE long long InclusiveScanStep(long long input, ::cuda::std::plus<> /*scan_op*/, int first_lane, int offset) { long long output; @@ -303,7 +303,7 @@ struct WarpScanShfl * @param[in] offset * Up-offset to pull from */ - _CCCL_DEVICE _CCCL_FORCEINLINE double + _CCCL_DEVICE_API _CCCL_FORCEINLINE double InclusiveScanStep(double input, ::cuda::std::plus<> /*scan_op*/, int first_lane, int offset) { double output; @@ -368,7 +368,7 @@ struct WarpScanShfl * Up-offset to pull from */ template - _CCCL_DEVICE _CCCL_FORCEINLINE Tp InclusiveScanStep(Tp input, ScanOpT scan_op, int first_lane, int offset) + _CCCL_DEVICE_API _CCCL_FORCEINLINE Tp InclusiveScanStep(Tp input, ScanOpT scan_op, int first_lane, int offset) { Tp temp = _CUB::ShuffleUp(input, offset, first_lane, member_mask); @@ -401,7 +401,7 @@ struct WarpScanShfl * Up-offset to pull from */ template - _CCCL_DEVICE _CCCL_FORCEINLINE Tp + _CCCL_DEVICE_API _CCCL_FORCEINLINE Tp InclusiveScanStepPartial(Tp input, ScanOpT scan_op, int valid_items, int first_lane, int offset) { _CCCL_ASSERT((first_lane >= 0) && (first_lane <= static_cast(lane_id)), @@ -438,7 +438,7 @@ struct WarpScanShfl * Marker type indicating whether T is a small integer */ template - _CCCL_DEVICE _CCCL_FORCEINLINE Tp + _CCCL_DEVICE_API _CCCL_FORCEINLINE Tp InclusiveScanStep(Tp input, ScanOpT scan_op, int first_lane, int offset, ::cuda::std::true_type /*is_small_unsigned*/) { return InclusiveScanStep(input, scan_op, first_lane, offset); @@ -464,7 +464,7 @@ struct WarpScanShfl * Marker type indicating whether T is a small integer */ template - _CCCL_DEVICE _CCCL_FORCEINLINE Tp InclusiveScanStep( + _CCCL_DEVICE_API _CCCL_FORCEINLINE Tp InclusiveScanStep( const Tp& input, ScanOpT scan_op, int first_lane, int offset, ::cuda::std::false_type /*is_small_unsigned*/) { return InclusiveScanStep(input, scan_op, first_lane, offset); @@ -487,7 +487,7 @@ struct WarpScanShfl * @param[in] src_lane * Which warp lane is to do the broadcasting */ - _CCCL_DEVICE _CCCL_FORCEINLINE T Broadcast(T input, int src_lane) + _CCCL_DEVICE_API _CCCL_FORCEINLINE T Broadcast(T input, int src_lane) { return _CUB::ShuffleIndex(input, src_lane, member_mask); } @@ -509,7 +509,7 @@ struct WarpScanShfl * Binary scan operator */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void InclusiveScan(Tp input, Tp& inclusive_output, ScanOpT scan_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScan(Tp input, Tp& inclusive_output, ScanOpT scan_op) { inclusive_output = input; @@ -545,7 +545,7 @@ struct WarpScanShfl * Number of valid items in warp */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScanPartial(Tp input, Tp& inclusive_output, ScanOpT scan_op, int valid_items) { if (static_cast(lane_id) < valid_items) @@ -575,7 +575,7 @@ struct WarpScanShfl * Binary scan operator */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void InclusiveScan( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScan( KeyValuePair input, KeyValuePair& inclusive_output, ReduceByKeyOp scan_op) { inclusive_output = input; @@ -619,7 +619,7 @@ struct WarpScanShfl * Warp-wide aggregate reduction of input items */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void InclusiveScan(T input, T& inclusive_output, ScanOpT scan_op, T& warp_aggregate) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScan(T input, T& inclusive_output, ScanOpT scan_op, T& warp_aggregate) { InclusiveScan(input, inclusive_output, scan_op); @@ -646,7 +646,7 @@ struct WarpScanShfl * Warp-wide aggregate reduction of input items */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScanPartial(T input, T& inclusive_output, ScanOpT scan_op, int valid_items, T& warp_aggregate) { InclusiveScanPartial(input, inclusive_output, scan_op, valid_items); @@ -675,7 +675,7 @@ struct WarpScanShfl * @param[in] is_integer */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Update(T /*input*/, T& inclusive, T& exclusive, ScanOpT /*scan_op*/, IsIntegerT /*is_integer*/) { // initial value unknown @@ -686,7 +686,7 @@ struct WarpScanShfl * @brief Update inclusive and exclusive using input and inclusive (specialized for summation of * integer types) */ - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Update(T input, T& inclusive, T& exclusive, ::cuda::std::plus<> /*scan_op*/, ::cuda::std::true_type /*is_integer*/) { // initial value presumed 0 @@ -698,7 +698,7 @@ struct WarpScanShfl * value */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Update(T /*input*/, T& inclusive, T& exclusive, ScanOpT scan_op, T initial_value, IsIntegerT /*is_integer*/) { inclusive = scan_op(initial_value, inclusive); @@ -714,7 +714,7 @@ struct WarpScanShfl * @brief Update inclusive and exclusive using initial value using input and inclusive * (specialized for summation of integer types) */ - _CCCL_DEVICE _CCCL_FORCEINLINE void Update( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Update( T input, T& inclusive, T& exclusive, @@ -730,7 +730,7 @@ struct WarpScanShfl * @brief Update inclusive, exclusive, and warp aggregate using input and inclusive */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Update(T input, T& inclusive, T& exclusive, T& warp_aggregate, ScanOpT scan_op, IsIntegerT is_integer) { warp_aggregate = _CUB::ShuffleIndex(inclusive, LogicalWarpThreads - 1, member_mask); @@ -742,7 +742,7 @@ struct WarpScanShfl * value */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void Update( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Update( T input, T& inclusive, T& exclusive, T& warp_aggregate, ScanOpT scan_op, T initial_value, IsIntegerT is_integer) { warp_aggregate = _CUB::ShuffleIndex(inclusive, LogicalWarpThreads - 1, member_mask); @@ -768,7 +768,7 @@ struct WarpScanShfl * Number of valid items in warp */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void UpdatePartial([[maybe_unused]] T input, T& inclusive, T& exclusive, [[maybe_unused]] ScanOpT scan_op, int valid_items) { if constexpr (::cuda::std::is_integral_v && ::cuda::__is_cuda_std_plus_v) @@ -814,7 +814,7 @@ struct WarpScanShfl * Initial value to seed the scan (uniform across warp) */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void UpdatePartial(T input, T& inclusive, T& exclusive, ScanOpT scan_op, int valid_items, T initial_value) { // Update inclusive @@ -858,7 +858,7 @@ struct WarpScanShfl * Number of valid items in warp */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void UpdatePartial(T input, T& inclusive, T& exclusive, T& warp_aggregate, ScanOpT scan_op, int valid_items) { // Get aggregate @@ -895,7 +895,7 @@ struct WarpScanShfl * Initial value to seed the scan (uniform across warp) */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void UpdatePartial( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void UpdatePartial( T input, T& inclusive, T& exclusive, T& warp_aggregate, ScanOpT scan_op, int valid_items, T initial_value) { // Get aggregate (excluding initial_value) diff --git a/cub/cub/warp/specializations/warp_scan_smem.cuh b/cub/cub/warp/specializations/warp_scan_smem.cuh index 913270c34544..ee05a4ba36bc 100644 --- a/cub/cub/warp/specializations/warp_scan_smem.cuh +++ b/cub/cub/warp/specializations/warp_scan_smem.cuh @@ -84,7 +84,7 @@ struct WarpScanSmem ******************************************************************************/ /// Constructor - explicit _CCCL_DEVICE _CCCL_FORCEINLINE WarpScanSmem(TempStorage& temp_storage) + explicit _CCCL_DEVICE_API _CCCL_FORCEINLINE WarpScanSmem(TempStorage& temp_storage) : temp_storage(temp_storage.Alias()) {} @@ -94,7 +94,7 @@ struct WarpScanSmem /// Basic inclusive scan iteration (template unrolled, inductive-case specialization) template - _CCCL_DEVICE _CCCL_FORCEINLINE void ScanStep(T& partial, ScanOp scan_op, constant_t /*step*/) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScanStep(T& partial, ScanOp scan_op, constant_t /*step*/) { constexpr int OFFSET = 1 << STEP; @@ -116,7 +116,7 @@ struct WarpScanSmem /// Basic inclusive scan iteration(template unrolled, base-case specialization) template - _CCCL_DEVICE _CCCL_FORCEINLINE void ScanStep(T& /*partial*/, ScanOp /*scan_op*/, constant_t /*step*/) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScanStep(T& /*partial*/, ScanOp /*scan_op*/, constant_t /*step*/) {} /****************************************************************************** @@ -136,7 +136,7 @@ struct WarpScanSmem * @param[in] src_lane * Which warp lane is to do the broadcasting */ - _CCCL_DEVICE _CCCL_FORCEINLINE T Broadcast(T input, unsigned int src_lane) + _CCCL_DEVICE_API _CCCL_FORCEINLINE T Broadcast(T input, unsigned int src_lane) { if (lane_id == src_lane) { @@ -165,7 +165,7 @@ struct WarpScanSmem * Binary scan operator */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void InclusiveScan(T input, T& inclusive_output, ScanOp scan_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScan(T input, T& inclusive_output, ScanOp scan_op) { if constexpr (::cuda::has_identity_element_v) { @@ -195,7 +195,7 @@ struct WarpScanSmem * Warp-wide aggregate reduction of input items. */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void InclusiveScan(T input, T& inclusive_output, ScanOp scan_op, T& warp_aggregate) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScan(T input, T& inclusive_output, ScanOp scan_op, T& warp_aggregate) { InclusiveScan(input, inclusive_output, scan_op); @@ -225,7 +225,8 @@ struct WarpScanSmem * Number of valid items in warp */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void InclusiveScanPartial(T input, T& inclusive_output, ScanOp scan_op, int valid_items) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void + InclusiveScanPartial(T input, T& inclusive_output, ScanOp scan_op, int valid_items) { // Avoid reading uninitialized memory // TODO(pauleonix): Is there a cheaper way of ensuring no uninitialized reads? @@ -282,7 +283,7 @@ struct WarpScanSmem * Warp-wide aggregate reduction of input items. */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScanPartial(T input, T& inclusive_output, ScanOp scan_op, int valid_items, T& warp_aggregate) { InclusiveScanPartial(input, inclusive_output, scan_op, valid_items); @@ -315,7 +316,7 @@ struct WarpScanSmem * @param[in] is_integer */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Update(T /*input*/, T& inclusive, T& exclusive, ScanOpT /*scan_op*/, IsIntegerT /*is_integer*/) { // initial value unknown @@ -330,7 +331,7 @@ struct WarpScanSmem * @brief Update inclusive and exclusive using input and inclusive (specialized for summation of * integer types) */ - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Update(T input, T& inclusive, T& exclusive, ::cuda::std::plus<> /*scan_op*/, ::cuda::std::true_type /*is_integer*/) { // initial value presumed 0 @@ -342,7 +343,7 @@ struct WarpScanSmem * value */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Update(T /*input*/, T& inclusive, T& exclusive, ScanOpT scan_op, T initial_value, IsIntegerT /*is_integer*/) { inclusive = scan_op(initial_value, inclusive); @@ -361,7 +362,7 @@ struct WarpScanSmem * @brief Update inclusive and exclusive using initial value using input and inclusive * (specialized for summation of integer types) */ - _CCCL_DEVICE _CCCL_FORCEINLINE void Update( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Update( T input, T& inclusive, T& exclusive, @@ -377,7 +378,7 @@ struct WarpScanSmem * @brief Update inclusive, exclusive, and warp aggregate using input and inclusive */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Update(T /*input*/, T& inclusive, T& exclusive, T& warp_aggregate, ScanOpT /*scan_op*/, IsIntegerT /*is_integer*/) { // Initial value presumed to be unknown or identity (either way our padding is correct) @@ -393,7 +394,7 @@ struct WarpScanSmem * @brief Update inclusive, exclusive, and warp aggregate using input and inclusive (specialized * for summation of integer types) */ - _CCCL_DEVICE _CCCL_FORCEINLINE void Update( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Update( T input, T& inclusive, T& exclusive, @@ -415,7 +416,7 @@ struct WarpScanSmem * value */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void Update( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Update( T /*input*/, T& inclusive, T& exclusive, @@ -468,7 +469,7 @@ struct WarpScanSmem * Number of valid items in warp */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void UpdatePartial([[maybe_unused]] T input, T& inclusive, T& exclusive, [[maybe_unused]] ScanOpT scan_op, int valid_items) { if constexpr (::cuda::std::is_integral_v && ::cuda::__is_cuda_std_plus_v) @@ -518,7 +519,7 @@ struct WarpScanSmem * Initial value to seed the scan (uniform across warp) */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void UpdatePartial(T input, T& inclusive, T& exclusive, ScanOpT scan_op, int valid_items, T initial_value) { if (static_cast(lane_id) < valid_items) @@ -559,7 +560,7 @@ struct WarpScanSmem * Number of valid items in warp */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void UpdatePartial( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void UpdatePartial( [[maybe_unused]] T input, T& inclusive, T& exclusive, @@ -617,7 +618,7 @@ struct WarpScanSmem * Initial value to seed the scan (uniform across warp) */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void UpdatePartial( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void UpdatePartial( T input, T& inclusive, T& exclusive, T& warp_aggregate, ScanOpT scan_op, int valid_items, T initial_value) { // Broadcast warp aggregate diff --git a/cub/cub/warp/warp_bitonic_sort.cuh b/cub/cub/warp/warp_bitonic_sort.cuh index 06eaced3c71c..f2ef8ee235f5 100644 --- a/cub/cub/warp/warp_bitonic_sort.cuh +++ b/cub/cub/warp/warp_bitonic_sort.cuh @@ -320,7 +320,7 @@ private: using SortSecondHalfT = WarpBitonicSort; template - _CCCL_DEVICE _CCCL_FORCEINLINE void sort(KeyT* keys, ValueT* values, CompareOp compare_op) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE void sort(KeyT* keys, ValueT* values, CompareOp compare_op) const { SortFirstHalfT{}.template sort(keys, values, compare_op); SortSecondHalfT{}.template sort( @@ -343,7 +343,7 @@ private: //! @param[in,out] values Values to merge (reordered to match key order) //! @param[in] compare_op Comparison functor which returns true if the first argument is ordered before the second template - _CCCL_DEVICE _CCCL_FORCEINLINE void merge(KeyT* keys, ValueT* values, CompareOp compare_op) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE void merge(KeyT* keys, ValueT* values, CompareOp compare_op) const { _CCCL_PRAGMA_UNROLL_FULL() for (int i = 0; i < second_half_len; ++i) @@ -370,7 +370,7 @@ private: } template - _CCCL_DEVICE _CCCL_FORCEINLINE void sort(KeyT* keys, ValueT* values, CompareOp compare_op, int valid_items) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE void sort(KeyT* keys, ValueT* values, CompareOp compare_op, int valid_items) const { if (valid_items > first_half_len * LogicalWarpThreads) { @@ -401,7 +401,7 @@ private: //! @param[in] compare_op Comparison functor which returns true if the first argument is ordered before the second //! @param[in] valid_items Total number of valid items across the logical warp template - _CCCL_DEVICE _CCCL_FORCEINLINE void merge(KeyT* keys, ValueT* values, CompareOp compare_op, int valid_items) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE void merge(KeyT* keys, ValueT* values, CompareOp compare_op, int valid_items) const { _CCCL_PRAGMA_UNROLL_FULL() for (int i = 0; i < second_half_len; ++i) @@ -536,7 +536,7 @@ private: unsigned int member_mask = _CUB::detail::warp_mask(); template - _CCCL_DEVICE _CCCL_FORCEINLINE void sort(KeyT* keys, ValueT* values, CompareOp compare_op) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE void sort(KeyT* keys, ValueT* values, CompareOp compare_op) const { ::cuda::static_for([&](auto stage) { merge_stage(keys, values, compare_op); @@ -544,7 +544,7 @@ private: } template - _CCCL_DEVICE _CCCL_FORCEINLINE void merge(KeyT* keys, ValueT* values, CompareOp compare_op) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE void merge(KeyT* keys, ValueT* values, CompareOp compare_op) const { // A single-lane logical warp holds a single item, so there is nothing to merge. if constexpr (num_stages > 0) @@ -554,7 +554,7 @@ private: } template - _CCCL_DEVICE _CCCL_FORCEINLINE void sort(KeyT* keys, ValueT* values, CompareOp compare_op, int valid_items) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE void sort(KeyT* keys, ValueT* values, CompareOp compare_op, int valid_items) const { ::cuda::static_for([&](auto stage) { merge_stage(keys, values, compare_op, valid_items); @@ -562,7 +562,7 @@ private: } template - _CCCL_DEVICE _CCCL_FORCEINLINE void merge(KeyT* keys, ValueT* values, CompareOp compare_op, int valid_items) const + _CCCL_DEVICE_API _CCCL_FORCEINLINE void merge(KeyT* keys, ValueT* values, CompareOp compare_op, int valid_items) const { if constexpr (num_stages > 0) { @@ -575,7 +575,7 @@ private: //! @tparam Full If true, all items are valid (no boundary checks: ``valid_items`` is not used) //! @tparam Stage Stage index in ``[0, num_stages)`` template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void merge_stage(KeyT* keys, ValueT* values, CompareOp compare_op, int valid_items = -1) const { // Each stage divides the inputs into groups and sorts within each group. diff --git a/cub/cub/warp/warp_exchange.cuh b/cub/cub/warp/warp_exchange.cuh index 8d260de8aebf..f15e1c415bd7 100644 --- a/cub/cub/warp/warp_exchange.cuh +++ b/cub/cub/warp/warp_exchange.cuh @@ -128,7 +128,7 @@ public: * @brief Collective constructor using the specified memory allocation as * temporary storage. */ - explicit _CCCL_DEVICE _CCCL_FORCEINLINE WarpExchange(TempStorage& temp_storage) + explicit _CCCL_DEVICE_API _CCCL_FORCEINLINE WarpExchange(TempStorage& temp_storage) : InternalWarpExchange(temp_storage) {} @@ -192,7 +192,7 @@ public: * blocked arrangements. May be aliased to @p input_items. */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void BlockedToStriped(const InputT (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread]) { InternalWarpExchange::BlockedToStriped(input_items, output_items); @@ -252,7 +252,7 @@ public: * Items from exchange. May be aliased to @p input_items. */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void StripedToBlocked(const InputT (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread]) { InternalWarpExchange::StripedToBlocked(input_items, output_items); @@ -314,7 +314,8 @@ public: * @param[in] ranks Corresponding scatter ranks */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterToStriped(InputT (&items)[ItemsPerThread], OffsetT (&ranks)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void + ScatterToStriped(InputT (&items)[ItemsPerThread], OffsetT (&ranks)[ItemsPerThread]) { InternalWarpExchange::ScatterToStriped(items, ranks); } @@ -382,7 +383,7 @@ public: * Corresponding scatter ranks */ template - _CCCL_DEVICE _CCCL_FORCEINLINE void ScatterToStriped( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScatterToStriped( const InputT (&input_items)[ItemsPerThread], OutputT (&output_items)[ItemsPerThread], OffsetT (&ranks)[ItemsPerThread]) diff --git a/cub/cub/warp/warp_load.cuh b/cub/cub/warp/warp_load.cuh index fd7d133ceb25..29077bc133bd 100644 --- a/cub/cub/warp/warp_load.cuh +++ b/cub/cub/warp/warp_load.cuh @@ -133,7 +133,7 @@ namespace detail #endif // _CCCL_HOSTED() #if _CCCL_HOSTED() -inline ::std::ostream& operator<<(::std::ostream& os, WarpLoadAlgorithm algo) +_CCCL_HOST_API inline ::std::ostream& operator<<(::std::ostream& os, WarpLoadAlgorithm algo) { return os << _CUB::detail::to_string(algo); } @@ -265,24 +265,25 @@ class WarpLoad int linear_tid; - _CCCL_DEVICE _CCCL_FORCEINLINE LoadInternal(TempStorage& /*temp_storage*/, int linear_tid) + _CCCL_DEVICE_API _CCCL_FORCEINLINE LoadInternal(TempStorage& /*temp_storage*/, int linear_tid) : linear_tid(linear_tid) {} template - _CCCL_DEVICE _CCCL_FORCEINLINE void Load(InputIteratorT block_itr, InputT (&items)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Load(InputIteratorT block_itr, InputT (&items)[ItemsPerThread]) { _CUB::LoadDirectBlocked(linear_tid, block_itr, items); } template - _CCCL_DEVICE _CCCL_FORCEINLINE void Load(InputIteratorT block_itr, InputT (&items)[ItemsPerThread], int valid_items) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void + Load(InputIteratorT block_itr, InputT (&items)[ItemsPerThread], int valid_items) { _CUB::LoadDirectBlocked(linear_tid, block_itr, items, valid_items); } template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Load(InputIteratorT block_itr, InputT (&items)[ItemsPerThread], int valid_items, DefaultT oob_default) { _CUB::LoadDirectBlocked(linear_tid, block_itr, items, valid_items, oob_default); @@ -296,24 +297,25 @@ class WarpLoad int linear_tid; - _CCCL_DEVICE _CCCL_FORCEINLINE LoadInternal(TempStorage& /*temp_storage*/, int linear_tid) + _CCCL_DEVICE_API _CCCL_FORCEINLINE LoadInternal(TempStorage& /*temp_storage*/, int linear_tid) : linear_tid(linear_tid) {} template - _CCCL_DEVICE _CCCL_FORCEINLINE void Load(InputIteratorT block_itr, InputT (&items)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Load(InputIteratorT block_itr, InputT (&items)[ItemsPerThread]) { _CUB::LoadDirectStriped(linear_tid, block_itr, items); } template - _CCCL_DEVICE _CCCL_FORCEINLINE void Load(InputIteratorT block_itr, InputT (&items)[ItemsPerThread], int valid_items) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void + Load(InputIteratorT block_itr, InputT (&items)[ItemsPerThread], int valid_items) { _CUB::LoadDirectStriped(linear_tid, block_itr, items, valid_items); } template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Load(InputIteratorT block_itr, InputT (&items)[ItemsPerThread], int valid_items, DefaultT oob_default) { _CUB::LoadDirectStriped(linear_tid, block_itr, items, valid_items, oob_default); @@ -327,41 +329,42 @@ class WarpLoad int linear_tid; - _CCCL_DEVICE _CCCL_FORCEINLINE LoadInternal(TempStorage& /*temp_storage*/, int linear_tid) + _CCCL_DEVICE_API _CCCL_FORCEINLINE LoadInternal(TempStorage& /*temp_storage*/, int linear_tid) : linear_tid(linear_tid) {} - _CCCL_DEVICE _CCCL_FORCEINLINE void Load(InputT* block_ptr, InputT (&items)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Load(InputT* block_ptr, InputT (&items)[ItemsPerThread]) { _CUB::LoadDirectBlockedVectorized(linear_tid, block_ptr, items); } - _CCCL_DEVICE _CCCL_FORCEINLINE void Load(const InputT* block_ptr, InputT (&items)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Load(const InputT* block_ptr, InputT (&items)[ItemsPerThread]) { _CUB::LoadDirectBlockedVectorized(linear_tid, block_ptr, items); } template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Load(CacheModifiedInputIterator block_itr, InputT (&items)[ItemsPerThread]) { _CUB::LoadDirectBlockedVectorized(linear_tid, block_itr.ptr, items); } template - _CCCL_DEVICE _CCCL_FORCEINLINE void Load(InputIteratorT block_itr, InputT (&items)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Load(InputIteratorT block_itr, InputT (&items)[ItemsPerThread]) { _CUB::LoadDirectBlocked(linear_tid, block_itr, items); } template - _CCCL_DEVICE _CCCL_FORCEINLINE void Load(InputIteratorT block_itr, InputT (&items)[ItemsPerThread], int valid_items) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void + Load(InputIteratorT block_itr, InputT (&items)[ItemsPerThread], int valid_items) { _CUB::LoadDirectBlocked(linear_tid, block_itr, items, valid_items); } template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Load(InputIteratorT block_itr, InputT (&items)[ItemsPerThread], int valid_items, DefaultT oob_default) { _CUB::LoadDirectBlocked(linear_tid, block_itr, items, valid_items, oob_default); @@ -382,27 +385,28 @@ class WarpLoad int linear_tid; - _CCCL_DEVICE _CCCL_FORCEINLINE LoadInternal(TempStorage& temp_storage, int linear_tid) + _CCCL_DEVICE_API _CCCL_FORCEINLINE LoadInternal(TempStorage& temp_storage, int linear_tid) : temp_storage(temp_storage.Alias()) , linear_tid(linear_tid) {} template - _CCCL_DEVICE _CCCL_FORCEINLINE void Load(InputIteratorT block_itr, InputT (&items)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Load(InputIteratorT block_itr, InputT (&items)[ItemsPerThread]) { _CUB::LoadDirectStriped(linear_tid, block_itr, items); WarpExchangeT(temp_storage).StripedToBlocked(items, items); } template - _CCCL_DEVICE _CCCL_FORCEINLINE void Load(InputIteratorT block_itr, InputT (&items)[ItemsPerThread], int valid_items) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void + Load(InputIteratorT block_itr, InputT (&items)[ItemsPerThread], int valid_items) { _CUB::LoadDirectStriped(linear_tid, block_itr, items, valid_items); WarpExchangeT(temp_storage).StripedToBlocked(items, items); } template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Load(InputIteratorT block_itr, InputT (&items)[ItemsPerThread], int valid_items, DefaultT oob_default) { _CUB::LoadDirectStriped(linear_tid, block_itr, items, valid_items, oob_default); @@ -425,7 +429,7 @@ class WarpLoad ****************************************************************************/ /// Internal storage allocator - _CCCL_DEVICE _CCCL_FORCEINLINE _TempStorage& PrivateStorage() + _CCCL_DEVICE_API _CCCL_FORCEINLINE _TempStorage& PrivateStorage() { __shared__ _TempStorage private_storage; return private_storage; @@ -450,13 +454,13 @@ public: //! @brief Collective constructor using a private static allocation of //! shared memory as temporary storage. - _CCCL_DEVICE _CCCL_FORCEINLINE WarpLoad() + _CCCL_DEVICE_API _CCCL_FORCEINLINE WarpLoad() : temp_storage(PrivateStorage()) {} //! @brief Collective constructor using the specified memory allocation as //! temporary storage. - _CCCL_DEVICE _CCCL_FORCEINLINE WarpLoad(TempStorage& temp_storage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE WarpLoad(TempStorage& temp_storage) : temp_storage(temp_storage.Alias()) {} @@ -511,7 +515,7 @@ public: //! @param[in] block_itr The thread block's base input iterator for loading from //! @param[out] items Data to load template - _CCCL_DEVICE _CCCL_FORCEINLINE void Load(InputIteratorT block_itr, InputT (&items)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Load(InputIteratorT block_itr, InputT (&items)[ItemsPerThread]) { InternalLoad(temp_storage, linear_tid).Load(block_itr, items); } @@ -566,7 +570,8 @@ public: //! @param[out] items Data to load //! @param[in] valid_items Number of valid items to load template - _CCCL_DEVICE _CCCL_FORCEINLINE void Load(InputIteratorT block_itr, InputT (&items)[ItemsPerThread], int valid_items) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void + Load(InputIteratorT block_itr, InputT (&items)[ItemsPerThread], int valid_items) { InternalLoad(temp_storage, linear_tid).Load(block_itr, items, valid_items); } @@ -624,7 +629,7 @@ public: //! @param[in] valid_items Number of valid items to load //! @param[in] oob_default Default value to assign out-of-bound items template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Load(InputIteratorT block_itr, InputT (&items)[ItemsPerThread], int valid_items, DefaultT oob_default) { InternalLoad(temp_storage, linear_tid).Load(block_itr, items, valid_items, oob_default); diff --git a/cub/cub/warp/warp_merge_sort.cuh b/cub/cub/warp/warp_merge_sort.cuh index 2b26bdbf61bf..690d725b5bc3 100644 --- a/cub/cub/warp/warp_merge_sort.cuh +++ b/cub/cub/warp/warp_merge_sort.cuh @@ -117,17 +117,17 @@ class WarpMergeSort public: WarpMergeSort() = delete; - _CCCL_DEVICE _CCCL_FORCEINLINE WarpMergeSort(typename BlockMergeSortStrategyT::TempStorage& temp_storage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE WarpMergeSort(typename BlockMergeSortStrategyT::TempStorage& temp_storage) : BlockMergeSortStrategyT(temp_storage, _CUB::detail::logical_lane_id()) {} - _CCCL_DEVICE _CCCL_FORCEINLINE unsigned int get_member_mask() const + _CCCL_DEVICE_API _CCCL_FORCEINLINE unsigned int get_member_mask() const { return member_mask; } private: - _CCCL_DEVICE _CCCL_FORCEINLINE void SyncImplementation() const + _CCCL_DEVICE_API _CCCL_FORCEINLINE void SyncImplementation() const { __syncwarp(member_mask); } diff --git a/cub/cub/warp/warp_reduce.cuh b/cub/cub/warp/warp_reduce.cuh index c271466613b5..cec44def3692 100644 --- a/cub/cub/warp/warp_reduce.cuh +++ b/cub/cub/warp/warp_reduce.cuh @@ -200,7 +200,7 @@ public: //! @endrst //! //! @param[in] temp_storage Reference to memory allocation having layout type TempStorage - _CCCL_DEVICE _CCCL_FORCEINLINE WarpReduce(TempStorage& temp_storage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE WarpReduce(TempStorage& temp_storage) : temp_storage{temp_storage.Alias()} {} @@ -245,42 +245,42 @@ public: //! ``3568``, respectively (and is undefined in other threads). //! @endrst //! - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T Sum(T input) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T Sum(T input) { return InternalWarpReduce{temp_storage}.template Reduce(input, LogicalWarpThreads, ::cuda::std::plus<>{}); } _CCCL_TEMPLATE(typename InputType) _CCCL_REQUIRES(detail::is_fixed_size_random_access_range_v) - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T Sum(const InputType& input) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T Sum(const InputType& input) { auto thread_reduction = _CUB::ThreadReduce(input, ::cuda::std::plus<>{}); return InternalWarpReduce{temp_storage}.template Reduce( thread_reduction, LogicalWarpThreads, ::cuda::std::plus<>{}); } - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T Max(T input) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T Max(T input) { return InternalWarpReduce{temp_storage}.template Reduce(input, LogicalWarpThreads, ::cuda::maximum<>{}); } _CCCL_TEMPLATE(typename InputType) _CCCL_REQUIRES(detail::is_fixed_size_random_access_range_v) - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T Max(const InputType& input) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T Max(const InputType& input) { auto thread_reduction = _CUB::ThreadReduce(input, ::cuda::maximum<>{}); return InternalWarpReduce{temp_storage}.template Reduce( thread_reduction, LogicalWarpThreads, ::cuda::maximum<>{}); } - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T Min(T input) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T Min(T input) { return InternalWarpReduce{temp_storage}.template Reduce(input, LogicalWarpThreads, ::cuda::minimum<>{}); } _CCCL_TEMPLATE(typename InputType) _CCCL_REQUIRES(detail::is_fixed_size_random_access_range_v) - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T Min(const InputType& input) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T Min(const InputType& input) { auto thread_reduction = _CUB::ThreadReduce(input, ::cuda::minimum<>{}); return InternalWarpReduce{temp_storage}.template Reduce( @@ -337,19 +337,19 @@ public: //! @param[in] valid_items //! Total number of valid items in the calling thread's logical warp //! (may be less than ``LogicalWarpThreads``) - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T Sum(T input, int valid_items) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T Sum(T input, int valid_items) { // Determine if we don't need bounds checking return InternalWarpReduce{temp_storage}.template Reduce(input, valid_items, ::cuda::std::plus<>{}); } - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T Max(T input, int valid_items) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T Max(T input, int valid_items) { // Determine if we don't need bounds checking return InternalWarpReduce{temp_storage}.template Reduce(input, valid_items, ::cuda::maximum<>{}); } - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T Min(T input, int valid_items) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T Min(T input, int valid_items) { // Determine if we don't need bounds checking return InternalWarpReduce{temp_storage}.template Reduce(input, valid_items, ::cuda::minimum<>{}); @@ -408,7 +408,7 @@ public: //! @param[in] head_flag //! Head flag denoting whether or not `input` is the start of a new segment template - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T HeadSegmentedSum(T input, FlagT head_flag) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T HeadSegmentedSum(T input, FlagT head_flag) { return HeadSegmentedReduce(input, head_flag, ::cuda::std::plus<>{}); } @@ -465,7 +465,7 @@ public: //! @param[in] tail_flag //! Head flag denoting whether or not `input` is the start of a new segment template - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T TailSegmentedSum(T input, FlagT tail_flag) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T TailSegmentedSum(T input, FlagT tail_flag) { return TailSegmentedReduce(input, tail_flag, ::cuda::std::plus<>{}); } @@ -530,14 +530,14 @@ public: //! @param[in] reduction_op //! Binary reduction operator template - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T Reduce(T input, ReductionOp reduction_op) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T Reduce(T input, ReductionOp reduction_op) { return InternalWarpReduce{temp_storage}.template Reduce(input, LogicalWarpThreads, reduction_op); } _CCCL_TEMPLATE(typename InputType, typename ReductionOp) _CCCL_REQUIRES(detail::is_fixed_size_random_access_range_v) - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T Reduce(const InputType& input, ReductionOp reduction_op) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T Reduce(const InputType& input, ReductionOp reduction_op) { auto thread_reduction = _CUB::ThreadReduce(input, reduction_op); return WarpReduce::Reduce(thread_reduction, reduction_op); @@ -605,7 +605,7 @@ public: //! Total number of valid items in the calling thread's logical warp //! (may be less than ``LogicalWarpThreads``) template - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T Reduce(T input, ReductionOp reduction_op, int valid_items) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T Reduce(T input, ReductionOp reduction_op, int valid_items) { return InternalWarpReduce{temp_storage}.template Reduce(input, valid_items, reduction_op); } @@ -670,7 +670,8 @@ public: //! @param[in] reduction_op //! Reduction operator template - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T HeadSegmentedReduce(T input, FlagT head_flag, ReductionOp reduction_op) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T + HeadSegmentedReduce(T input, FlagT head_flag, ReductionOp reduction_op) { return InternalWarpReduce{temp_storage}.template SegmentedReduce(input, head_flag, reduction_op); } @@ -735,7 +736,8 @@ public: //! @param[in] reduction_op //! Reduction operator template - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T TailSegmentedReduce(T input, FlagT tail_flag, ReductionOp reduction_op) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T + TailSegmentedReduce(T input, FlagT tail_flag, ReductionOp reduction_op) { return InternalWarpReduce{temp_storage}.template SegmentedReduce(input, tail_flag, reduction_op); } @@ -754,17 +756,17 @@ public: { using TempStorage = Uninitialized<_TempStorage>; - _CCCL_DEVICE _CCCL_FORCEINLINE InternalWarpReduce(TempStorage& /*temp_storage */) {} + _CCCL_DEVICE_API _CCCL_FORCEINLINE InternalWarpReduce(TempStorage& /*temp_storage */) {} template - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T Reduce(T input, int /* valid_items */, ReductionOp /* reduction_op */) { return input; } template - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T SegmentedReduce(T input, FlagT /* flag */, ReductionOp /* reduction_op */) { return input; @@ -773,99 +775,100 @@ public: using TempStorage = typename InternalWarpReduce::TempStorage; - _CCCL_DEVICE _CCCL_FORCEINLINE WarpReduce(TempStorage& /*temp_storage */) {} + _CCCL_DEVICE_API _CCCL_FORCEINLINE WarpReduce(TempStorage& /*temp_storage */) {} - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T Sum(T input) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T Sum(T input) { return input; } _CCCL_TEMPLATE(typename InputType) _CCCL_REQUIRES(detail::is_fixed_size_random_access_range_v) - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T Sum(const InputType& input) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T Sum(const InputType& input) { return _CUB::ThreadReduce(input, ::cuda::std::plus<>{}); } - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T Sum(T input, int /* valid_items */) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T Sum(T input, int /* valid_items */) { return input; } - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T Max(T input) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T Max(T input) { return input; } _CCCL_TEMPLATE(typename InputType) _CCCL_REQUIRES(detail::is_fixed_size_random_access_range_v) - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T Max(const InputType& input) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T Max(const InputType& input) { return _CUB::ThreadReduce(input, ::cuda::maximum<>{}); } - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T Max(T input, int /* valid_items */) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T Max(T input, int /* valid_items */) { return input; } - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T Min(T input) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T Min(T input) { return input; } _CCCL_TEMPLATE(typename InputType) _CCCL_REQUIRES(detail::is_fixed_size_random_access_range_v) - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T Min(const InputType& input) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T Min(const InputType& input) { return _CUB::ThreadReduce(input, ::cuda::minimum<>{}); } - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T Min(T input, int /* valid_items */) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T Min(T input, int /* valid_items */) { return input; } template - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T HeadSegmentedSum(T input, FlagT /* head_flag */) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T HeadSegmentedSum(T input, FlagT /* head_flag */) { return input; } template - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T TailSegmentedSum(T input, FlagT /* tail_flag */) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T TailSegmentedSum(T input, FlagT /* tail_flag */) { return input; } template - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T Reduce(T input, ReductionOp /* reduction_op */) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T Reduce(T input, ReductionOp /* reduction_op */) { return input; } _CCCL_TEMPLATE(typename InputType, typename ReductionOp) _CCCL_REQUIRES(detail::is_fixed_size_random_access_range_v) - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T Reduce(const InputType& input, ReductionOp reduction_op) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T Reduce(const InputType& input, ReductionOp reduction_op) { return _CUB::ThreadReduce(input, reduction_op); } template - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T Reduce(T input, ReductionOp /* reduction_op */, int /* valid_items */) + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T + Reduce(T input, ReductionOp /* reduction_op */, int /* valid_items */) { return input; } template - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T HeadSegmentedReduce(T input, FlagT /* head_flag */, ReductionOp /* reduction_op */) { return input; } template - [[nodiscard]] _CCCL_DEVICE _CCCL_FORCEINLINE T + [[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE T TailSegmentedReduce(T input, FlagT /* tail_flag */, ReductionOp /* reduction_op */) { return input; diff --git a/cub/cub/warp/warp_reduce_batched.cuh b/cub/cub/warp/warp_reduce_batched.cuh index db4382f2ec28..b772114675a2 100644 --- a/cub/cub/warp/warp_reduce_batched.cuh +++ b/cub/cub/warp/warp_reduce_batched.cuh @@ -115,7 +115,7 @@ private: static constexpr auto max_out_per_thread = ::cuda::ceil_div(Batches, LogicalWarpThreads); template - _CCCL_DEVICE _CCCL_FORCEINLINE void check_constraints() + _CCCL_DEVICE_API _CCCL_FORCEINLINE void check_constraints() { static_assert(detail::is_fixed_size_random_access_range_v, "InputT must support the subscript operator[] and have a compile-time size"); diff --git a/cub/cub/warp/warp_scan.cuh b/cub/cub/warp/warp_scan.cuh index 014493609102..c7db1f1b4266 100644 --- a/cub/cub/warp/warp_scan.cuh +++ b/cub/cub/warp/warp_scan.cuh @@ -182,7 +182,7 @@ public: //! //! @param[in] temp_storage //! Reference to memory allocation having layout type TempStorage - _CCCL_DEVICE _CCCL_FORCEINLINE WarpScan(TempStorage& temp_storage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE WarpScan(TempStorage& temp_storage) : temp_storage(temp_storage.Alias()) {} @@ -234,7 +234,7 @@ public: //! //! @param[out] inclusive_output //! Calling thread's output item. May be aliased with `input`. - _CCCL_DEVICE _CCCL_FORCEINLINE void InclusiveSum(T input, T& inclusive_output) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveSum(T input, T& inclusive_output) { InclusiveScan(input, inclusive_output, ::cuda::std::plus<>{}); } @@ -289,7 +289,7 @@ public: //! //! @param[out] warp_aggregate //! Warp-wide aggregate reduction of input items - _CCCL_DEVICE _CCCL_FORCEINLINE void InclusiveSum(T input, T& inclusive_output, T& warp_aggregate) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveSum(T input, T& inclusive_output, T& warp_aggregate) { InclusiveScan(input, inclusive_output, ::cuda::std::plus<>{}, warp_aggregate); } @@ -344,7 +344,7 @@ public: //! //! @param[out] exclusive_output //! Calling thread's output item. May be aliased with `input`. - _CCCL_DEVICE _CCCL_FORCEINLINE void ExclusiveSum(T input, T& exclusive_output) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveSum(T input, T& exclusive_output) { T initial_value{}; ExclusiveScan(input, exclusive_output, initial_value, ::cuda::std::plus<>{}); @@ -404,7 +404,7 @@ public: //! //! @param[out] warp_aggregate //! Warp-wide aggregate reduction of input items - _CCCL_DEVICE _CCCL_FORCEINLINE void ExclusiveSum(T input, T& exclusive_output, T& warp_aggregate) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveSum(T input, T& exclusive_output, T& warp_aggregate) { T initial_value{}; ExclusiveScan(input, exclusive_output, initial_value, ::cuda::std::plus<>{}, warp_aggregate); @@ -467,7 +467,7 @@ public: //! @param[in] scan_op //! Binary scan operator template - _CCCL_DEVICE _CCCL_FORCEINLINE void InclusiveScan(T input, T& inclusive_output, ScanOp scan_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScan(T input, T& inclusive_output, ScanOp scan_op) { InternalWarpScan(temp_storage).InclusiveScan(input, inclusive_output, scan_op); } @@ -516,7 +516,7 @@ public: //! @param[in] scan_op //! Binary scan operator template - _CCCL_DEVICE _CCCL_FORCEINLINE void InclusiveScan(T input, T& inclusive_output, T initial_value, ScanOp scan_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScan(T input, T& inclusive_output, T initial_value, ScanOp scan_op) { InternalWarpScan internal(temp_storage); @@ -587,7 +587,7 @@ public: //! @param[out] warp_aggregate //! Warp-wide aggregate reduction of input items. template - _CCCL_DEVICE _CCCL_FORCEINLINE void InclusiveScan(T input, T& inclusive_output, ScanOp scan_op, T& warp_aggregate) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScan(T input, T& inclusive_output, ScanOp scan_op, T& warp_aggregate) { InternalWarpScan(temp_storage).InclusiveScan(input, inclusive_output, scan_op, warp_aggregate); } @@ -642,7 +642,7 @@ public: //! @param[out] warp_aggregate //! Warp-wide aggregate reduction of input items. template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScan(T input, T& inclusive_output, T initial_value, ScanOp scan_op, T& warp_aggregate) { InternalWarpScan internal(temp_storage); @@ -721,7 +721,8 @@ public: //! @param[in] valid_items //! Number of valid items in warp template - _CCCL_DEVICE _CCCL_FORCEINLINE void InclusiveScanPartial(T input, T& inclusive_output, ScanOp scan_op, int valid_items) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void + InclusiveScanPartial(T input, T& inclusive_output, ScanOp scan_op, int valid_items) { InternalWarpScan(temp_storage).InclusiveScanPartial(input, inclusive_output, scan_op, valid_items); } @@ -772,7 +773,7 @@ public: //! @param[in] valid_items //! Number of valid items in warp template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScanPartial(T input, T& inclusive_output, T initial_value, ScanOp scan_op, int valid_items) { InternalWarpScan internal(temp_storage); @@ -850,7 +851,7 @@ public: //! @param[out] warp_aggregate //! Warp-wide aggregate reduction of input items. template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScanPartial(T input, T& inclusive_output, ScanOp scan_op, int valid_items, T& warp_aggregate) { InternalWarpScan(temp_storage).InclusiveScanPartial(input, inclusive_output, scan_op, valid_items, warp_aggregate); @@ -910,7 +911,7 @@ public: //! @param[out] warp_aggregate //! Warp-wide aggregate reduction of input items. template - _CCCL_DEVICE _CCCL_FORCEINLINE void InclusiveScanPartial( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void InclusiveScanPartial( T input, T& inclusive_output, T initial_value, ScanOp scan_op, int valid_items, T& warp_aggregate) { InternalWarpScan internal(temp_storage); @@ -985,7 +986,7 @@ public: //! @param[in] scan_op //! Binary scan operator template - _CCCL_DEVICE _CCCL_FORCEINLINE void ExclusiveScan(T input, T& exclusive_output, ScanOp scan_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveScan(T input, T& exclusive_output, ScanOp scan_op) { InternalWarpScan internal(temp_storage); @@ -1054,7 +1055,7 @@ public: //! @param[in] scan_op //! Binary scan operator template - _CCCL_DEVICE _CCCL_FORCEINLINE void ExclusiveScan(T input, T& exclusive_output, T initial_value, ScanOp scan_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveScan(T input, T& exclusive_output, T initial_value, ScanOp scan_op) { InternalWarpScan internal(temp_storage); @@ -1131,7 +1132,7 @@ public: //! @param[out] warp_aggregate //! Warp-wide aggregate reduction of input items template - _CCCL_DEVICE _CCCL_FORCEINLINE void ExclusiveScan(T input, T& exclusive_output, ScanOp scan_op, T& warp_aggregate) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveScan(T input, T& exclusive_output, ScanOp scan_op, T& warp_aggregate) { InternalWarpScan internal(temp_storage); @@ -1209,7 +1210,7 @@ public: //! Warp-wide aggregate reduction of input items //! template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveScan(T input, T& exclusive_output, T initial_value, ScanOp scan_op, T& warp_aggregate) { InternalWarpScan internal(temp_storage); @@ -1288,7 +1289,8 @@ public: //! @param[in] valid_items //! Number of valid items in warp template - _CCCL_DEVICE _CCCL_FORCEINLINE void ExclusiveScanPartial(T input, T& exclusive_output, ScanOp scan_op, int valid_items) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void + ExclusiveScanPartial(T input, T& exclusive_output, ScanOp scan_op, int valid_items) { InternalWarpScan internal(temp_storage); @@ -1360,7 +1362,7 @@ public: //! @param[in] valid_items //! Number of valid items in warp template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveScanPartial(T input, T& exclusive_output, T initial_value, ScanOp scan_op, int valid_items) { InternalWarpScan internal(temp_storage); @@ -1438,7 +1440,7 @@ public: //! @param[out] warp_aggregate //! Warp-wide aggregate reduction of input items template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveScanPartial(T input, T& exclusive_output, ScanOp scan_op, int valid_items, T& warp_aggregate) { InternalWarpScan internal(temp_storage); @@ -1520,7 +1522,7 @@ public: //! Warp-wide aggregate reduction of input items //! template - _CCCL_DEVICE _CCCL_FORCEINLINE void ExclusiveScanPartial( + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ExclusiveScanPartial( T input, T& exclusive_output, T initial_value, ScanOp scan_op, int valid_items, T& warp_aggregate) { InternalWarpScan internal(temp_storage); @@ -1601,7 +1603,7 @@ public: //! @param[in] scan_op //! Binary scan operator template - _CCCL_DEVICE _CCCL_FORCEINLINE void Scan(T input, T& inclusive_output, T& exclusive_output, ScanOp scan_op) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Scan(T input, T& inclusive_output, T& exclusive_output, ScanOp scan_op) { InternalWarpScan internal(temp_storage); @@ -1676,7 +1678,7 @@ public: //! @param[in] scan_op //! Binary scan operator template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Scan(T input, T& inclusive_output, T& exclusive_output, T initial_value, ScanOp scan_op) { InternalWarpScan internal(temp_storage); @@ -1755,7 +1757,7 @@ public: //! @param[in] valid_items //! Number of valid items in warp template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScanPartial(T input, T& inclusive_output, T& exclusive_output, ScanOp scan_op, int valid_items) { InternalWarpScan internal(temp_storage); @@ -1834,7 +1836,7 @@ public: //! @param[in] valid_items //! Number of valid items in warp template - _CCCL_DEVICE _CCCL_FORCEINLINE void + _CCCL_DEVICE_API _CCCL_FORCEINLINE void ScanPartial(T input, T& inclusive_output, T& exclusive_output, T initial_value, ScanOp scan_op, int valid_items) { InternalWarpScan internal(temp_storage); @@ -1895,7 +1897,7 @@ public: //! //! @param[in] src_lane //! Which warp lane is to do the broadcasting - _CCCL_DEVICE _CCCL_FORCEINLINE T Broadcast(T input, unsigned int src_lane) + _CCCL_DEVICE_API _CCCL_FORCEINLINE T Broadcast(T input, unsigned int src_lane) { return InternalWarpScan(temp_storage).Broadcast(input, src_lane); } diff --git a/cub/cub/warp/warp_store.cuh b/cub/cub/warp/warp_store.cuh index aeb65b4895cb..b0a2f41b6114 100644 --- a/cub/cub/warp/warp_store.cuh +++ b/cub/cub/warp/warp_store.cuh @@ -136,7 +136,7 @@ namespace detail } } // namespace detail -inline ::std::ostream& operator<<(::std::ostream& os, WarpStoreAlgorithm algo) +_CCCL_HOST_API inline ::std::ostream& operator<<(::std::ostream& os, WarpStoreAlgorithm algo) { return os << _CUB::detail::to_string(algo); } @@ -268,18 +268,18 @@ class WarpStore int linear_tid; - _CCCL_DEVICE _CCCL_FORCEINLINE StoreInternal(TempStorage& /*temp_storage*/, int linear_tid) + _CCCL_DEVICE_API _CCCL_FORCEINLINE StoreInternal(TempStorage& /*temp_storage*/, int linear_tid) : linear_tid(linear_tid) {} template - _CCCL_DEVICE _CCCL_FORCEINLINE void Store(OutputIteratorT block_itr, T (&items)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Store(OutputIteratorT block_itr, T (&items)[ItemsPerThread]) { _CUB::StoreDirectBlocked(linear_tid, block_itr, items); } template - _CCCL_DEVICE _CCCL_FORCEINLINE void Store(OutputIteratorT block_itr, T (&items)[ItemsPerThread], int valid_items) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Store(OutputIteratorT block_itr, T (&items)[ItemsPerThread], int valid_items) { _CUB::StoreDirectBlocked(linear_tid, block_itr, items, valid_items); } @@ -292,18 +292,18 @@ class WarpStore int linear_tid; - _CCCL_DEVICE _CCCL_FORCEINLINE StoreInternal(TempStorage& /*temp_storage*/, int linear_tid) + _CCCL_DEVICE_API _CCCL_FORCEINLINE StoreInternal(TempStorage& /*temp_storage*/, int linear_tid) : linear_tid(linear_tid) {} template - _CCCL_DEVICE _CCCL_FORCEINLINE void Store(OutputIteratorT block_itr, T (&items)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Store(OutputIteratorT block_itr, T (&items)[ItemsPerThread]) { _CUB::StoreDirectStriped(linear_tid, block_itr, items); } template - _CCCL_DEVICE _CCCL_FORCEINLINE void Store(OutputIteratorT block_itr, T (&items)[ItemsPerThread], int valid_items) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Store(OutputIteratorT block_itr, T (&items)[ItemsPerThread], int valid_items) { _CUB::StoreDirectStriped(linear_tid, block_itr, items, valid_items); } @@ -316,23 +316,23 @@ class WarpStore int linear_tid; - _CCCL_DEVICE _CCCL_FORCEINLINE StoreInternal(TempStorage& /*temp_storage*/, int linear_tid) + _CCCL_DEVICE_API _CCCL_FORCEINLINE StoreInternal(TempStorage& /*temp_storage*/, int linear_tid) : linear_tid(linear_tid) {} - _CCCL_DEVICE _CCCL_FORCEINLINE void Store(T* block_ptr, T (&items)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Store(T* block_ptr, T (&items)[ItemsPerThread]) { _CUB::StoreDirectBlockedVectorized(linear_tid, block_ptr, items); } template - _CCCL_DEVICE _CCCL_FORCEINLINE void Store(OutputIteratorT block_itr, T (&items)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Store(OutputIteratorT block_itr, T (&items)[ItemsPerThread]) { _CUB::StoreDirectBlocked(linear_tid, block_itr, items); } template - _CCCL_DEVICE _CCCL_FORCEINLINE void Store(OutputIteratorT block_itr, T (&items)[ItemsPerThread], int valid_items) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Store(OutputIteratorT block_itr, T (&items)[ItemsPerThread], int valid_items) { _CUB::StoreDirectBlocked(linear_tid, block_itr, items, valid_items); } @@ -352,20 +352,20 @@ class WarpStore int linear_tid; - _CCCL_DEVICE _CCCL_FORCEINLINE StoreInternal(TempStorage& temp_storage, int linear_tid) + _CCCL_DEVICE_API _CCCL_FORCEINLINE StoreInternal(TempStorage& temp_storage, int linear_tid) : temp_storage(temp_storage.Alias()) , linear_tid(linear_tid) {} template - _CCCL_DEVICE _CCCL_FORCEINLINE void Store(OutputIteratorT block_itr, T (&items)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Store(OutputIteratorT block_itr, T (&items)[ItemsPerThread]) { WarpExchangeT(temp_storage).BlockedToStriped(items, items); _CUB::StoreDirectStriped(linear_tid, block_itr, items); } template - _CCCL_DEVICE _CCCL_FORCEINLINE void Store(OutputIteratorT block_itr, T (&items)[ItemsPerThread], int valid_items) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Store(OutputIteratorT block_itr, T (&items)[ItemsPerThread], int valid_items) { WarpExchangeT(temp_storage).BlockedToStriped(items, items); _CUB::StoreDirectStriped(linear_tid, block_itr, items, valid_items); @@ -378,7 +378,7 @@ class WarpStore /// Shared memory storage layout type using _TempStorage _CCCL_NODEBUG = typename InternalStore::TempStorage; - _CCCL_DEVICE _CCCL_FORCEINLINE _TempStorage& PrivateStorage() + _CCCL_DEVICE_API _CCCL_FORCEINLINE _TempStorage& PrivateStorage() { __shared__ _TempStorage private_storage; return private_storage; @@ -396,13 +396,13 @@ public: //! @brief Collective constructor using a private static allocation of shared //! memory as temporary storage. - _CCCL_DEVICE _CCCL_FORCEINLINE WarpStore() + _CCCL_DEVICE_API _CCCL_FORCEINLINE WarpStore() : temp_storage(PrivateStorage()) {} //! @brief Collective constructor using the specified memory allocation as //! temporary storage. - _CCCL_DEVICE _CCCL_FORCEINLINE WarpStore(TempStorage& temp_storage) + _CCCL_DEVICE_API _CCCL_FORCEINLINE WarpStore(TempStorage& temp_storage) : temp_storage(temp_storage.Alias()) {} @@ -466,7 +466,7 @@ public: //! @param[out] block_itr The thread block's base output iterator for storing to //! @param[in] items Data to store template - _CCCL_DEVICE _CCCL_FORCEINLINE void Store(OutputIteratorT block_itr, T (&items)[ItemsPerThread]) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Store(OutputIteratorT block_itr, T (&items)[ItemsPerThread]) { InternalStore(temp_storage, linear_tid).Store(block_itr, items); } @@ -532,7 +532,7 @@ public: //! @param[in] valid_items Number of valid items to write //! template - _CCCL_DEVICE _CCCL_FORCEINLINE void Store(OutputIteratorT block_itr, T (&items)[ItemsPerThread], int valid_items) + _CCCL_DEVICE_API _CCCL_FORCEINLINE void Store(OutputIteratorT block_itr, T (&items)[ItemsPerThread], int valid_items) { InternalStore(temp_storage, linear_tid).Store(block_itr, items, valid_items); } diff --git a/cudax/benchmarks/bench/cuco/common/key_generator.cuh b/cudax/benchmarks/bench/cuco/common/key_generator.cuh index c099e3de1fa0..4c54cd03af48 100644 --- a/cudax/benchmarks/bench/cuco/common/key_generator.cuh +++ b/cudax/benchmarks/bench/cuco/common/key_generator.cuh @@ -44,7 +44,7 @@ struct uniform //! //! @param multiplicity Average number of generated keys that map to each unique key value. //! @throws std::invalid_argument if `multiplicity` is not finite or is less than 1.0. - explicit uniform(double multiplicity) + _CCCL_HOST_API explicit uniform(double multiplicity) : multiplicity{multiplicity} { if (!cuda::std::isfinite(multiplicity) || multiplicity < 1.0) @@ -63,13 +63,13 @@ namespace detail template struct generate_uniform_fn { - __host__ __device__ constexpr generate_uniform_fn(cuda::std::size_t num, Distribution dist, cuda::std::size_t seed) + _CCCL_HOST_DEVICE_API constexpr generate_uniform_fn(cuda::std::size_t num, Distribution dist, cuda::std::size_t seed) : num{num} , dist{dist} , seed{seed} {} - __host__ __device__ constexpr Key operator()(cuda::std::size_t idx) const noexcept + _CCCL_HOST_DEVICE_API constexpr Key operator()(cuda::std::size_t idx) const noexcept { Rng rng; rng.seed(seed + (idx * 1664525ull) + 1013904223ull); @@ -90,11 +90,11 @@ struct generate_uniform_fn template struct dropout_fn { - __host__ __device__ constexpr explicit dropout_fn(cuda::std::size_t num) + _CCCL_HOST_DEVICE_API constexpr explicit dropout_fn(cuda::std::size_t num) : num{num} {} - __host__ __device__ Key operator()(cuda::std::size_t seed) const noexcept + _CCCL_HOST_DEVICE_API Key operator()(cuda::std::size_t seed) const noexcept { Rng rng; thrust::uniform_int_distribution dist{static_cast(num), cuda::std::numeric_limits::max()}; @@ -108,11 +108,11 @@ struct dropout_fn template struct dropout_pred { - __host__ __device__ constexpr explicit dropout_pred(double keep_prob) + _CCCL_HOST_DEVICE_API constexpr explicit dropout_pred(double keep_prob) : keep_prob{keep_prob} {} - __host__ __device__ bool operator()(cuda::std::size_t seed) const noexcept + _CCCL_HOST_DEVICE_API bool operator()(cuda::std::size_t seed) const noexcept { Rng rng; thrust::uniform_real_distribution dist{0.0, 1.0}; // NOLINT(misc-const-correctness) @@ -136,7 +136,7 @@ public: //! Constructs a key generator with the given seed. //! //! @param seed Seed used to initialize the generator state. - explicit key_generator(cuda::std::uint32_t seed = defaults::seed) + _CCCL_HOST_API explicit key_generator(cuda::std::uint32_t seed = defaults::seed) : rng{seed} {} @@ -149,7 +149,7 @@ public: //! @param out_end End of the output key range. //! @throws std::invalid_argument if `Distribution` is not a supported distribution tag. template - void generate(Distribution dist, OutputIt out_begin, OutputIt out_end) + _CCCL_HOST_API void generate(Distribution dist, OutputIt out_begin, OutputIt out_end) { generate(dist, out_begin, out_end, thrust::device); } @@ -165,7 +165,7 @@ public: //! @param exec_policy Execution policy used for the underlying Thrust algorithms. //! @throws std::invalid_argument if `Distribution` is not a supported distribution tag. template - void generate(Distribution dist, OutputIt out_begin, OutputIt out_end, ExecPolicy exec_policy) + _CCCL_HOST_API void generate(Distribution dist, OutputIt out_begin, OutputIt out_end, ExecPolicy exec_policy) { using value_type = typename cuda::std::iterator_traits::value_type; @@ -204,7 +204,7 @@ public: //! @param keep_prob Probability of keeping each original key. Must be in `[0, 1]`. //! @throws std::invalid_argument if `keep_prob` is outside `[0, 1]`. template - void dropout(InOutIt begin, InOutIt end, double keep_prob) + _CCCL_HOST_API void dropout(InOutIt begin, InOutIt end, double keep_prob) { dropout(begin, end, keep_prob, thrust::device); } @@ -223,7 +223,7 @@ public: //! @param exec_policy Execution policy used for the underlying Thrust algorithms. //! @throws std::invalid_argument if `keep_prob` is outside `[0, 1]`. template - void dropout(InOutIt begin, InOutIt end, double keep_prob, ExecPolicy exec_policy) + _CCCL_HOST_API void dropout(InOutIt begin, InOutIt end, double keep_prob, ExecPolicy exec_policy) { using value_type = typename cuda::std::iterator_traits::value_type; @@ -262,7 +262,7 @@ private: //! @return Distribution tag initialized from the benchmark state. //! @throws std::invalid_argument if `Distribution` is not a supported distribution tag. template -Distribution dist_from_state(nvbench::state const& state) +_CCCL_HOST_API Distribution dist_from_state(nvbench::state const& state) { if constexpr (cuda::std::is_same_v) { diff --git a/cudax/include/cuda/experimental/__places/cute_partition.cuh b/cudax/include/cuda/experimental/__places/cute_partition.cuh index 004196c3ca8b..5e57bafe0b37 100644 --- a/cudax/include/cuda/experimental/__places/cute_partition.cuh +++ b/cudax/include/cuda/experimental/__places/cute_partition.cuh @@ -144,7 +144,7 @@ public: * coordinates outside [lo, hi) are skipped by the parallel_for * loops (predication: interior regions and padding phantoms alike) */ - _CCCL_HOST_DEVICE cute_sub_shape( + _CCCL_HOST_DEVICE_API cute_sub_shape( ::cuda::std::span local_leaves, size_t offset, dim4 padded_dims, @@ -163,7 +163,7 @@ public: } //! Whether the given coordinates are within the iterated region - _CCCL_HOST_DEVICE bool contains(const coords_t& coords) const + _CCCL_HOST_DEVICE_API bool contains(const coords_t& coords) const { for (size_t d = 0; d < rank; d++) { @@ -176,7 +176,7 @@ public: } //! Number of elements this place owns - _CCCL_HOST_DEVICE size_t size() const + _CCCL_HOST_DEVICE_API size_t size() const { size_t n = 1; for (size_t k = 0; k < num_leaves_; k++) @@ -187,7 +187,7 @@ public: } //! Global tensor coordinates of the place's index-th element - _CCCL_HOST_DEVICE coords_t index_to_coords(size_t index) const + _CCCL_HOST_DEVICE_API coords_t index_to_coords(size_t index) const { // Local index -> linear element index (mixed radix over the local leaves) size_t linear = offset_; @@ -229,7 +229,7 @@ class static_cute_sub_shape public: using coords_t = ::cuda::std::array; - _CCCL_HOST_DEVICE static_cute_sub_shape( + _CCCL_HOST_DEVICE_API static_cute_sub_shape( const ::cuda::std::array& local_leaves, size_t offset, dim4 padded_dims, @@ -242,7 +242,7 @@ public: , hi_(hi) {} - _CCCL_HOST_DEVICE bool contains(const coords_t& coords) const + _CCCL_HOST_DEVICE_API bool contains(const coords_t& coords) const { for (size_t d = 0; d < rank; d++) { @@ -254,7 +254,7 @@ public: return true; } - _CCCL_HOST_DEVICE size_t size() const + _CCCL_HOST_DEVICE_API size_t size() const { size_t n = 1; _CCCL_PRAGMA_UNROLL_FULL() @@ -265,7 +265,7 @@ public: return n; } - _CCCL_HOST_DEVICE coords_t index_to_coords(size_t index) const + _CCCL_HOST_DEVICE_API coords_t index_to_coords(size_t index) const { size_t linear = offset_; _CCCL_PRAGMA_UNROLL_FULL() @@ -321,7 +321,7 @@ public: * Throws std::invalid_argument unless the two modes together tile the * padded space exactly (bijectivity - validated in O(leaves)). */ - cute_partition_descriptor( + _CCCL_HOST_API cute_partition_descriptor( const ::std::vector& place_leaves, const ::std::vector& place_axes, const ::std::vector& local_leaves, @@ -366,19 +366,19 @@ public: } //! True tensor extents (the predicate for the padded space) - _CCCL_HOST_DEVICE const dim4& true_dims() const + _CCCL_HOST_DEVICE_API const dim4& true_dims() const { return true_dims_; } //! Padded tensor extents the leaf strides refer to - _CCCL_HOST_DEVICE const dim4& padded_dims() const + _CCCL_HOST_DEVICE_API const dim4& padded_dims() const { return padded_dims_; } //! Extents of the grid of places - _CCCL_HOST_DEVICE const dim4& grid_dims() const + _CCCL_HOST_DEVICE_API const dim4& grid_dims() const { return grid_dims_; } @@ -397,7 +397,7 @@ public: * padded space aligned to the smallest place-leaf stride). */ template - bool for_each_owner_byte_run(size_t elemsize, size_t max_runs, F&& emit) const + _CCCL_HOST_API bool for_each_owner_byte_run(size_t elemsize, size_t max_runs, F&& emit) const { const size_t t0 = true_dims_.get(0); const size_t t1 = true_dims_.get(1); @@ -484,14 +484,14 @@ public: * * Returns 0 when there is no place mode (single-owner layouts). */ - size_t min_owner_run_bytes(size_t elemsize) const + _CCCL_HOST_API size_t min_owner_run_bytes(size_t elemsize) const { return min_owner_run_elems() * elemsize; } //! The same granularity in ELEMENTS of the padded space (the walk's //! native unit); 0 means no ownership boundary at all (single owner). - size_t min_owner_run_elems() const + _CCCL_HOST_API size_t min_owner_run_elems() const { // Leaves with extent <= 1 never change the owning coordinate (owner() // skips them); including them would drag the minimum down -- extent-1 @@ -535,7 +535,7 @@ public: * exceed max_runs (dense sub-block interleavings such as * element-cyclic): callers fall back to sampled majority. */ - ::cuda::std::optional<::std::vector> + _CCCL_HOST_API ::cuda::std::optional<::std::vector> try_block_owners(size_t block_size_bytes, size_t elemsize, size_t* misplaced_bytes, size_t max_runs = 0) const { _CCCL_ASSERT(elemsize > 0 && block_size_bytes >= elemsize, "invalid block geometry"); @@ -669,7 +669,7 @@ public: * * @param max_runs walk budget, as in try_block_owners (0 = auto) */ - ::cuda::std::optional<::std::vector> + _CCCL_HOST_API ::cuda::std::optional<::std::vector> try_block_runs(size_t block_size_bytes, size_t elemsize, size_t max_runs = 0) const { _CCCL_ASSERT(elemsize > 0 && block_size_bytes >= elemsize, "invalid block geometry"); @@ -706,19 +706,19 @@ public: } //! Leaves of the place mode (leaf 0 fastest) - _CCCL_HOST_DEVICE ::cuda::std::span place_leaves() const + _CCCL_HOST_DEVICE_API ::cuda::std::span place_leaves() const { return {place_leaves_.data(), num_place_leaves_}; } //! Grid axis associated with each place leaf - _CCCL_HOST_DEVICE ::cuda::std::span place_axes() const + _CCCL_HOST_DEVICE_API ::cuda::std::span place_axes() const { return {place_axes_.data(), num_place_leaves_}; } //! Leaves of the local mode (leaf 0 fastest) - _CCCL_HOST_DEVICE ::cuda::std::span local_leaves() const + _CCCL_HOST_DEVICE_API ::cuda::std::span local_leaves() const { return {local_leaves_.data(), num_local_leaves_}; } @@ -726,7 +726,7 @@ public: //! Number of places the partition distributes over (product of place //! extents; grid axes not bound to any dimension receive coordinate 0 and //! do not count) - _CCCL_HOST_DEVICE size_t num_places() const + _CCCL_HOST_DEVICE_API size_t num_places() const { size_t p = 1; for (size_t k = 0; k < num_place_leaves_; k++) @@ -737,7 +737,7 @@ public: } //! Number of padded elements owned by each place (product of local extents) - _CCCL_HOST_DEVICE size_t tiles_per_place() const + _CCCL_HOST_DEVICE_API size_t tiles_per_place() const { size_t n = 1; for (size_t k = 0; k < num_local_leaves_; k++) @@ -753,7 +753,7 @@ public: * Total on all true coordinates (true extents never exceed the padded * ones); grid axes not bound to any dimension get coordinate 0. */ - _CCCL_HOST_DEVICE pos4 owner(pos4 data_coords) const + _CCCL_HOST_DEVICE_API pos4 owner(pos4 data_coords) const { const size_t linear = padded_dims_.get_index(data_coords); @@ -790,7 +790,7 @@ public: * @param grid_dims Extents of the grid (must match the partition's) */ template - auto apply(const S& s, pos4 place_position, dim4 grid_dims) const + _CCCL_HOST_API auto apply(const S& s, pos4 place_position, dim4 grid_dims) const { constexpr size_t rank = S::rank(); validate_iteration_rank(); @@ -829,7 +829,7 @@ public: * @param grid_dims Extents of the grid (must match the partition's) */ template - auto apply(const box& b, pos4 place_position, dim4 grid_dims) const + _CCCL_HOST_API auto apply(const box& b, pos4 place_position, dim4 grid_dims) const { validate_iteration_rank(); @@ -855,7 +855,7 @@ public: * element, given the place's linear index in place-mode order (leaf 0 * fastest) */ - size_t place_offset(size_t place_index) const + _CCCL_HOST_API size_t place_offset(size_t place_index) const { if (place_index >= num_places()) { @@ -871,7 +871,7 @@ public: } //! Structural comparison (used for data place ordering) - int cmp(const cute_partition_descriptor& o) const + _CCCL_HOST_API int cmp(const cute_partition_descriptor& o) const { const auto cmp_sizes = [](size_t a, size_t b) { return (a < b) ? -1 : (a > b) ? 1 : 0; @@ -937,19 +937,19 @@ public: return 0; } - bool operator==(const cute_partition_descriptor& o) const + _CCCL_HOST_API bool operator==(const cute_partition_descriptor& o) const { return cmp(o) == 0; } - bool operator!=(const cute_partition_descriptor& o) const + _CCCL_HOST_API bool operator!=(const cute_partition_descriptor& o) const { return !(*this == o); } private: template - void validate_iteration_rank() const + _CCCL_HOST_API void validate_iteration_rank() const { static_assert(rank <= 4, "cute_partition supports at most four-dimensional iteration"); for (size_t d = rank; d < 4; d++) @@ -964,7 +964,7 @@ private: } template - cute_sub_shape apply_region( + _CCCL_HOST_API cute_sub_shape apply_region( const ::cuda::std::array& lo, const ::cuda::std::array& hi, pos4 place_position, @@ -989,7 +989,7 @@ private: return cute_sub_shape(local_leaves(), offset, padded_dims_, lo, hi); } - void validate() const + _CCCL_HOST_API void validate() const { for (size_t k = 0; k < num_place_leaves_; k++) { @@ -1123,7 +1123,7 @@ struct block_cyclic_dim_spec template struct partition_spec { - explicit partition_spec(Specs... values_) + _CCCL_HOST_API explicit partition_spec(Specs... values_) : values(mv(values_)...) {} @@ -1142,7 +1142,7 @@ template inline constexpr cyclic_dim_spec cyclic{}; template -block_cyclic_dim_spec block_cyclic(size_t block) +_CCCL_HOST_API block_cyclic_dim_spec block_cyclic(size_t block) { return {block}; } @@ -1194,7 +1194,7 @@ public: static constexpr size_t place_leaf_count_v = num_place_leaves; static constexpr size_t local_leaf_count_v = num_local_leaves; - explicit cute_partition(const cute_partition_descriptor& descriptor) + _CCCL_HOST_API explicit cute_partition(const cute_partition_descriptor& descriptor) : padded_dims_(descriptor.padded_dims()) , true_dims_(descriptor.true_dims()) , grid_dims_(descriptor.grid_dims()) @@ -1219,37 +1219,37 @@ public: } } - _CCCL_HOST_DEVICE const dim4& true_dims() const + _CCCL_HOST_DEVICE_API const dim4& true_dims() const { return true_dims_; } - _CCCL_HOST_DEVICE const dim4& padded_dims() const + _CCCL_HOST_DEVICE_API const dim4& padded_dims() const { return padded_dims_; } - _CCCL_HOST_DEVICE const dim4& grid_dims() const + _CCCL_HOST_DEVICE_API const dim4& grid_dims() const { return grid_dims_; } - _CCCL_HOST_DEVICE ::cuda::std::span place_leaves() const + _CCCL_HOST_DEVICE_API ::cuda::std::span place_leaves() const { return {place_leaves_.data(), num_place_leaves}; } - _CCCL_HOST_DEVICE ::cuda::std::span place_axes() const + _CCCL_HOST_DEVICE_API ::cuda::std::span place_axes() const { return {place_axes_.data(), num_place_leaves}; } - _CCCL_HOST_DEVICE ::cuda::std::span local_leaves() const + _CCCL_HOST_DEVICE_API ::cuda::std::span local_leaves() const { return {local_leaves_.data(), num_local_leaves}; } - _CCCL_HOST_DEVICE size_t num_places() const + _CCCL_HOST_DEVICE_API size_t num_places() const { size_t result = 1; _CCCL_PRAGMA_UNROLL_FULL() @@ -1260,7 +1260,7 @@ public: return result; } - _CCCL_HOST_DEVICE size_t tiles_per_place() const + _CCCL_HOST_DEVICE_API size_t tiles_per_place() const { size_t result = 1; _CCCL_PRAGMA_UNROLL_FULL() @@ -1271,7 +1271,7 @@ public: return result; } - _CCCL_HOST_DEVICE pos4 owner(pos4 data_coords) const + _CCCL_HOST_DEVICE_API pos4 owner(pos4 data_coords) const { const size_t linear = padded_dims_.get_index(data_coords); ssize_t place_coord[4] = {0, 0, 0, 0}; @@ -1291,7 +1291,7 @@ public: } template - auto apply(const S& s, pos4 place_position, dim4 grid_dims) const + _CCCL_HOST_API auto apply(const S& s, pos4 place_position, dim4 grid_dims) const { constexpr size_t shape_rank = S::rank(); static_assert(shape_rank == rank, "the task shape rank must match the cute_partition rank"); @@ -1311,7 +1311,7 @@ public: } template - auto apply(const box& b, pos4 place_position, dim4 grid_dims) const + _CCCL_HOST_API auto apply(const box& b, pos4 place_position, dim4 grid_dims) const { static_assert(dims == rank, "the task box rank must match the cute_partition rank"); @@ -1332,7 +1332,7 @@ public: return apply_region(lo, hi, place_position, grid_dims); } - size_t place_offset(size_t place_index) const + _CCCL_HOST_API size_t place_offset(size_t place_index) const { if (place_index >= num_places()) { @@ -1348,7 +1348,7 @@ public: return offset; } - cute_partition_descriptor descriptor() const + _CCCL_HOST_API cute_partition_descriptor descriptor() const { return cute_partition_descriptor( ::std::vector(place_leaves_.begin(), place_leaves_.end()), @@ -1359,7 +1359,7 @@ public: grid_dims_); } - int cmp(const cute_partition& other) const + _CCCL_HOST_API int cmp(const cute_partition& other) const { const auto cmp_sizes = [](size_t a, size_t b) { return (a < b) ? -1 : (a > b) ? 1 : 0; @@ -1418,18 +1418,18 @@ public: return 0; } - bool operator==(const cute_partition& other) const + _CCCL_HOST_API bool operator==(const cute_partition& other) const { return cmp(other) == 0; } - bool operator!=(const cute_partition& other) const + _CCCL_HOST_API bool operator!=(const cute_partition& other) const { return !(*this == other); } private: - static_cute_sub_shape apply_region( + _CCCL_HOST_API static_cute_sub_shape apply_region( const ::cuda::std::array& lo, const ::cuda::std::array& hi, pos4 place_position, @@ -1474,7 +1474,7 @@ private: * @param spec One entry per tensor dimension (at most 4) * @param grid_dims Extents of the grid of places */ -inline cute_partition_descriptor +_CCCL_HOST_API inline cute_partition_descriptor make_partition_descriptor(dim4 true_dims, const ::std::vector& spec, dim4 grid_dims) { if (spec.size() > 4) @@ -1597,25 +1597,25 @@ make_partition_descriptor(dim4 true_dims, const ::std::vector& spec, d grid_dims); } -inline dim_spec __make_runtime_dim_spec(whole_dim_spec) +_CCCL_HOST_API inline dim_spec __make_runtime_dim_spec(whole_dim_spec) { return {}; } template -dim_spec __make_runtime_dim_spec(blocked_dim_spec) +_CCCL_HOST_API dim_spec __make_runtime_dim_spec(blocked_dim_spec) { return {dim_policy::blocked, axis, 0}; } template -dim_spec __make_runtime_dim_spec(cyclic_dim_spec) +_CCCL_HOST_API dim_spec __make_runtime_dim_spec(cyclic_dim_spec) { return {dim_policy::cyclic, axis, 0}; } template -dim_spec __make_runtime_dim_spec(block_cyclic_dim_spec spec) +_CCCL_HOST_API dim_spec __make_runtime_dim_spec(block_cyclic_dim_spec spec) { return {dim_policy::block_cyclic, axis, spec.block}; } @@ -1647,7 +1647,7 @@ inline constexpr bool __has_get_data_dims_v< * @return The partition descriptor built from `shape.get_data_dims()` */ template >> -cute_partition_descriptor +_CCCL_HOST_API cute_partition_descriptor make_partition_descriptor(const Shape& shape, const ::std::vector& spec, dim4 grid_dims) { return ::cuda::experimental::places::make_partition_descriptor(shape.get_data_dims(), spec, grid_dims); @@ -1657,7 +1657,7 @@ make_partition_descriptor(const Shape& shape, const ::std::vector& spe * @brief Build a statically shaped partition from typed dimension specs */ template -auto make_partition(dim4 true_dims, partition_spec spec, dim4 grid_dims) +_CCCL_HOST_API auto make_partition(dim4 true_dims, partition_spec spec, dim4 grid_dims) { constexpr size_t rank = sizeof...(Specs); constexpr size_t num_place_leaves = (__partition_spec_traits::num_place_leaves + ... + 0); @@ -1699,7 +1699,7 @@ auto make_partition(dim4 true_dims, partition_spec spec, dim4 grid_dim * @return The statically shaped `cute_partition` built from `shape.get_data_dims()` */ template >> -auto make_partition(const Shape& shape, partition_spec spec, dim4 grid_dims) +_CCCL_HOST_API auto make_partition(const Shape& shape, partition_spec spec, dim4 grid_dims) { return ::cuda::experimental::places::make_partition(shape.get_data_dims(), spec, grid_dims); } @@ -1708,7 +1708,7 @@ auto make_partition(const Shape& shape, partition_spec spec, dim4 grid * @brief Build an expert statically shaped partition from exact leaf arrays */ template -cute_partition make_partition( +_CCCL_HOST_API cute_partition make_partition( const ::cuda::std::array& place_leaves, const ::cuda::std::array& place_axes, const ::cuda::std::array& local_leaves, @@ -1744,7 +1744,7 @@ cute_partition make_partition( * statistics: the sample counters then hold byte counts); falls back to the * sampled majority vote for layouts denser than the placement blocks. */ -inline auto make_partition_placement_provider( +_CCCL_HOST_API inline auto make_partition_placement_provider( const cute_partition_descriptor& partition, dim4 data_dims, size_t total_size, @@ -1798,7 +1798,7 @@ inline auto make_partition_placement_provider( * is an estimate. `probes` only affects that fallback. */ template -[[nodiscard]] localized_stats evaluate_localized_placement( +[[nodiscard]] _CCCL_HOST_API localized_stats evaluate_localized_placement( const exec_place& grid, const Partition& partition, size_t elemsize, @@ -1862,7 +1862,7 @@ template class data_place_cute_composite final : public data_place_interface { public: - data_place_cute_composite(exec_place grid, cute_partition_descriptor partition) + _CCCL_HOST_API data_place_cute_composite(exec_place grid, cute_partition_descriptor partition) : grid_(mv(grid)) , partition_(mv(partition)) { @@ -1872,32 +1872,32 @@ public: } } - bool is_resolved() const override + _CCCL_HOST_API bool is_resolved() const override { return true; } - bool is_composite() const override + _CCCL_HOST_API bool is_composite() const override { return true; } - int get_device_ordinal() const override + _CCCL_HOST_API int get_device_ordinal() const override { return data_place_interface::composite; } - ::std::string to_string() const override + _CCCL_HOST_API ::std::string to_string() const override { return "composite_cute"; } - size_t hash() const override + _CCCL_HOST_API size_t hash() const override { _CCCL_THROW(::std::logic_error, "hash() not supported for composite data_place"); } - int cmp(const data_place_interface& other) const override + _CCCL_HOST_API int cmp(const data_place_interface& other) const override { if (typeid(*this) != typeid(other)) { @@ -1916,14 +1916,14 @@ public: return (grid_ < o.grid_) ? -1 : 1; } - void* allocate(::cuda::std::ptrdiff_t, cudaStream_t) const override + _CCCL_HOST_API void* allocate(::cuda::std::ptrdiff_t, cudaStream_t) const override { _CCCL_THROW(::std::runtime_error, "cute-partition composite data_place cannot allocate from a byte count alone: use allocate_nd with the " "partition's true extents or allocate through logical data"); } - void* allocate_nd(dim4 data_dims, size_t elemsize, cudaStream_t) const override + _CCCL_HOST_API void* allocate_nd(dim4 data_dims, size_t elemsize, cudaStream_t) const override { // A padded partition is specific to one tensor: the requested extents // must be the ones the partition was built for. @@ -1945,27 +1945,27 @@ public: return ptr; } - void deallocate(void* ptr, size_t, cudaStream_t) const override + _CCCL_HOST_API void deallocate(void* ptr, size_t, cudaStream_t) const override { deallocate_composite_data_place(ptr); } - bool allocation_is_stream_ordered() const override + _CCCL_HOST_API bool allocation_is_stream_ordered() const override { return false; } - ::std::shared_ptr get_affine_exec_impl() const override + _CCCL_HOST_API ::std::shared_ptr get_affine_exec_impl() const override { return grid_.get_impl(); } - const cute_partition_descriptor& get_partition() const + _CCCL_HOST_API const cute_partition_descriptor& get_partition() const { return partition_; } - const exec_place& get_grid() const + _CCCL_HOST_API const exec_place& get_grid() const { return grid_; } @@ -1978,7 +1978,7 @@ private: /** * @brief Create a composite data place backed by an erased partition */ -inline data_place make_composite_data_place(const exec_place& grid, cute_partition_descriptor partition) +_CCCL_HOST_API inline data_place make_composite_data_place(const exec_place& grid, cute_partition_descriptor partition) { return data_place(::std::make_shared(grid, mv(partition))); } @@ -1987,8 +1987,8 @@ inline data_place make_composite_data_place(const exec_place& grid, cute_partiti * @brief Create a composite data place from a typed partition */ template -data_place make_composite_data_place(const exec_place& grid, - const cute_partition& partition) +_CCCL_HOST_API data_place make_composite_data_place( + const exec_place& grid, const cute_partition& partition) { return ::cuda::experimental::places::make_composite_data_place(grid, partition.descriptor()); } diff --git a/cudax/include/cuda/experimental/__places/exec/cuda_context.cuh b/cudax/include/cuda/experimental/__places/exec/cuda_context.cuh index 6edd24a60d95..17fb3c62c2b8 100644 --- a/cudax/include/cuda/experimental/__places/exec/cuda_context.cuh +++ b/cudax/include/cuda/experimental/__places/exec/cuda_context.cuh @@ -65,7 +65,7 @@ public: * from the context (via cuCtxGetDevice). * @param pool_size Number of streams in the place's stream pool. */ - exec_place_cuda_ctx_impl(CUcontext ctx, int devid = -1, size_t pool_size = exec_place::impl::pool_size) + _CCCL_HOST_API exec_place_cuda_ctx_impl(CUcontext ctx, int devid = -1, size_t pool_size = exec_place::impl::pool_size) : exec_place_cuda_ctx_impl(ctx, resolve_devid(ctx, devid), stream_pool(pool_size)) {} @@ -77,7 +77,7 @@ public: * @param pool A stream pool to use for this place (shared handle) * @param affine The affine data place for this place */ - exec_place_cuda_ctx_impl(CUcontext ctx, int devid, stream_pool pool, data_place affine) + _CCCL_HOST_API exec_place_cuda_ctx_impl(CUcontext ctx, int devid, stream_pool pool, data_place affine) : exec_place::impl(mv(affine)) , devid_(devid) , driver_context_(ctx) @@ -87,13 +87,13 @@ public: _CCCL_ASSERT(devid_ >= 0, "cuda_ctx exec_place requires a valid device ordinal"); } - ::std::shared_ptr get_place(size_t idx) override + _CCCL_HOST_API ::std::shared_ptr get_place(size_t idx) override { _CCCL_ASSERT(idx == 0, "Index out of bounds for cuda_ctx exec_place"); return shared_from_this(); } - exec_place activate(size_t idx) const override + _CCCL_HOST_API exec_place activate(size_t idx) const override { _CCCL_ASSERT(idx == 0, "Index out of bounds for cuda_ctx exec_place"); @@ -106,7 +106,7 @@ public: return result; } - void deactivate(const exec_place& prev, size_t idx) const override + _CCCL_HOST_API void deactivate(const exec_place& prev, size_t idx) const override { _CCCL_ASSERT(idx == 0, "Index out of bounds for cuda_ctx exec_place"); @@ -116,18 +116,18 @@ public: cuda_try(saved_ctx); } - bool is_device() const override + _CCCL_HOST_API bool is_device() const override { return true; } - ::std::string to_string() const override + _CCCL_HOST_API ::std::string to_string() const override { return "cuda_ctx(ctx=" + ::std::to_string(reinterpret_cast<::std::uintptr_t>(driver_context_)) + " dev=" + ::std::to_string(devid_) + ")"; } - stream_pool& get_stream_pool(bool, exec_place_resources&, const exec_place&) const override + _CCCL_HOST_API stream_pool& get_stream_pool(bool, exec_place_resources&, const exec_place&) const override { // This place carries its own pool and bypasses the registry. The user is // responsible for keeping the underlying CUcontext alive while the pool @@ -135,7 +135,7 @@ public: return pool_; } - int cmp(const exec_place::impl& rhs) const override + _CCCL_HOST_API int cmp(const exec_place::impl& rhs) const override { if (typeid(*this) != typeid(rhs)) { @@ -147,7 +147,7 @@ public: - ::std::less{}(driver_context_, other.driver_context_); } - size_t hash() const override + _CCCL_HOST_API size_t hash() const override { return ::std::hash()(driver_context_); } @@ -162,16 +162,16 @@ protected: public: // Wrap an existing context with no pool: only used to carry the saved // context through the activate/deactivate round trip. - exec_place_cuda_ctx_impl(saved_tag, CUcontext saved_context) + _CCCL_HOST_API exec_place_cuda_ctx_impl(saved_tag, CUcontext saved_context) : driver_context_(saved_context) {} protected: - exec_place_cuda_ctx_impl(CUcontext ctx, int devid, stream_pool pool) + _CCCL_HOST_API exec_place_cuda_ctx_impl(CUcontext ctx, int devid, stream_pool pool) : exec_place_cuda_ctx_impl(ctx, devid, mv(pool), data_place::device(devid)) {} - static int resolve_devid(CUcontext ctx, int devid) + _CCCL_HOST_API static int resolve_devid(CUcontext ctx, int devid) { const ::cuda::__ensure_current_context guard{ctx}; const int context_devid = static_cast(cuda_try()); @@ -188,7 +188,7 @@ private: mutable stream_pool pool_; }; -inline exec_place exec_place::cuda_context(CUcontext ctx, int devid, size_t pool_size) +_CCCL_HOST_API inline exec_place exec_place::cuda_context(CUcontext ctx, int devid, size_t pool_size) { if (ctx == nullptr) { @@ -204,7 +204,7 @@ inline exec_place exec_place::cuda_context(CUcontext ctx, int devid, size_t pool //! RAII holder for the primary context of device 0, used by the unittests below. struct primary_ctx_guard { - primary_ctx_guard() + _CCCL_HOST_API primary_ctx_guard() : dev([] { cuda_try(0); return cuda_try(0); @@ -212,7 +212,7 @@ struct primary_ctx_guard , ctx(cuda_try(dev)) {} - ~primary_ctx_guard() + _CCCL_HOST_API ~primary_ctx_guard() { cuda_try(cuDevicePrimaryCtxRelease(dev)); } diff --git a/cudax/include/cuda/experimental/__places/exec/green_context.cuh b/cudax/include/cuda/experimental/__places/exec/green_context.cuh index da76ae1bbb4e..048ccb5f6ba2 100644 --- a/cudax/include/cuda/experimental/__places/exec/green_context.cuh +++ b/cudax/include/cuda/experimental/__places/exec/green_context.cuh @@ -42,13 +42,13 @@ namespace cuda::experimental::places { /* Get the unique ID associated with a context (overloaded) */ -inline unsigned long long get_cuda_context_id(CUcontext ctx) +_CCCL_HOST_API inline unsigned long long get_cuda_context_id(CUcontext ctx) { return cuda_try(ctx); } /* Get the unique ID associated with a green context (overloaded) */ -inline unsigned long long get_cuda_context_id(CUgreenCtx gctx) +_CCCL_HOST_API inline unsigned long long get_cuda_context_id(CUgreenCtx gctx) { return get_cuda_context_id(cuda_try(gctx)); } @@ -63,32 +63,32 @@ inline unsigned long long get_cuda_context_id(CUgreenCtx gctx) class green_ctx_data_place_impl : public data_place_interface { public: - explicit green_ctx_data_place_impl(green_ctx_view view) + _CCCL_HOST_API explicit green_ctx_data_place_impl(green_ctx_view view) : view_(mv(view)) {} - bool is_resolved() const override + _CCCL_HOST_API bool is_resolved() const override { return true; } - int get_device_ordinal() const override + _CCCL_HOST_API int get_device_ordinal() const override { return view_.devid; } - ::std::string to_string() const override + _CCCL_HOST_API ::std::string to_string() const override { return "green_ctx(dev=" + ::std::to_string(view_.devid) + ", ctx=" + ::std::to_string(get_cuda_context_id(view_.g_ctx)) + ")"; } - size_t hash() const override + _CCCL_HOST_API size_t hash() const override { return hash_all(view_.g_ctx, view_.devid); } - int cmp(const data_place_interface& other) const override + _CCCL_HOST_API int cmp(const data_place_interface& other) const override { if (typeid(*this) != typeid(other)) { @@ -99,12 +99,12 @@ public: return (o.view_ < view_) - (view_ < o.view_); } - const green_ctx_view& get_view() const + _CCCL_HOST_API const green_ctx_view& get_view() const { return view_; } - void* allocate(::std::ptrdiff_t size, cudaStream_t stream) const override + _CCCL_HOST_API void* allocate(::std::ptrdiff_t size, cudaStream_t stream) const override { void* result = nullptr; const int prev_dev = cuda_try(); @@ -126,7 +126,7 @@ public: return result; } - void deallocate(void* ptr, size_t /*size*/, cudaStream_t stream) const override + _CCCL_HOST_API void deallocate(void* ptr, size_t /*size*/, cudaStream_t stream) const override { const int prev_dev = cuda_try(); @@ -146,12 +146,12 @@ public: cuda_try(cudaFreeAsync(ptr, stream)); } - bool allocation_is_stream_ordered() const override + _CCCL_HOST_API bool allocation_is_stream_ordered() const override { return true; } - CUresult mem_create(CUmemGenericAllocationHandle* handle, size_t size) const override + _CCCL_HOST_API CUresult mem_create(CUmemGenericAllocationHandle* handle, size_t size) const override { CUmemAllocationProp prop = {}; prop.type = CU_MEM_ALLOCATION_TYPE_PINNED; @@ -172,7 +172,7 @@ private: * @param gc_view The green context view * @return data_place for the green context */ -inline data_place make_green_ctx_data_place(const green_ctx_view& gc_view) +_CCCL_HOST_API inline data_place make_green_ctx_data_place(const green_ctx_view& gc_view) { return data_place(::std::make_shared(gc_view)); } @@ -184,7 +184,7 @@ class green_context_helper { public: /* Create green contexts with sm_count SMs per context on a specific device (current device by default) */ - green_context_helper(int sm_count, int devid = cuda_try()) + _CCCL_HOST_API green_context_helper(int sm_count, int devid = cuda_try()) : devid(devid) , numsm(sm_count) { @@ -243,30 +243,30 @@ public: } } - green_context_helper() = default; - ~green_context_helper() = default; + _CCCL_HIDE_FROM_ABI green_context_helper() = default; + _CCCL_HIDE_FROM_ABI ~green_context_helper() = default; - size_t get_device_id() const + _CCCL_HOST_API size_t get_device_id() const { return devid; } - CUgreenCtx partition(size_t partition = 0) + _CCCL_HOST_API CUgreenCtx partition(size_t partition = 0) { return ctxs[partition]; } - green_ctx_view get_view(size_t id) + _CCCL_HOST_API green_ctx_view get_view(size_t id) { return green_ctx_view(ctxs[id], pools[id], devid); } - stream_pool& get_pool(size_t gc_id) + _CCCL_HOST_API stream_pool& get_pool(size_t gc_id) { assert(gc_id < pools.size()); return pools[gc_id]; } - size_t get_count() const + _CCCL_HOST_API size_t get_count() const { return ctxs.size(); } @@ -301,19 +301,19 @@ private: * The user is responsible for keeping the underlying CUgreenCtx alive while * the place (and its stream pool) is in use. */ -inline exec_place exec_place::green_ctx(const green_ctx_view& gc_view, bool use_green_ctx_data_place) +_CCCL_HOST_API inline exec_place exec_place::green_ctx(const green_ctx_view& gc_view, bool use_green_ctx_data_place) { CUcontext ctx = cuda_try(gc_view.g_ctx); data_place affine = use_green_ctx_data_place ? make_green_ctx_data_place(gc_view) : data_place::device(gc_view.devid); return exec_place(::std::make_shared(ctx, gc_view.devid, gc_view.pool, mv(affine))); } -inline ::std::shared_ptr green_ctx_data_place_impl::get_affine_exec_impl() const +_CCCL_HOST_API inline ::std::shared_ptr green_ctx_data_place_impl::get_affine_exec_impl() const { return exec_place::green_ctx(view_).get_impl(); } -inline data_place data_place::green_ctx(const green_ctx_view& gc_view) +_CCCL_HOST_API inline data_place data_place::green_ctx(const green_ctx_view& gc_view) { return make_green_ctx_data_place(gc_view); } diff --git a/cudax/include/cuda/experimental/__places/partitions/blocked_partition.cuh b/cudax/include/cuda/experimental/__places/partitions/blocked_partition.cuh index c337dd4d650e..fe45a0959b96 100644 --- a/cudax/include/cuda/experimental/__places/partitions/blocked_partition.cuh +++ b/cudax/include/cuda/experimental/__places/partitions/blocked_partition.cuh @@ -36,10 +36,10 @@ template <::std::ptrdiff_t which_dim = -1> class blocked_partition_custom { public: - blocked_partition_custom() = default; + _CCCL_HIDE_FROM_ABI blocked_partition_custom() = default; template - _CCCL_HOST_DEVICE static auto apply(const box& in, pos4 place_position, dim4 grid_dims) + _CCCL_HOST_DEVICE_API static auto apply(const box& in, pos4 place_position, dim4 grid_dims) { ::std::array<::std::pair<::std::ptrdiff_t, ::std::ptrdiff_t>, dimensions> bounds; size_t target_dim = (which_dim == -1) ? dimensions - 1 : size_t(which_dim); @@ -74,7 +74,7 @@ public: } template - _CCCL_HOST_DEVICE static auto apply(const mdspan_shape_t& in, pos4 place_position, dim4 grid_dims) + _CCCL_HOST_DEVICE_API static auto apply(const mdspan_shape_t& in, pos4 place_position, dim4 grid_dims) { constexpr size_t dimensions = mdspan_shape_t::rank(); @@ -103,7 +103,7 @@ public: return box(bounds); } - _CCCL_HOST_DEVICE static void get_executor(pos4* result, pos4 data_coords, dim4 data_dims, dim4 grid_dims) + _CCCL_HOST_DEVICE_API static void get_executor(pos4* result, pos4 data_coords, dim4 data_dims, dim4 grid_dims) { // Find the largest dimension const size_t rank = data_dims.get_rank(); diff --git a/cudax/include/cuda/experimental/__places/partitions/cyclic_shape.cuh b/cudax/include/cuda/experimental/__places/partitions/cyclic_shape.cuh index 9a0b6440178c..9db735657edc 100644 --- a/cudax/include/cuda/experimental/__places/partitions/cyclic_shape.cuh +++ b/cudax/include/cuda/experimental/__places/partitions/cyclic_shape.cuh @@ -65,15 +65,15 @@ public: } /// Construct and explicit shape from a list of lower and upper bounds - _CCCL_HOST_DEVICE cyclic_shape(const ::std::array& begins_, - const ::std::array& ends_, - const ::std::array& strides_) + _CCCL_HOST_DEVICE_API cyclic_shape(const ::std::array& begins_, + const ::std::array& ends_, + const ::std::array& strides_) : begins(begins_) , ends(ends_) , strides(strides_) {} - _CCCL_HOST_DEVICE void print() const + _CCCL_HOST_DEVICE_API void print() const { printf("CYCLIC SHAPE\n"); for (size_t i = 0; i < dimensions; i++) @@ -85,7 +85,7 @@ public: ///@} /// Get the total number of elements in this explicit shape - _CCCL_HOST_DEVICE ::std::ptrdiff_t size() const + _CCCL_HOST_DEVICE_API ::std::ptrdiff_t size() const { size_t res = 1; for (size_t d = 0; d < dimensions; d++) @@ -97,7 +97,7 @@ public: } // Overload the equality operator to check if two shapes are equal - _CCCL_HOST_DEVICE bool operator==(const cyclic_shape& other) const + _CCCL_HOST_DEVICE_API bool operator==(const cyclic_shape& other) const { for (size_t i = 0; i < dimensions; ++i) { @@ -111,7 +111,7 @@ public: } /// get the dimensionnality of the explicit shape - _CCCL_HOST_DEVICE size_t get_rank() const + _CCCL_HOST_DEVICE_API size_t get_rank() const { return dimensions; } @@ -124,14 +124,14 @@ public: ::std::array current; // Array to store the current position in each dimension public: - _CCCL_HOST_DEVICE iterator(cyclic_shape& s, size_t idx = 0) + _CCCL_HOST_DEVICE_API iterator(cyclic_shape& s, size_t idx = 0) : shape(&s) , index(idx) , current(s.begins) {} // Overload the dereference operator to get the current position - _CCCL_HOST_DEVICE auto& operator*() + _CCCL_HOST_DEVICE_API auto& operator*() { if constexpr (dimensions == 1) { @@ -144,7 +144,7 @@ public: } // Overload the pre-increment operator to move to the next position - _CCCL_HOST_DEVICE iterator& operator++() + _CCCL_HOST_DEVICE_API iterator& operator++() { index++; @@ -166,7 +166,7 @@ public: } // Overload the equality operator to check if two iterators are equal - _CCCL_HOST_DEVICE bool operator==(const iterator& other) const + _CCCL_HOST_DEVICE_API bool operator==(const iterator& other) const { /*printf("EQUALITY TEST index %d %d shape equal ? %s\n", index, other.index, (&shape == &other.shape)?"yes":"no"); */ // printf("check == : index %d %d => %s shape equal ? %s\n", index, other.index, (index == @@ -175,19 +175,19 @@ public: } // Overload the inequality operator to check if two iterators are not equal - _CCCL_HOST_DEVICE bool operator!=(const iterator& other) const + _CCCL_HOST_DEVICE_API bool operator!=(const iterator& other) const { return !(*this == other); } }; // Functions to create the begin and end iterators - _CCCL_HOST_DEVICE iterator begin() + _CCCL_HOST_DEVICE_API iterator begin() { return iterator(*this); } - _CCCL_HOST_DEVICE iterator end() + _CCCL_HOST_DEVICE_API iterator end() { size_t total_positions = 1; for (auto i : each(0, dimensions)) @@ -213,10 +213,10 @@ private: class cyclic_partition { public: - cyclic_partition() = default; + _CCCL_HIDE_FROM_ABI cyclic_partition() = default; template - _CCCL_HOST_DEVICE static auto apply(const box& in, pos4 place_position, dim4 grid_dims) + _CCCL_HOST_DEVICE_API static auto apply(const box& in, pos4 place_position, dim4 grid_dims) { ::std::array begins{}; ::std::array ends{}; @@ -232,7 +232,7 @@ public: } template - _CCCL_HOST_DEVICE static auto apply(const mdspan_shape_t& in, pos4 place_position, dim4 grid_dims) + _CCCL_HOST_DEVICE_API static auto apply(const mdspan_shape_t& in, pos4 place_position, dim4 grid_dims) { constexpr size_t dimensions = mdspan_shape_t::rank(); @@ -253,7 +253,7 @@ public: * distribution the owner of an element is its coordinate modulo the grid * extent, independently in each dimension. */ - _CCCL_HOST_DEVICE static void get_executor(pos4* result, pos4 data_coords, dim4 /*data_dims*/, dim4 grid_dims) + _CCCL_HOST_DEVICE_API static void get_executor(pos4* result, pos4 data_coords, dim4 /*data_dims*/, dim4 grid_dims) { _CCCL_ASSERT(data_coords.x >= 0 && data_coords.y >= 0 && data_coords.z >= 0 && data_coords.t >= 0, "get_executor requires zero-based (non-negative) coordinates"); diff --git a/cudax/include/cuda/experimental/__places/partitions/tiled_partition.cuh b/cudax/include/cuda/experimental/__places/partitions/tiled_partition.cuh index d7504fe150fb..8bca97339cd0 100644 --- a/cudax/include/cuda/experimental/__places/partitions/tiled_partition.cuh +++ b/cudax/include/cuda/experimental/__places/partitions/tiled_partition.cuh @@ -40,14 +40,14 @@ class tiled_mdspan_shape { public: // parts id, nparts ? - tiled_mdspan_shape(const mdspan_shape_t& s, size_t part_id, size_t nparts) + _CCCL_HOST_API tiled_mdspan_shape(const mdspan_shape_t& s, size_t part_id, size_t nparts) : original_shape(s) , part_id(part_id) , nparts(nparts) {} /* The number of elements in this part */ - size_t size() const + _CCCL_HOST_API size_t size() const { _CCCL_ASSERT(mdspan_shape_t::rank() == 1, "Tiled mdspan shape only implemented in 1D yet"); @@ -90,7 +90,7 @@ public: using coords_t = typename mdspan_shape_t::coords_t; - _CCCL_HOST_DEVICE coords_t index_to_coords(size_t index) const + _CCCL_HOST_DEVICE_API coords_t index_to_coords(size_t index) const { // First transform from nparts and part_id to one coordinate const size_t remain = index % tile_size; @@ -116,7 +116,7 @@ private: * function to implement tiled and have the other type deduced. */ template -auto tiled(const mdspan_shape_t s, size_t part_id, size_t nparts) +_CCCL_HOST_API auto tiled(const mdspan_shape_t s, size_t part_id, size_t nparts) { return reserved::tiled_mdspan_shape(s, part_id, nparts); } @@ -127,10 +127,10 @@ class tiled_partition static_assert(tile_size > 0); public: - tiled_partition() = default; + _CCCL_HIDE_FROM_ABI tiled_partition() = default; template - static reserved::tiled_mdspan_shape + _CCCL_HOST_API static reserved::tiled_mdspan_shape apply(const mdspan_shape_t& in, pos4 place_position, dim4 grid_dims) { // TODO assert 1D ! @@ -138,7 +138,7 @@ public: return reserved::tiled_mdspan_shape(in, place_position.x, grid_dims.x); } - _CCCL_HOST_DEVICE static void get_executor(pos4* result, pos4 data_coords, dim4 /*unused*/, dim4 grid_dims) + _CCCL_HOST_DEVICE_API static void get_executor(pos4* result, pos4 data_coords, dim4 /*unused*/, dim4 grid_dims) { assert(grid_dims.x > 0); *result = pos4((data_coords.x / tile_size) % grid_dims.x); diff --git a/cudax/include/cuda/experimental/__places/places.cuh b/cudax/include/cuda/experimental/__places/places.cuh index cfbbcb7109e8..6724985316fe 100644 --- a/cudax/include/cuda/experimental/__places/places.cuh +++ b/cudax/include/cuda/experimental/__places/places.cuh @@ -106,7 +106,7 @@ inline constexpr replicate_over_t replicate_over{}; class data_place { template - static ::std::shared_ptr make_static_instance() + _CCCL_HOST_API static ::std::shared_ptr make_static_instance() { static T instance; // Build the aliasing shared_ptr exactly once and hand out copies. Besides @@ -121,25 +121,25 @@ class data_place } public: - explicit data_place(::std::shared_ptr impl) + _CCCL_HOST_API explicit data_place(::std::shared_ptr impl) : pimpl_(mv(impl)) {} /** * @brief Default constructor. The object is initialized as invalid. */ - data_place() + _CCCL_HOST_API data_place() : pimpl_(make_static_instance()) {} - data_place(const data_place&) = default; - data_place(data_place&&) = default; - data_place& operator=(const data_place&) = default; - data_place& operator=(data_place&&) = default; + _CCCL_HIDE_FROM_ABI data_place(const data_place&) = default; + _CCCL_HIDE_FROM_ABI data_place(data_place&&) = default; + _CCCL_HIDE_FROM_ABI data_place& operator=(const data_place&) = default; + _CCCL_HIDE_FROM_ABI data_place& operator=(data_place&&) = default; /** * @brief Represents an invalid `data_place` object. */ - static data_place invalid() + _CCCL_HOST_API static data_place invalid() { return data_place(make_static_instance()); } @@ -148,7 +148,7 @@ public: * @brief Represents the host CPU as the `data_place` (pinned host memory, or * memory which should be pinned by CUDASTF). */ - static data_place host() + _CCCL_HOST_API static data_place host() { return data_place(make_static_instance()); } @@ -156,14 +156,14 @@ public: /** * @brief Represents a managed memory location as the `data_place`. */ - static data_place managed() + _CCCL_HOST_API static data_place managed() { return data_place(make_static_instance()); } /// This actually does not define a data_place, but means that we should use /// the data place affine to the execution place - static data_place affine() + _CCCL_HOST_API static data_place affine() { return data_place(make_static_instance()); } @@ -172,13 +172,13 @@ public: * @brief Constant representing a placeholder that lets the library automatically select a GPU device as the * `data_place`. */ - static data_place device_auto() + _CCCL_HOST_API static data_place device_auto() { return data_place(make_static_instance()); } /** @brief Data is placed on device with index dev_id. */ - static data_place device(int dev_id = 0) + _CCCL_HOST_API static data_place device(int dev_id = 0) { static int const ndevs = cuda_try(); @@ -198,7 +198,7 @@ public: /** * @brief Select the embedded memory of the current device as `data_place`. */ - static data_place current_device() + _CCCL_HOST_API static data_place current_device() { return device(cuda_try()); } @@ -209,9 +209,9 @@ public: template < typename partitioner_t, typename = ::cuda::std::enable_if_t<::cuda::std::is_class_v> /*, typename scalar_exec_place_t */> - static data_place composite(partitioner_t p, const exec_place& g); + _CCCL_HOST_API static data_place composite(partitioner_t p, const exec_place& g); - static data_place composite(partition_mapper f, const exec_place& grid); + _CCCL_HOST_API static data_place composite(partition_mapper f, const exec_place& grid); /** * @brief Replicated data place: one full copy of the data in the affine @@ -221,7 +221,7 @@ public: * place's affine data place (a plain place, not replicated): a live * replicated place always has at least two instances. */ - static data_place replicated(const exec_place& grid); + _CCCL_HOST_API static data_place replicated(const exec_place& grid); /** * @brief Deferred replicated data place: replicated over the grid of @@ -231,7 +231,7 @@ public: * always replicates over every grid axis (axis grouping requires the * explicit replicate_over overload). */ - static data_place replicated(); + _CCCL_HOST_API static data_place replicated(); /** * @brief Axis-grouped replication: one copy per coordinate of the @@ -245,10 +245,10 @@ public: * grid of the all-axes overload. */ template - static data_place replicated(const exec_place& grid, replicate_over_t); + _CCCL_HOST_API static data_place replicated(const exec_place& grid, replicate_over_t); #if _CCCL_CTK_AT_LEAST(12, 4) - static data_place green_ctx(const green_ctx_view& gc_view); + _CCCL_HOST_API static data_place green_ctx(const green_ctx_view& gc_view); #endif // _CCCL_CTK_AT_LEAST(12, 4) /** @@ -260,7 +260,7 @@ public: static data_place locality_domain(const locality_domain_view& view); static data_place locality_domain(int dev_id, int domain_id); - bool operator==(const data_place& rhs) const + _CCCL_HOST_API bool operator==(const data_place& rhs) const { // Same pointer means same place if (pimpl_.get() == rhs.pimpl_.get()) @@ -270,88 +270,88 @@ public: return pimpl_->cmp(*rhs.pimpl_) == 0; } - bool operator!=(const data_place& rhs) const + _CCCL_HOST_API bool operator!=(const data_place& rhs) const { return !(*this == rhs); } // To use in a ::std::map indexed by data_place - bool operator<(const data_place& rhs) const + _CCCL_HOST_API bool operator<(const data_place& rhs) const { return pimpl_->cmp(*rhs.pimpl_) < 0; } - bool operator>(const data_place& rhs) const + _CCCL_HOST_API bool operator>(const data_place& rhs) const { return rhs < *this; } - bool operator<=(const data_place& rhs) const + _CCCL_HOST_API bool operator<=(const data_place& rhs) const { return !(rhs < *this); } - bool operator>=(const data_place& rhs) const + _CCCL_HOST_API bool operator>=(const data_place& rhs) const { return !(*this < rhs); } // Defined later after data_place_composite is complete - bool is_composite() const; - bool is_replicated() const noexcept; + _CCCL_HOST_API bool is_composite() const; + _CCCL_HOST_API bool is_replicated() const noexcept; //! Number of data instances a dependency at this place resolves to - size_t instance_count() const; + _CCCL_HOST_API size_t instance_count() const; //! Data place of the r-th instance (r < instance_count()); *this when //! the place resolves to a single instance - data_place member(size_t r) const; + _CCCL_HOST_API data_place member(size_t r) const; //! Instance index the given (linear) grid place resolves to; 0 when the //! place resolves to a single instance. With axis-grouped replication, //! places differing only along SHARED axes map to the same instance. - size_t instance_of(size_t place_index) const; + _CCCL_HOST_API size_t instance_of(size_t place_index) const; - bool is_invalid() const + _CCCL_HOST_API bool is_invalid() const { const auto& ref = *pimpl_; return typeid(ref) == typeid(data_place_invalid); } - bool is_host() const + _CCCL_HOST_API bool is_host() const { const auto& ref = *pimpl_; return typeid(ref) == typeid(data_place_host); } - bool is_managed() const + _CCCL_HOST_API bool is_managed() const { const auto& ref = *pimpl_; return typeid(ref) == typeid(data_place_managed); } - bool is_affine() const + _CCCL_HOST_API bool is_affine() const { const auto& ref = *pimpl_; return typeid(ref) == typeid(data_place_affine); } - bool is_device() const + _CCCL_HOST_API bool is_device() const { const auto& ref = *pimpl_; return typeid(ref) == typeid(data_place_device); } - bool is_device_auto() const + _CCCL_HOST_API bool is_device_auto() const { const auto& ref = *pimpl_; return typeid(ref) == typeid(data_place_device_auto); } - bool is_resolved() const + _CCCL_HOST_API bool is_resolved() const { return pimpl_->is_resolved(); } - ::std::string to_string() const + _CCCL_HOST_API ::std::string to_string() const { return pimpl_->to_string(); } @@ -360,7 +360,7 @@ public: * @brief Returns an index guaranteed to be >= 0 (0 for managed CPU, 1 for pinned CPU, 2 for device 0, 3 for device * 1, ...). Requires that `p` is initialized and different from `data_place::invalid()`. */ - friend size_t to_index(const data_place& p) + _CCCL_HOST_API friend size_t to_index(const data_place& p) { const int devid = p.pimpl_->get_device_ordinal(); EXPECT(devid >= -2, "Data place with device id ", devid, " does not refer to a device."); @@ -372,7 +372,7 @@ public: * @brief Inverse of `to_index`: converts an index back to a `data_place`. * Index 0 -> managed, 1 -> host, 2 -> device(0), 3 -> device(1), ... */ - friend data_place from_index(size_t n) + _CCCL_HOST_API friend data_place from_index(size_t n) { if (n == 0) { @@ -389,37 +389,37 @@ public: * @brief Returns the device ordinal (0 = first GPU, 1 = second GPU, ... and by convention the CPU is -1) * Requires that `p` is initialized. */ - friend int device_ordinal(const data_place& p) + _CCCL_HOST_API friend int device_ordinal(const data_place& p) { return p.pimpl_->get_device_ordinal(); } - const partition_mapper& get_partitioner() const + _CCCL_HOST_API const partition_mapper& get_partitioner() const { return pimpl_->get_partitioner(); } // Defined later after exec_place is complete - exec_place affine_exec_place() const; + _CCCL_HOST_API exec_place affine_exec_place() const; /** * @brief Compute a hash value for this data place * * Used by std::hash specialization for unordered containers. */ - size_t hash() const + _CCCL_HOST_API size_t hash() const { return pimpl_->hash(); } - augmented_stream getDataStream(exec_place_resources& res) const; + _CCCL_HOST_API augmented_stream getDataStream(exec_place_resources& res) const; /** * @brief Get the underlying interface pointer * * This is primarily for internal use and backward compatibility. */ - const ::std::shared_ptr& get_impl() const + _CCCL_HOST_API const ::std::shared_ptr& get_impl() const { return pimpl_; } @@ -430,7 +430,7 @@ public: * Standalone call: the place does not need to be activated first, and the * calling thread's current device is left unchanged. */ - CUresult mem_create(CUmemGenericAllocationHandle* handle, size_t size) const + _CCCL_HOST_API CUresult mem_create(CUmemGenericAllocationHandle* handle, size_t size) const { return pimpl_->mem_create(handle, size); } @@ -441,7 +441,7 @@ public: * Standalone call: the place does not need to be activated first, and the * calling thread's current device is left unchanged. */ - void* allocate(::std::ptrdiff_t size, cudaStream_t stream = nullptr) const + _CCCL_HOST_API void* allocate(::std::ptrdiff_t size, cudaStream_t stream = nullptr) const { return pimpl_->allocate(size, stream); } @@ -461,7 +461,7 @@ public: * @throws std::invalid_argument if the product of the extents and elemsize * overflows size_t or exceeds PTRDIFF_MAX */ - void* allocate_nd(dim4 data_dims, size_t elemsize, cudaStream_t stream = nullptr) const + _CCCL_HOST_API void* allocate_nd(dim4 data_dims, size_t elemsize, cudaStream_t stream = nullptr) const { // An unchecked x*y*z*t*elemsize product can wrap: an ordinary place would // then silently return a tiny allocation for an astronomically large @@ -490,7 +490,7 @@ public: * Standalone call: the place does not need to be activated first, and the * calling thread's current device is left unchanged. */ - void deallocate(void* ptr, size_t size, cudaStream_t stream = nullptr) const + _CCCL_HOST_API void deallocate(void* ptr, size_t size, cudaStream_t stream = nullptr) const { pimpl_->deallocate(ptr, size, stream); } @@ -498,7 +498,7 @@ public: /** * @brief Returns true if allocation/deallocation is stream-ordered */ - bool allocation_is_stream_ordered() const + _CCCL_HOST_API bool allocation_is_stream_ordered() const { return pimpl_->allocation_is_stream_ordered(); } @@ -508,9 +508,9 @@ private: }; /** Declarations for unqualified lookup (friends are only found via ADL when a \c data_place argument is present). */ -inline data_place from_index(size_t n); -inline size_t to_index(const data_place& p); -inline int device_ordinal(const data_place& p); +_CCCL_HOST_API inline data_place from_index(size_t n); +_CCCL_HOST_API inline size_t to_index(const data_place& p); +_CCCL_HOST_API inline int device_ordinal(const data_place& p); // Forward declaration class exec_place_scope; @@ -531,12 +531,12 @@ public: class impl : public ::std::enable_shared_from_this { public: - impl() = default; - impl(const impl&) = delete; - impl& operator=(const impl&) = delete; - virtual ~impl() = default; + _CCCL_HIDE_FROM_ABI impl() = default; + impl(const impl&) = delete; + impl& operator=(const impl&) = delete; + _CCCL_HIDE_FROM_ABI virtual ~impl() = default; - explicit impl(data_place place) + _CCCL_HOST_API explicit impl(data_place place) : affine(mv(place)) {} @@ -547,7 +547,7 @@ public: * * For scalar places, returns dim4(1, 1, 1, 1). */ - virtual dim4 get_dims() const + _CCCL_HOST_API virtual dim4 get_dims() const { return dim4(1, 1, 1, 1); } @@ -555,7 +555,7 @@ public: /** * @brief Get the total number of places in this grid */ - virtual size_t size() const + _CCCL_HOST_API virtual size_t size() const { return 1; } @@ -566,7 +566,7 @@ public: * For scalar places, idx must be 0 and returns shared_from_this(). * For grids, returns the impl of the stored sub-place. */ - virtual ::std::shared_ptr get_place(size_t idx); + _CCCL_HOST_API virtual ::std::shared_ptr get_place(size_t idx); // ===== Activation/deactivation (indexed) ===== @@ -585,27 +585,27 @@ public: // ===== Properties ===== - virtual bool is_host() const + _CCCL_HOST_API virtual bool is_host() const { return false; } - virtual bool is_device() const + _CCCL_HOST_API virtual bool is_device() const { return false; } - virtual data_place affine_data_place() const + _CCCL_HOST_API virtual data_place affine_data_place() const { return affine; } - virtual ::std::string to_string() const + _CCCL_HOST_API virtual ::std::string to_string() const { return "exec(" + affine.to_string() + ")"; } - virtual void set_affine_data_place(data_place place) + _CCCL_HOST_API virtual void set_affine_data_place(data_place place) { affine = mv(place); } @@ -616,7 +616,7 @@ public: * @brief Three-way comparison * @return -1 if *this < rhs, 0 if *this == rhs, 1 if *this > rhs */ - virtual int cmp(const impl& rhs) const + _CCCL_HOST_API virtual int cmp(const impl& rhs) const { if (typeid(*this) != typeid(rhs)) { @@ -625,7 +625,7 @@ public: return (rhs.affine < affine) - (affine < rhs.affine); } - virtual size_t hash() const + _CCCL_HOST_API virtual size_t hash() const { return affine.hash(); } @@ -654,7 +654,7 @@ public: * derived overrides that need access to the * public-facing place). */ - [[nodiscard]] virtual stream_pool& + [[nodiscard]] _CCCL_HOST_API virtual stream_pool& get_stream_pool(bool for_computation, exec_place_resources& res, [[maybe_unused]] const exec_place& self) const { auto& slot = res.get(this); @@ -670,7 +670,7 @@ public: }; template - static ::std::shared_ptr make_static_instance() + _CCCL_HOST_API static ::std::shared_ptr make_static_instance() { static T instance; // Build the aliasing shared_ptr exactly once and hand out copies. @@ -684,9 +684,9 @@ public: return handle; } - exec_place() = default; + _CCCL_HIDE_FROM_ABI exec_place() = default; - bool operator==(const exec_place& rhs) const + _CCCL_HOST_API bool operator==(const exec_place& rhs) const { if (pimpl.get() == rhs.pimpl.get()) { @@ -695,32 +695,32 @@ public: return pimpl->cmp(*rhs.pimpl) == 0; } - bool operator!=(const exec_place& rhs) const + _CCCL_HOST_API bool operator!=(const exec_place& rhs) const { return !(*this == rhs); } - bool operator<(const exec_place& rhs) const + _CCCL_HOST_API bool operator<(const exec_place& rhs) const { return pimpl->cmp(*rhs.pimpl) < 0; } - bool operator>(const exec_place& rhs) const + _CCCL_HOST_API bool operator>(const exec_place& rhs) const { return rhs < *this; } - bool operator<=(const exec_place& rhs) const + _CCCL_HOST_API bool operator<=(const exec_place& rhs) const { return !(rhs < *this); } - bool operator>=(const exec_place& rhs) const + _CCCL_HOST_API bool operator>=(const exec_place& rhs) const { return !(*this < rhs); } - size_t hash() const + _CCCL_HOST_API size_t hash() const { return pimpl->hash(); } @@ -732,7 +732,7 @@ public: * * For scalar places (host, single device), returns dim4(1, 1, 1, 1). */ - dim4 get_dims() const + _CCCL_HOST_API dim4 get_dims() const { return pimpl->get_dims(); } @@ -740,7 +740,7 @@ public: /** * @brief Get the total number of places in this grid */ - size_t size() const + _CCCL_HOST_API size_t size() const { return pimpl->size(); } @@ -750,7 +750,7 @@ public: * * For scalar places, idx must be 0 and returns the place itself. */ - exec_place get_place(size_t idx) const + _CCCL_HOST_API exec_place get_place(size_t idx) const { return exec_place(pimpl->get_place(idx)); } @@ -758,7 +758,7 @@ public: /** * @brief Get the sub-place at the given multi-dimensional position */ - exec_place get_place(pos4 p) const + _CCCL_HOST_API exec_place get_place(pos4 p) const { return get_place(get_dims().get_index(p)); } @@ -799,21 +799,21 @@ public: * @param idx The index of the sub-place to activate (default 0 for scalar places) * @return An exec_place_scope guard that manages the activation lifetime */ - inline exec_place_scope activate(size_t idx = 0) const; + _CCCL_HOST_API inline exec_place_scope activate(size_t idx = 0) const; // ===== Properties ===== - ::std::string to_string() const + _CCCL_HOST_API ::std::string to_string() const { return pimpl->to_string(); } - data_place affine_data_place() const + _CCCL_HOST_API data_place affine_data_place() const { return pimpl->affine_data_place(); } - void set_affine_data_place(data_place place) + _CCCL_HOST_API void set_affine_data_place(data_place place) { pimpl->set_affine_data_place(mv(place)); } @@ -824,58 +824,60 @@ public: * `res`; self-contained places (cuda_stream, green-context) ignore `res` * and return their embedded pool. */ - stream_pool& get_stream_pool(bool for_computation, exec_place_resources& res) const + _CCCL_HOST_API stream_pool& get_stream_pool(bool for_computation, exec_place_resources& res) const { return pimpl->get_stream_pool(for_computation, res, *this); } /// @brief Convenience overload taking an `async_resources_handle`. Defined /// inline in `__stf/internal/async_resources_handle.cuh`. - inline stream_pool& get_stream_pool(bool for_computation, ::cuda::experimental::stf::async_resources_handle& h) const; + _CCCL_HOST_API inline stream_pool& + get_stream_pool(bool for_computation, ::cuda::experimental::stf::async_resources_handle& h) const; - augmented_stream getStream(exec_place_resources& res, bool for_computation = true) const; + _CCCL_HOST_API augmented_stream getStream(exec_place_resources& res, bool for_computation = true) const; /// @brief Convenience overload taking an `async_resources_handle`. Defined /// inline in `__stf/internal/async_resources_handle.cuh`. - inline augmented_stream getStream(::cuda::experimental::stf::async_resources_handle& h, - bool for_computation = true) const; + _CCCL_HOST_API inline augmented_stream + getStream(::cuda::experimental::stf::async_resources_handle& h, bool for_computation = true) const; - cudaStream_t pick_stream(exec_place_resources& res, bool for_computation = true) const + _CCCL_HOST_API cudaStream_t pick_stream(exec_place_resources& res, bool for_computation = true) const { return getStream(res, for_computation).stream; } /// @brief Convenience overload taking an `async_resources_handle`. Defined /// inline in `__stf/internal/async_resources_handle.cuh`. - inline cudaStream_t pick_stream(::cuda::experimental::stf::async_resources_handle& h, - bool for_computation = true) const; + _CCCL_HOST_API inline cudaStream_t + pick_stream(::cuda::experimental::stf::async_resources_handle& h, bool for_computation = true) const; /// @brief Number of streams in this place's pool (slots, not initialized). - inline size_t stream_pool_size(exec_place_resources& res) const; + _CCCL_HOST_API inline size_t stream_pool_size(exec_place_resources& res) const; /// @brief Convenience overload taking an `async_resources_handle`. Defined /// inline in `__stf/internal/async_resources_handle.cuh`. - inline size_t stream_pool_size(::cuda::experimental::stf::async_resources_handle& h) const; + _CCCL_HOST_API inline size_t stream_pool_size(::cuda::experimental::stf::async_resources_handle& h) const; /// @brief Materialize all streams in the pool as a vector. Triggers lazy /// creation of every empty slot. - ::std::vector pick_all_streams(exec_place_resources& res) const; + _CCCL_HOST_API ::std::vector pick_all_streams(exec_place_resources& res) const; /// @brief Convenience overload taking an `async_resources_handle`. Defined /// inline in `__stf/internal/async_resources_handle.cuh`. - ::std::vector pick_all_streams(::cuda::experimental::stf::async_resources_handle& h) const; + _CCCL_HOST_API ::std::vector + pick_all_streams(::cuda::experimental::stf::async_resources_handle& h) const; - const ::std::shared_ptr& get_impl() const + _CCCL_HOST_API const ::std::shared_ptr& get_impl() const { return pimpl; } - bool is_host() const + _CCCL_HOST_API bool is_host() const { return pimpl->is_host(); } - bool is_device() const + _CCCL_HOST_API bool is_device() const { return pimpl->is_device(); } @@ -884,7 +886,7 @@ public: * @brief Get the dimension along a specific axis * @deprecated Use get_dims().get(axis_id) instead */ - size_t grid_dim(int axis_id) const + _CCCL_HOST_API size_t grid_dim(int axis_id) const { return get_dims().get(axis_id); } @@ -893,7 +895,7 @@ public: * @brief Get all dimensions * @deprecated Use get_dims() instead */ - dim4 grid_dims() const + _CCCL_HOST_API dim4 grid_dims() const { return get_dims(); } @@ -902,7 +904,7 @@ public: * @brief Returns *this for compatibility * @deprecated All places are grids now; use exec_place methods directly */ - const exec_place& as_grid() const + _CCCL_HOST_API const exec_place& as_grid() const { EXPECT(size() > 1, "as_grid() called on scalar exec_place"); return *this; @@ -911,10 +913,10 @@ public: /* These helper methods provide convenient way to express execution places, * for example exec_place::host or exec_place::device(4). */ - static exec_place host(); - static exec_place device_auto(); + _CCCL_HOST_API static exec_place host(); + _CCCL_HOST_API static exec_place device_auto(); - static exec_place device(int devid); + _CCCL_HOST_API static exec_place device(int devid); // Green contexts are only supported since CUDA 12.4 #if _CCCL_CTK_AT_LEAST(12, 4) @@ -925,7 +927,7 @@ public: * @param use_green_ctx_data_place If true, use a green context data place as the * affine data place. If false (default), use a regular device data place instead. */ - static exec_place green_ctx(const green_ctx_view& gc_view, bool use_green_ctx_data_place = false); + _CCCL_HOST_API static exec_place green_ctx(const green_ctx_view& gc_view, bool use_green_ctx_data_place = false); #endif // _CCCL_CTK_AT_LEAST(12, 4) /** @@ -959,19 +961,19 @@ public: * @param devid The device ordinal of the context, or -1 to derive it from the context * @param pool_size Number of streams in the place's stream pool */ - static exec_place cuda_context(CUcontext ctx, int devid = -1, size_t pool_size = impl::pool_size); + _CCCL_HOST_API static exec_place cuda_context(CUcontext ctx, int devid = -1, size_t pool_size = impl::pool_size); /** * @brief Returns the currently active device. * * @return exec_place */ - static exec_place current_device() + _CCCL_HOST_API static exec_place current_device() { return exec_place::device(cuda_try()); } - static exec_place all_devices(); + _CCCL_HOST_API static exec_place all_devices(); /** * @brief Returns a grid of execution places over all locality domains of a device @@ -1001,12 +1003,12 @@ public: //! @return exec_place grid with one place per locality domain per device static exec_place all_locality_domains(locality_domain_sm_split split = locality_domain_sm_split::backfill); - static exec_place n_devices(size_t n, dim4 dims); + _CCCL_HOST_API static exec_place n_devices(size_t n, dim4 dims); - static exec_place n_devices(size_t n); + _CCCL_HOST_API static exec_place n_devices(size_t n); // For debug purpose on a machine with a single GPU, for example - static exec_place repeat(const exec_place& e, size_t cnt); + _CCCL_HOST_API static exec_place repeat(const exec_place& e, size_t cnt); template auto partition_by_scope(Args&&... args); @@ -1025,9 +1027,9 @@ public: * */ template - auto operator->*(Fun&& fun) const; + _CCCL_HOST_API auto operator->*(Fun&& fun) const; - exec_place(::std::shared_ptr pimpl) + _CCCL_HOST_API exec_place(::std::shared_ptr pimpl) : pimpl(mv(pimpl)) {} @@ -1073,7 +1075,7 @@ public: /** * @brief Default constructor creates an inactive scope. */ - exec_place_scope() = default; + _CCCL_HIDE_FROM_ABI exec_place_scope() = default; /** * @brief Constructs the guard and activates the sub-place at the given index. @@ -1081,7 +1083,7 @@ public: * @param place The execution place (or grid) containing the sub-place to activate * @param idx The index of the sub-place to activate (default 0 for scalar places) */ - exec_place_scope(exec_place place, size_t idx = 0) + _CCCL_HOST_API exec_place_scope(exec_place place, size_t idx = 0) : place_(mv(place)) , idx_(idx) , current_(place_.get_place(idx_)) @@ -1094,7 +1096,7 @@ public: * Use data_place::affine_exec_place() to get the exec_place first. */ template - exec_place_scope(const data_place&) + _CCCL_HOST_API exec_place_scope(const data_place&) { static_assert(!::cuda::std::is_same_v, "exec_place_scope cannot be constructed from data_place; " @@ -1109,7 +1111,7 @@ public: //! Returning to the device we came from must always succeed; if it does not, continuing would //! run every subsequent launch on the wrong device. deactivate() reaches cuda_try and can also //! allocate, so report and abort rather than propagate out of a destructor. - ~exec_place_scope() + _CCCL_HOST_API ~exec_place_scope() { if (place_.get_impl()) { @@ -1125,7 +1127,7 @@ public: exec_place_scope& operator=(const exec_place_scope&) = delete; // Movable (like unique_lock) - exec_place_scope(exec_place_scope&& other) noexcept + _CCCL_HOST_API exec_place_scope(exec_place_scope&& other) noexcept : place_(mv(other.place_)) , idx_(other.idx_) , current_(mv(other.current_)) @@ -1134,7 +1136,7 @@ public: other.place_ = exec_place(); // Mark other as inactive } - exec_place_scope& operator=(exec_place_scope&& other) noexcept + _CCCL_HOST_API exec_place_scope& operator=(exec_place_scope&& other) noexcept { if (this != &other) { @@ -1158,7 +1160,7 @@ public: /** * @brief Get the currently active sub-place */ - const exec_place& place() const + _CCCL_HOST_API const exec_place& place() const { return current_; } @@ -1166,7 +1168,7 @@ public: /** * @brief Get the index within the grid (0 for scalar places) */ - size_t index() const + _CCCL_HOST_API size_t index() const { return idx_; } @@ -1174,7 +1176,7 @@ public: /** * @brief Check if this scope is active (not moved-from) */ - bool is_active() const + _CCCL_HOST_API bool is_active() const { return place_.get_impl() != nullptr; } @@ -1185,7 +1187,7 @@ public: * After calling reset(), the destructor becomes a no-op. * Calling reset() on an inactive scope is safe (no-op). */ - void reset() + _CCCL_HOST_API void reset() { if (place_.get_impl()) { @@ -1204,19 +1206,19 @@ private: exec_place prev_; // Previous state to restore }; -inline exec_place_scope exec_place::activate(size_t idx) const +_CCCL_HOST_API inline exec_place_scope exec_place::activate(size_t idx) const { return exec_place_scope(*this, idx); } template -auto exec_place::operator->*(Fun&& fun) const +_CCCL_HOST_API auto exec_place::operator->*(Fun&& fun) const { auto active = activate(); return ::cuda::std::forward(fun)(); } -inline augmented_stream stream_pool::next(const exec_place& place) +_CCCL_HOST_API inline augmented_stream stream_pool::next(const exec_place& place) { _CCCL_ASSERT(pimpl, "stream_pool::next called on empty pool"); const ::std::scoped_lock locker(pimpl->mtx); @@ -1262,17 +1264,17 @@ inline augmented_stream stream_pool::next(const exec_place& place) return result; } -inline augmented_stream exec_place::getStream(exec_place_resources& res, bool for_computation) const +_CCCL_HOST_API inline augmented_stream exec_place::getStream(exec_place_resources& res, bool for_computation) const { return get_stream_pool(for_computation, res).next(*this); } -inline size_t exec_place::stream_pool_size(exec_place_resources& res) const +_CCCL_HOST_API inline size_t exec_place::stream_pool_size(exec_place_resources& res) const { return get_stream_pool(true, res).size(); } -inline ::std::vector exec_place::pick_all_streams(exec_place_resources& res) const +_CCCL_HOST_API inline ::std::vector exec_place::pick_all_streams(exec_place_resources& res) const { auto& pool = get_stream_pool(true, res); ::std::vector result; @@ -1292,12 +1294,12 @@ inline ::std::vector exec_place::pick_all_streams(exec_place_resou class exec_place_host_impl : public exec_place::impl { public: - exec_place_host_impl() + _CCCL_HOST_API exec_place_host_impl() : exec_place::impl(data_place::host()) {} // Grid interface - host is a 1-element grid - ::std::shared_ptr get_place(size_t idx) override + _CCCL_HOST_API ::std::shared_ptr get_place(size_t idx) override { _CCCL_ASSERT(idx == 0, "Index out of bounds for host exec_place"); // This singleton is owned by the permanent shared_ptr created once in @@ -1309,29 +1311,30 @@ public: } // Activation - no-op for host - exec_place activate(size_t idx) const override + _CCCL_HOST_API exec_place activate(size_t idx) const override { _CCCL_ASSERT(idx == 0, "Index out of bounds for host exec_place"); return exec_place(); } - void deactivate(const exec_place& prev, size_t idx) const override + _CCCL_HOST_API void deactivate(const exec_place& prev, size_t idx) const override { _CCCL_ASSERT(idx == 0, "Index out of bounds for host exec_place"); _CCCL_ASSERT(!prev.get_impl(), "Host deactivate expects empty prev"); } - bool is_host() const override + _CCCL_HOST_API bool is_host() const override { return true; } - data_place affine_data_place() const override + _CCCL_HOST_API data_place affine_data_place() const override { return data_place::host(); } - stream_pool& get_stream_pool(bool for_computation, exec_place_resources& res, const exec_place&) const override + _CCCL_HOST_API stream_pool& + get_stream_pool(bool for_computation, exec_place_resources& res, const exec_place&) const override { // Forward to the current device place: host work that needs a CUDA stream // borrows the current device's pool entry from the same registry. @@ -1339,13 +1342,13 @@ public: return cur.get_stream_pool(for_computation, res); } - ::std::string to_string() const override + _CCCL_HOST_API ::std::string to_string() const override { return "host"; } }; -inline exec_place exec_place::host() +_CCCL_HOST_API inline exec_place exec_place::host() { return exec_place(make_static_instance()); } @@ -1354,26 +1357,26 @@ inline exec_place exec_place::host() class exec_place_device_auto_impl : public exec_place::impl { public: - exec_place_device_auto_impl() + _CCCL_HOST_API exec_place_device_auto_impl() : exec_place::impl(data_place::device_auto()) {} - exec_place activate(size_t) const override + _CCCL_HOST_API exec_place activate(size_t) const override { _CCCL_THROW(::std::logic_error, "activate() called on device_auto exec_place - should be resolved first"); } - void deactivate(const exec_place&, size_t) const override + _CCCL_HOST_API void deactivate(const exec_place&, size_t) const override { _CCCL_THROW(::std::logic_error, "deactivate() called on device_auto exec_place - should be resolved first"); } - bool is_device() const override + _CCCL_HOST_API bool is_device() const override { return true; } - ::std::shared_ptr get_place(size_t idx) override + _CCCL_HOST_API ::std::shared_ptr get_place(size_t idx) override { _CCCL_ASSERT(idx == 0, "Index out of bounds for device_auto exec_place"); // This singleton is owned by the permanent shared_ptr created once in @@ -1384,13 +1387,13 @@ public: return shared_from_this(); } - ::std::string to_string() const override + _CCCL_HOST_API ::std::string to_string() const override { return "device_auto"; } }; -inline exec_place exec_place::device_auto() +_CCCL_HOST_API inline exec_place exec_place::device_auto() { return make_static_instance(); } @@ -1415,7 +1418,7 @@ public: class impl : public exec_place::impl { public: - explicit impl(int devid) + _CCCL_HOST_API explicit impl(int devid) : exec_place::impl(data_place::device(devid)) , devid_(devid) { @@ -1426,9 +1429,9 @@ public: } // Grid interface - device is a 1-element grid - ::std::shared_ptr get_place(size_t idx) override; + _CCCL_HOST_API ::std::shared_ptr get_place(size_t idx) override; - exec_place activate(size_t idx) const override + _CCCL_HOST_API exec_place activate(size_t idx) const override { _CCCL_ASSERT(idx == 0, "Index out of bounds for device exec_place"); auto old_dev_id = cuda_try(); @@ -1439,7 +1442,7 @@ public: return exec_place::device(old_dev_id); } - void deactivate(const exec_place& prev, size_t idx) const override + _CCCL_HOST_API void deactivate(const exec_place& prev, size_t idx) const override { _CCCL_ASSERT(idx == 0, "Index out of bounds for device exec_place"); auto current_dev_id = cuda_try(); @@ -1450,17 +1453,17 @@ public: } } - bool is_device() const override + _CCCL_HOST_API bool is_device() const override { return true; } - int get_devid() const + _CCCL_HOST_API int get_devid() const { return devid_; } - ::std::string to_string() const override + _CCCL_HOST_API ::std::string to_string() const override { return "device(" + ::std::to_string(devid_) + ")"; } @@ -1470,7 +1473,7 @@ public: }; }; -inline exec_place exec_place::device(int devid) +_CCCL_HOST_API inline exec_place exec_place::device(int devid) { static const int ndevices = cuda_try(); // One process-global ``shared_ptr`` per device, created exactly once in this @@ -1579,7 +1582,7 @@ UNITTEST("exec_place_scope reset") class exec_place_grid_impl : public exec_place::impl { public: - exec_place_grid_impl(::std::vector _places) + _CCCL_HOST_API exec_place_grid_impl(::std::vector _places) : dims_(_places.size(), 1, 1, 1) , places_(mv(_places)) { @@ -1587,7 +1590,7 @@ public: _CCCL_ASSERT(dims_.x > 0, "Grid dimensions must be positive"); } - exec_place_grid_impl(::std::vector _places, const dim4& _dims) + _CCCL_HOST_API exec_place_grid_impl(::std::vector _places, const dim4& _dims) : dims_(_dims) , places_(mv(_places)) { @@ -1607,17 +1610,17 @@ public: // ===== Grid interface ===== - dim4 get_dims() const override + _CCCL_HOST_API dim4 get_dims() const override { return dims_; } - size_t size() const override + _CCCL_HOST_API size_t size() const override { return dims_.size(); } - ::std::shared_ptr get_place(size_t idx) override + _CCCL_HOST_API ::std::shared_ptr get_place(size_t idx) override { EXPECT(idx < places_.size(), "Index out of bounds"); return places_[idx].get_impl(); @@ -1625,13 +1628,13 @@ public: // ===== Activation (delegates to sub-places) ===== - exec_place activate(size_t idx) const override + _CCCL_HOST_API exec_place activate(size_t idx) const override { EXPECT(idx < places_.size(), "Index out of bounds"); return places_[idx].get_impl()->activate(0); } - void deactivate(const exec_place& prev, size_t idx) const override + _CCCL_HOST_API void deactivate(const exec_place& prev, size_t idx) const override { EXPECT(idx < places_.size(), "Index out of bounds"); places_[idx].get_impl()->deactivate(prev, 0); @@ -1639,7 +1642,7 @@ public: // ===== Properties ===== - ::std::string to_string() const override + _CCCL_HOST_API ::std::string to_string() const override { return "grid(" + ::std::to_string(dims_.x) + "x" + ::std::to_string(dims_.y) + "x" + ::std::to_string(dims_.z) + "x" + ::std::to_string(dims_.t) + ")"; @@ -1647,7 +1650,7 @@ public: // ===== Comparison ===== - int cmp(const exec_place::impl& rhs) const override + _CCCL_HOST_API int cmp(const exec_place::impl& rhs) const override { if (typeid(*this) != typeid(rhs)) { @@ -1666,7 +1669,7 @@ public: return (other.places_ < places_) - (places_ < other.places_); } - size_t hash() const override + _CCCL_HOST_API size_t hash() const override { size_t h = ::cuda::experimental::stf::hash{}(dims_); for (const auto& p : places_) @@ -1678,7 +1681,8 @@ public: // ===== Stream management ===== - stream_pool& get_stream_pool(bool for_computation, exec_place_resources& res, const exec_place&) const override + _CCCL_HOST_API stream_pool& + get_stream_pool(bool for_computation, exec_place_resources& res, const exec_place&) const override { _CCCL_ASSERT(!for_computation, "Expected data transfer stream pool"); _CCCL_ASSERT(!places_.empty(), "Grid must have at least one place"); @@ -1695,7 +1699,7 @@ private: //! Creates a grid of execution places with specified dimensions //! Returns the single element if size == 1 (no grid wrapper needed) -inline exec_place make_grid(::std::vector places, const dim4& dims) +_CCCL_HOST_API inline exec_place make_grid(::std::vector places, const dim4& dims) { if (places.empty()) { @@ -1718,7 +1722,7 @@ inline exec_place make_grid(::std::vector places, const dim4& dims) //! Creates a linear grid from a vector of execution places //! Returns the single element if size == 1 (no grid wrapper needed) -inline exec_place make_grid(::std::vector places) +_CCCL_HOST_API inline exec_place make_grid(::std::vector places) { _CCCL_ASSERT(!places.empty(), "invalid places"); const size_t n = places.size(); @@ -1770,7 +1774,7 @@ _CCCL_HOST_API inline exec_place exec_place::collapse_axes(const size_t first_ax // === data_place::affine_exec_place implementation === -inline exec_place data_place::affine_exec_place() const +_CCCL_HOST_API inline exec_place data_place::affine_exec_place() const { if (is_host()) { @@ -1804,13 +1808,13 @@ inline exec_place data_place::affine_exec_place() const // === Deferred implementations for get_place() === -inline ::std::shared_ptr exec_place::impl::get_place(size_t idx) +_CCCL_HOST_API inline ::std::shared_ptr exec_place::impl::get_place(size_t idx) { _CCCL_ASSERT(idx == 0, "Index out of bounds for scalar exec_place"); return shared_from_this(); } -inline ::std::shared_ptr exec_place_device::impl::get_place(size_t idx) +_CCCL_HOST_API inline ::std::shared_ptr exec_place_device::impl::get_place(size_t idx) { _CCCL_ASSERT(idx == 0, "Index out of bounds for device exec_place"); // These device impls are always owned by the per-device ``shared_ptr`` @@ -1823,7 +1827,7 @@ inline ::std::shared_ptr exec_place_device::impl::get_place(si //! Creates a grid by replicating an execution place multiple times //! Returns the original place if cnt == 1 (no grid wrapper needed) -inline exec_place exec_place::repeat(const exec_place& e, size_t cnt) +_CCCL_HOST_API inline exec_place exec_place::repeat(const exec_place& e, size_t cnt) { if (cnt == 1) { @@ -1834,7 +1838,7 @@ inline exec_place exec_place::repeat(const exec_place& e, size_t cnt) /* Get the first N available devices */ //! Returns single device if n == 1 (no grid wrapper needed) -inline exec_place exec_place::n_devices(size_t n, dim4 dims) +_CCCL_HOST_API inline exec_place exec_place::n_devices(size_t n, dim4 dims) { const int ndevs = cuda_try(); @@ -1852,20 +1856,20 @@ inline exec_place exec_place::n_devices(size_t n, dim4 dims) /* Get the first N available devices */ //! Returns single device if n == 1 (no grid wrapper needed) -inline exec_place exec_place::n_devices(size_t n) +_CCCL_HOST_API inline exec_place exec_place::n_devices(size_t n) { return n_devices(n, dim4(n, 1, 1, 1)); } //! Returns all available devices, or single device if only one GPU -inline exec_place exec_place::all_devices() +_CCCL_HOST_API inline exec_place exec_place::all_devices() { return n_devices(cuda_try()); } //! Creates a cyclic partition of an execution place grid with specified strides //! Returns single place if partition contains only one element -inline exec_place partition_cyclic(exec_place e_place, dim4 strides, pos4 tile_id) +_CCCL_HOST_API inline exec_place partition_cyclic(exec_place e_place, dim4 strides, pos4 tile_id) { dim4 g_dims = e_place.get_dims(); @@ -1913,7 +1917,7 @@ inline exec_place partition_cyclic(exec_place e_place, dim4 strides, pos4 tile_i //! //! example : //! auto sub_g = partition_tile(g, dim4(2,2), dim4(0,1)) -inline exec_place partition_tile(exec_place e_place, dim4 tile_sizes, pos4 tile_id) +_CCCL_HOST_API inline exec_place partition_tile(exec_place e_place, dim4 tile_sizes, pos4 tile_id) { const dim4 g_dims = e_place.get_dims(); @@ -1975,38 +1979,38 @@ inline exec_place partition_tile(exec_place e_place, dim4 tile_sizes, pos4 tile_ class data_place_composite final : public data_place_interface { public: - data_place_composite(exec_place grid, partition_mapper partitioner_func) + _CCCL_HOST_API data_place_composite(exec_place grid, partition_mapper partitioner_func) : grid_(mv(grid)) , partitioner_func_(mv(partitioner_func)) {} - bool is_resolved() const override + _CCCL_HOST_API bool is_resolved() const override { return true; } - bool is_composite() const override + _CCCL_HOST_API bool is_composite() const override { return true; } - int get_device_ordinal() const override + _CCCL_HOST_API int get_device_ordinal() const override { return data_place_interface::composite; } - ::std::string to_string() const override + _CCCL_HOST_API ::std::string to_string() const override { return "composite"; } - size_t hash() const override + _CCCL_HOST_API size_t hash() const override { // Composite places don't support hashing _CCCL_THROW(::std::logic_error, "hash() not supported for composite data_place"); } - int cmp(const data_place_interface& other) const override + _CCCL_HOST_API int cmp(const data_place_interface& other) const override { if (typeid(*this) != typeid(other)) { @@ -2026,7 +2030,7 @@ public: return (grid_ < o.grid_) ? -1 : 1; } - void* allocate(::std::ptrdiff_t, cudaStream_t) const override + _CCCL_HOST_API void* allocate(::std::ptrdiff_t, cudaStream_t) const override { // A byte count alone does not carry the tensor geometry the partitioner // needs (it maps element coordinates to places), so there is no meaningful @@ -2036,32 +2040,32 @@ public: "allocate through a logical data"); } - void* allocate_nd(dim4 data_dims, size_t elemsize, cudaStream_t) const override + _CCCL_HOST_API void* allocate_nd(dim4 data_dims, size_t elemsize, cudaStream_t) const override { return allocate_composite_data_place(*this, data_dims, elemsize); } - void deallocate(void* ptr, size_t, cudaStream_t) const override + _CCCL_HOST_API void deallocate(void* ptr, size_t, cudaStream_t) const override { deallocate_composite_data_place(ptr); } - bool allocation_is_stream_ordered() const override + _CCCL_HOST_API bool allocation_is_stream_ordered() const override { return false; } - ::std::shared_ptr get_affine_exec_impl() const override + _CCCL_HOST_API ::std::shared_ptr get_affine_exec_impl() const override { return grid_.get_impl(); } - const partition_mapper& get_partitioner() const override + _CCCL_HOST_API const partition_mapper& get_partitioner() const override { return partitioner_func_; } - const exec_place& get_grid() const + _CCCL_HOST_API const exec_place& get_grid() const { return grid_; } @@ -2087,29 +2091,29 @@ public: //! Bit a of \p axes_mask set = grid axis a is REPLICATED; unset axes are //! SHARED (their fibers use one common instance). The default replicates //! over every axis (one copy per grid member). - explicit data_place_replicated(exec_place grid, unsigned axes_mask = all_axes) + _CCCL_HOST_API explicit data_place_replicated(exec_place grid, unsigned axes_mask = all_axes) : grid_(mv(grid)) , deferred_(false) , axes_mask_(axes_mask) {} //! Deferred form: the grid is bound at task acquisition - data_place_replicated() + _CCCL_HOST_API data_place_replicated() : deferred_(true) , axes_mask_(all_axes) {} - bool is_resolved() const override + _CCCL_HOST_API bool is_resolved() const override { return true; } - bool is_replicated() const noexcept override + _CCCL_HOST_API bool is_replicated() const noexcept override { return true; } - size_t instance_count() const override + _CCCL_HOST_API size_t instance_count() const override { if (deferred_) { @@ -2129,7 +2133,7 @@ public: //! Instance index of a linear grid place: mixed radix over the //! REPLICATED axes (dimension 0 fastest); shared-axis coordinates drop out - size_t instance_of(size_t place_index) const + _CCCL_HOST_API size_t instance_of(size_t place_index) const { if (deferred_) { @@ -2151,7 +2155,7 @@ public: } //! Linear grid place of the instance's representative (shared coords = 0) - size_t representative_place(size_t instance_index) const + _CCCL_HOST_API size_t representative_place(size_t instance_index) const { if (deferred_) { @@ -2172,7 +2176,7 @@ public: //! Every fiber member of every instance must live at the SAME affine data //! place -- "share" only means something where a common memory exists - void validate_colocation() const + _CCCL_HOST_API void validate_colocation() const { const size_t total = grid_.size(); for (size_t p = 0; p < total; p++) @@ -2191,27 +2195,27 @@ public: } } - bool is_deferred() const noexcept + _CCCL_HOST_API bool is_deferred() const noexcept { return deferred_; } - int get_device_ordinal() const override + _CCCL_HOST_API int get_device_ordinal() const override { return data_place_interface::composite; } - ::std::string to_string() const override + _CCCL_HOST_API ::std::string to_string() const override { return deferred_ ? "replicated(deferred)" : "replicated"; } - size_t hash() const override + _CCCL_HOST_API size_t hash() const override { _CCCL_THROW(::std::logic_error, "hash() not supported for replicated data_place"); } - int cmp(const data_place_interface& other) const override + _CCCL_HOST_API int cmp(const data_place_interface& other) const override { if (typeid(*this) != typeid(other)) { @@ -2232,17 +2236,17 @@ public: return (grid_ < o.grid_) ? -1 : (grid_ != o.grid_); } - void* allocate(::std::ptrdiff_t, cudaStream_t) const override + _CCCL_HOST_API void* allocate(::std::ptrdiff_t, cudaStream_t) const override { _CCCL_THROW(::std::runtime_error, "replicated data_place: allocate through a logical data"); } - void deallocate(void*, size_t, cudaStream_t) const override + _CCCL_HOST_API void deallocate(void*, size_t, cudaStream_t) const override { _CCCL_THROW(::std::runtime_error, "replicated data_place: instances deallocate through their member places"); } - bool allocation_is_stream_ordered() const override + _CCCL_HOST_API bool allocation_is_stream_ordered() const override { // Instances at this place are one ordinary allocation per member, so // teardown is stream-ordered exactly when every member's is: report @@ -2257,12 +2261,12 @@ public: return true; } - ::std::shared_ptr get_affine_exec_impl() const override + _CCCL_HOST_API ::std::shared_ptr get_affine_exec_impl() const override { return grid_.get_impl(); } - const exec_place& get_grid() const noexcept + _CCCL_HOST_API const exec_place& get_grid() const noexcept { return grid_; } @@ -2276,7 +2280,7 @@ private: }; //! Whether this replicated data place still needs its grid bound -inline bool replicated_is_deferred(const data_place& dp) +_CCCL_HOST_API inline bool replicated_is_deferred(const data_place& dp) { // NOLINTNEXTLINE(cppcoreguidelines-pro-type-static-cast-downcast) -- the caller checked the kind const auto* rep = static_cast(dp.get_impl().get()); @@ -2284,29 +2288,29 @@ inline bool replicated_is_deferred(const data_place& dp) } //! Grid of a replicated data place -inline const exec_place& replicated_grid(const data_place& dp) +_CCCL_HOST_API inline const exec_place& replicated_grid(const data_place& dp) { // NOLINTNEXTLINE(cppcoreguidelines-pro-type-static-cast-downcast) -- the caller checked the kind const auto* rep = static_cast(dp.get_impl().get()); return rep->get_grid(); } -inline bool data_place::is_composite() const +_CCCL_HOST_API inline bool data_place::is_composite() const { return pimpl_->is_composite(); } -inline bool data_place::is_replicated() const noexcept +_CCCL_HOST_API inline bool data_place::is_replicated() const noexcept { return pimpl_->is_replicated(); } -inline size_t data_place::instance_count() const +_CCCL_HOST_API inline size_t data_place::instance_count() const { return pimpl_->instance_count(); } -inline data_place data_place::member(size_t r) const +_CCCL_HOST_API inline data_place data_place::member(size_t r) const { _CCCL_ASSERT(r < instance_count(), "member index out of range"); if (instance_count() == 1) @@ -2318,7 +2322,7 @@ inline data_place data_place::member(size_t r) const return rep->get_grid().get_place(rep->representative_place(r)).affine_data_place(); } -inline size_t data_place::instance_of(size_t place_index) const +_CCCL_HOST_API inline size_t data_place::instance_of(size_t place_index) const { if (!is_replicated()) { @@ -2329,12 +2333,12 @@ inline size_t data_place::instance_of(size_t place_index) const return rep->instance_of(place_index); } -inline data_place data_place::composite(partition_mapper f, const exec_place& grid) +_CCCL_HOST_API inline data_place data_place::composite(partition_mapper f, const exec_place& grid) { return data_place(::std::make_shared(grid, mv(f))); } -inline data_place data_place::replicated(const exec_place& grid) +_CCCL_HOST_API inline data_place data_place::replicated(const exec_place& grid) { if (!grid.get_impl()) { @@ -2351,13 +2355,13 @@ inline data_place data_place::replicated(const exec_place& grid) return data_place(::std::make_shared(grid)); } -inline data_place data_place::replicated() +_CCCL_HOST_API inline data_place data_place::replicated() { return data_place(::std::make_shared()); } template -data_place data_place::replicated(const exec_place& grid, replicate_over_t) +_CCCL_HOST_API data_place data_place::replicated(const exec_place& grid, replicate_over_t) { static_assert(sizeof...(axes) >= 1, "replicate_over needs at least one axis"); static_assert(((axes < 4) && ...), "grid axes are 0..3"); @@ -2380,12 +2384,12 @@ data_place data_place::replicated(const exec_place& grid, replicate_over_t -data_place data_place::composite(partitioner_t, const exec_place& g) +_CCCL_HOST_API data_place data_place::composite(partitioner_t, const exec_place& g) { return data_place::composite(partition_mapper(&partitioner_t::get_executor), g); } -inline augmented_stream data_place::getDataStream(exec_place_resources& res) const +_CCCL_HOST_API inline augmented_stream data_place::getDataStream(exec_place_resources& res) const { return affine_exec_place().getStream(res, false); } @@ -2598,7 +2602,7 @@ UNITTEST("dim4 very large total size calculation") template <> struct hash { - ::std::size_t operator()(const data_place& k) const + _CCCL_HOST_API ::std::size_t operator()(const data_place& k) const { return k.hash(); } @@ -2610,7 +2614,7 @@ struct hash template <> struct hash { - ::std::size_t operator()(const exec_place& k) const + _CCCL_HOST_API ::std::size_t operator()(const exec_place& k) const { return k.hash(); } diff --git a/cudax/include/cuda/experimental/__stf/allocators/buddy_allocator.cuh b/cudax/include/cuda/experimental/__stf/allocators/buddy_allocator.cuh index 3c04fb4755a9..471dafcf2f77 100644 --- a/cudax/include/cuda/experimental/__stf/allocators/buddy_allocator.cuh +++ b/cudax/include/cuda/experimental/__stf/allocators/buddy_allocator.cuh @@ -51,7 +51,7 @@ class buddy_allocator_metadata struct avail_block { // Copy prereqs - avail_block(size_t index_, event_list prereqs_) + _CCCL_HOST_API avail_block(size_t index_, event_list prereqs_) : index(index_) , prereqs(mv(prereqs_)) {} @@ -61,7 +61,7 @@ class buddy_allocator_metadata }; public: - buddy_allocator_metadata(size_t size, event_list init_prereqs) + _CCCL_HOST_API buddy_allocator_metadata(size_t size, event_list init_prereqs) : free_lists_(int_log2(next_power_of_two(size)) + 1) { _CCCL_ASSERT(size && (size & (size - 1)) == 0, @@ -70,7 +70,7 @@ public: free_lists_.back().emplace_back(0, mv(init_prereqs)); } - ::std::ptrdiff_t allocate(size_t size, event_list& prereqs) + _CCCL_HOST_API ::std::ptrdiff_t allocate(size_t size, event_list& prereqs) { size = next_power_of_two(size); const size_t level = int_log2(size); @@ -90,7 +90,7 @@ public: return alloc_index; } - void deallocate(::std::ptrdiff_t index, size_t size, event_list& prereqs) + _CCCL_HOST_API void deallocate(::std::ptrdiff_t index, size_t size, event_list& prereqs) { size = next_power_of_two(size); size_t level = int_log2(size); @@ -123,7 +123,7 @@ public: free_lists_[level].emplace_back(index, prereqs); } - void deinit(event_list& prereqs) + _CCCL_HOST_API void deinit(event_list& prereqs) { for (auto& level : free_lists_) { @@ -135,7 +135,7 @@ public: } } - void debug_print() const + _CCCL_HOST_API void debug_print() const { size_t power = 1; for (size_t i = 0; i < free_lists_.size(); ++i, power *= 2) @@ -153,7 +153,7 @@ public: } private: - static size_t next_power_of_two(size_t size) + _CCCL_HOST_API static size_t next_power_of_two(size_t size) { static_assert(sizeof(size_t) <= 8, "You must be from the future. Review and adjust this code."); if (size == 0) @@ -173,7 +173,7 @@ private: return size + 1; } - static size_t int_log2(size_t n) + _CCCL_HOST_API static size_t int_log2(size_t n) { assert(n > 0); @@ -185,7 +185,7 @@ private: return log; } - ::std::ptrdiff_t find_free_block(size_t level, event_list& prereqs) + _CCCL_HOST_API ::std::ptrdiff_t find_free_block(size_t level, event_list& prereqs) { for (size_t current_level : each(level, free_lists_.size())) { @@ -215,7 +215,7 @@ private: return -1; // No block available } - size_t get_buddy_index(size_t index, size_t level) + _CCCL_HOST_API size_t get_buddy_index(size_t index, size_t level) { assert(level <= 63); return index ^ (1ull << level); // XOR to find the buddy block @@ -232,8 +232,8 @@ private: class buddy_allocator : public block_allocator_interface { public: - buddy_allocator() = default; - buddy_allocator(block_allocator_untyped root_allocator_) + _CCCL_HIDE_FROM_ABI buddy_allocator() = default; + _CCCL_HOST_API buddy_allocator(block_allocator_untyped root_allocator_) : root_allocator(mv(root_allocator_)) {} @@ -241,16 +241,16 @@ private: // Per data place buffer and its corresponding metadata struct per_place { - per_place(void* base_, size_t size, event_list prereqs) + _CCCL_HOST_API per_place(void* base_, size_t size, event_list prereqs) : base(base_) , buffer_size(size) , metadata(buffer_size, mv(prereqs)) {} - per_place& operator=(const per_place&) = delete; - per_place& operator=(per_place&&) = default; - per_place(const per_place&) = delete; - per_place(per_place&&) = default; + per_place& operator=(const per_place&) = delete; + _CCCL_HIDE_FROM_ABI per_place& operator=(per_place&&) = default; + per_place(const per_place&) = delete; + _CCCL_HIDE_FROM_ABI per_place(per_place&&) = default; void* base = nullptr; size_t buffer_size = 0; @@ -258,7 +258,7 @@ private: }; public: - void* + _CCCL_HOST_API void* allocate(backend_ctx_untyped& ctx, const data_place& memory_node, ::std::ptrdiff_t& s, event_list& prereqs) override { auto it = map.find(memory_node); @@ -283,7 +283,7 @@ public: return static_cast(m.base) + offset; } - void deallocate( + _CCCL_HOST_API void deallocate( backend_ctx_untyped& ctx, const data_place& memory_node, event_list& prereqs, void* ptr, size_t sz) override { (void) ctx; @@ -296,7 +296,7 @@ public: m.metadata.deallocate(static_cast<::std::ptrdiff_t>(offset), sz, prereqs); } - event_list deinit(backend_ctx_untyped& ctx) override + _CCCL_HOST_API event_list deinit(backend_ctx_untyped& ctx) override { event_list result; // For every place in the map @@ -316,7 +316,7 @@ public: return result; } - ::std::string to_string() const override + _CCCL_HOST_API ::std::string to_string() const override { return "buddy allocator"; } diff --git a/cudax/include/cuda/experimental/__stf/graph/graph_ctx.cuh b/cudax/include/cuda/experimental/__stf/graph/graph_ctx.cuh index 7a8d570b8799..0daa5e41c307 100644 --- a/cudax/include/cuda/experimental/__stf/graph/graph_ctx.cuh +++ b/cudax/include/cuda/experimental/__stf/graph/graph_ctx.cuh @@ -54,9 +54,9 @@ namespace cuda::experimental::stf class uncached_graph_allocator : public block_allocator_interface { public: - uncached_graph_allocator() = default; + _CCCL_HIDE_FROM_ABI uncached_graph_allocator() = default; - void* + _CCCL_HOST_API void* allocate(backend_ctx_untyped& ctx, const data_place& memory_node, ::std::ptrdiff_t& s, event_list& prereqs) override { // This is not implemented yet @@ -110,7 +110,7 @@ public: return result; } - void deallocate( + _CCCL_HOST_API void deallocate( backend_ctx_untyped& ctx, const data_place& memory_node, event_list& prereqs, void* ptr, size_t /*sz*/) override { const cudaGraph_t graph = ctx.graph(); @@ -129,19 +129,19 @@ public: reserved::fork_from_graph_node(ctx, out, graph, graph_stage, prereqs, "dealloc"); } - ::std::string to_string() const override + _CCCL_HOST_API ::std::string to_string() const override { return "uncached graph allocator"; } // Nothing is done as all deallocation are done immediately - event_list deinit(backend_ctx_untyped& /* ctx */) override + _CCCL_HOST_API event_list deinit(backend_ctx_untyped& /* ctx */) override { return event_list(); } private: - void* add_mem_alloc_node( + _CCCL_HOST_API void* add_mem_alloc_node( const data_place& memory_node, cudaGraphNode_t& out, cudaGraph_t graph, @@ -213,7 +213,7 @@ class graph_ctx : public backend_ctx class impl : public backend_impl { public: - impl(async_resources_handle _async_resources = async_resources_handle(nullptr)) + _CCCL_HOST_API impl(async_resources_handle _async_resources = async_resources_handle(nullptr)) : backend_impl(mv(_async_resources)) , _graph(shared_cuda_graph()) { @@ -221,7 +221,7 @@ class graph_ctx : public backend_ctx } // Note that graph contexts with an explicit graph passed by the user cannot use stages - impl(cudaGraph_t g, async_resources_handle _async_resources = async_resources_handle(nullptr)) + _CCCL_HOST_API impl(cudaGraph_t g, async_resources_handle _async_resources = async_resources_handle(nullptr)) : backend_impl(mv(_async_resources)) , _graph(wrap_cuda_graph(g)) , explicit_graph(true) @@ -230,9 +230,9 @@ class graph_ctx : public backend_ctx } // Constructor with explicit graph, user stream, and async resources - impl(cudaGraph_t g, - cudaStream_t user_stream, - async_resources_handle _async_resources = async_resources_handle(nullptr)) + _CCCL_HOST_API impl(cudaGraph_t g, + cudaStream_t user_stream, + async_resources_handle _async_resources = async_resources_handle(nullptr)) : backend_impl(mv(_async_resources)) , submitted_stream(user_stream) , _graph(wrap_cuda_graph(g)) @@ -242,43 +242,43 @@ class graph_ctx : public backend_ctx reserved::backend_ctx_setup_allocators(*this); } - ~impl() override = default; + _CCCL_HIDE_FROM_ABI ~impl() override = default; - ::std::string to_string() const override + _CCCL_HOST_API ::std::string to_string() const override { return "graph backend context"; } // Due to circular dependencies, we need to define it here, and not in backend_ctx_untyped - void update_uncached_allocator(block_allocator_untyped custom) override + _CCCL_HOST_API void update_uncached_allocator(block_allocator_untyped custom) override { reserved::backend_ctx_update_uncached_allocator(*this, mv(custom)); } - cudaGraph_t graph() const override + _CCCL_HOST_API cudaGraph_t graph() const override { assert(_graph.get()); return *_graph; } - bool is_graph_ctx() const override + _CCCL_HOST_API bool is_graph_ctx() const override { return true; } - executable_graph_cache_stat* graph_get_cache_stat() override + _CCCL_HOST_API executable_graph_cache_stat* graph_get_cache_stat() override { return &cache_stats; } - size_t stage() const override + _CCCL_HOST_API size_t stage() const override { return graph_stage; } // The completion of the CUDA graph implies the completion of all nodes in // the graph - bool track_dangling_events() const override + _CCCL_HOST_API bool track_dangling_events() const override { return false; } @@ -317,11 +317,11 @@ public: /// @brief This type is copyable, assignable, and movable. However, copies have reference semantics. ///@{ - graph_ctx(async_resources_handle handle = async_resources_handle(nullptr)) + _CCCL_HOST_API graph_ctx(async_resources_handle handle = async_resources_handle(nullptr)) : backend_ctx(::std::make_shared(mv(handle))) {} - graph_ctx(cudaStream_t user_stream, async_resources_handle handle = async_resources_handle(nullptr)) + _CCCL_HOST_API graph_ctx(cudaStream_t user_stream, async_resources_handle handle = async_resources_handle(nullptr)) : backend_ctx(::std::make_shared(mv(handle))) { auto& state = this->state(); @@ -333,11 +333,12 @@ public: } /// @brief Constructor taking a user-provided graph. User code is not supposed to destroy the graph later. - graph_ctx(cudaGraph_t g, async_resources_handle handle = async_resources_handle(nullptr)) + _CCCL_HOST_API graph_ctx(cudaGraph_t g, async_resources_handle handle = async_resources_handle(nullptr)) : backend_ctx(::std::make_shared(g, mv(handle))) {} /// @brief Constructor with explicit graph, support stream, and async resources + _CCCL_HOST_API graph_ctx(cudaGraph_t g, cudaStream_t user_stream, async_resources_handle handle = async_resources_handle(nullptr)) : backend_ctx(::std::make_shared(g, user_stream, mv(handle))) {} @@ -349,14 +350,14 @@ public: * @brief Creates a typed task on the specified execution place */ template - auto task(exec_place e_place, task_dep... deps) + _CCCL_HOST_API auto task(exec_place e_place, task_dep... deps) { return graph_task( *this, get_graph(), this->state().graph_mutex, get_graph_stage(), mv(e_place), mv(deps)...); } // submit a new stage : this will submit a graph in a stream that we return - cudaStream_t fence() + _CCCL_HOST_API cudaStream_t fence() { change_stage(); // Get the stream used to submit the graph of the stage we have @@ -364,7 +365,7 @@ public: return this->state().submitted_stream; } - void finalize() + _CCCL_HOST_API void finalize() { _CCCL_ASSERT(get_phase() < backend_ctx_untyped::phase::finalized, ""); auto& state = this->state(); @@ -400,7 +401,7 @@ public: } } - void submit(cudaStream_t stream = nullptr) + _CCCL_HOST_API void submit(cudaStream_t stream = nullptr) { _CCCL_ASSERT(get_phase() < backend_ctx_untyped::phase::submitted, ""); auto& state = this->state(); @@ -430,7 +431,7 @@ public: } // Start a new stage ! - void change_stage() + _CCCL_HOST_API void change_stage() { auto& state = this->state(); @@ -449,29 +450,29 @@ public: } /// @brief Get the "support" graph associated with the context - ::std::shared_ptr get_shared_graph() const + _CCCL_HOST_API ::std::shared_ptr get_shared_graph() const { return state()._graph; } - cudaGraph_t& get_graph() const + _CCCL_HOST_API cudaGraph_t& get_graph() const { assert(state()._graph); return *(state()._graph); } - size_t get_graph_stage() const + _CCCL_HOST_API size_t get_graph_stage() const { return state().graph_stage; } - auto get_exec_graph() const + _CCCL_HOST_API auto get_exec_graph() const { return state().exec_graph; } /* Make sure all pending operations are finished, and release resources */ - ::std::shared_ptr finalize_as_graph() + _CCCL_HOST_API ::std::shared_ptr finalize_as_graph() { // Write-back data and erase automatically created data instances get_state().erase_all_logical_data(); @@ -491,7 +492,7 @@ public: } // Execute the CUDA graph in the provided stream. - ::std::shared_ptr instantiate() + _CCCL_HOST_API ::std::shared_ptr instantiate() { const ::std::shared_ptr g = finalize_as_graph(); @@ -555,7 +556,7 @@ public: return state.exec_graph; } - void display_graph_info(cudaGraph_t g) + _CCCL_HOST_API void display_graph_info(cudaGraph_t g) { size_t numNodes; cuda_try(cudaGraphGetNodes(g, nullptr, &numNodes)); @@ -575,13 +576,14 @@ public: } // flags = cudaGraphDebugDotFlagsVerbose to display all available debug information - void print_to_dot(const ::std::string& filename, enum cudaGraphDebugDotFlags flags = cudaGraphDebugDotFlags(0)) const + _CCCL_HOST_API void print_to_dot(const ::std::string& filename, + enum cudaGraphDebugDotFlags flags = cudaGraphDebugDotFlags(0)) const { cuda_try(cudaGraphDebugDotPrint(get_graph(), filename.c_str(), flags)); } template - auto wait(cuda::experimental::stf::logical_data& ldata) + _CCCL_HOST_API auto wait(cuda::experimental::stf::logical_data& ldata) { if constexpr (::cuda::std::is_same_v) { @@ -610,17 +612,17 @@ public: } private: - impl& state() + _CCCL_HOST_API impl& state() { return dynamic_cast(get_state()); } - const impl& state() const + _CCCL_HOST_API const impl& state() const { return dynamic_cast(get_state()); } /// @brief Inserts an empty node that depends on all pending operations - cudaStream_t fence_impl(backend_ctx_untyped& bctx) + _CCCL_HOST_API cudaStream_t fence_impl(backend_ctx_untyped& bctx) { auto& state = this->state(); @@ -638,7 +640,7 @@ private: } // Instantiate a CUDA graph - static ::std::shared_ptr graph_instantiate(cudaGraph_t g) + _CCCL_HOST_API static ::std::shared_ptr graph_instantiate(cudaGraph_t g) { // Custom deleter specifically for cudaGraphExec_t. The handle is // value-initialized and stays null if instantiation throws: do not @@ -664,7 +666,7 @@ private: } // Creates a new CUDA graph and wrap it into a shared_ptr - static ::std::shared_ptr shared_cuda_graph() + _CCCL_HOST_API static ::std::shared_ptr shared_cuda_graph() { // Same precautions as cudaGraphExecDeleter above, for the same reasons. A custom deleter // replaces the default `delete`, so it has to free the cell itself. And the handle is @@ -692,7 +694,7 @@ private: } // Wrap an existing CUDA graph into a shared_ptr, the destruction of the graph is let to the application - static ::std::shared_ptr wrap_cuda_graph(cudaGraph_t g) + _CCCL_HOST_API static ::std::shared_ptr wrap_cuda_graph(cudaGraph_t g) { // No custom deleter: the graph's lifetime belongs to the application, so only the cell // holding the handle is freed. make_shared keeps that allocation exception-safe -- a raw @@ -701,7 +703,7 @@ private: return ::std::make_shared(g); } - cudaStream_t submit_one_stage(cudaGraph_t g, size_t stage) + _CCCL_HOST_API cudaStream_t submit_one_stage(cudaGraph_t g, size_t stage) { auto& state = this->state(); // TODO rework to make sure we don't push a specific stream later ? @@ -772,12 +774,12 @@ private: } public: - friend cudaGraph_t ctx_to_graph(backend_ctx_untyped& ctx) + _CCCL_HOST_API friend cudaGraph_t ctx_to_graph(backend_ctx_untyped& ctx) { return ctx.graph(); } - friend size_t ctx_to_graph_stage(backend_ctx_untyped& ctx) + _CCCL_HOST_API friend size_t ctx_to_graph_stage(backend_ctx_untyped& ctx) { return ctx.stage(); } @@ -860,7 +862,7 @@ UNITTEST("set_symbol on graph_task and graph_task<>") # if !defined(CUDASTF_DISABLE_CODE_GENERATION) && _CCCL_CUDA_COMPILATION() namespace reserved { -inline void unit_test_graph_stage() +_CCCL_HOST_API inline void unit_test_graph_stage() { graph_ctx ctx; @@ -908,7 +910,7 @@ UNITTEST("graph with stage") unit_test_graph_stage(); }; -inline void unit_test_graph_empty_stage() +_CCCL_HOST_API inline void unit_test_graph_empty_stage() { graph_ctx ctx; @@ -958,7 +960,7 @@ UNITTEST("graph with empty stage") unit_test_graph_empty_stage(); }; -inline void unit_test_graph_stage_2() +_CCCL_HOST_API inline void unit_test_graph_stage_2() { graph_ctx ctx; @@ -1016,7 +1018,7 @@ UNITTEST("graph with stage 2") unit_test_graph_stage_2(); }; -inline void unit_test_graph_stage_3() +_CCCL_HOST_API inline void unit_test_graph_stage_3() { graph_ctx ctx; @@ -1088,7 +1090,7 @@ UNITTEST("graph with stage 3") unit_test_graph_stage_3(); }; -inline void unit_test_launch_graph() +_CCCL_HOST_API inline void unit_test_launch_graph() { graph_ctx ctx; // finalize() submits pending work and synchronizes, so it belongs on the normal path only: @@ -1118,7 +1120,7 @@ UNITTEST("basic launch test (graph_ctx)") unit_test_launch_graph(); }; -inline void unit_test_launch_many_graph_ctx() +_CCCL_HOST_API inline void unit_test_launch_many_graph_ctx() { // Stress the allocators and all resources ! for (size_t i = 0; i < 256; i++) diff --git a/cudax/include/cuda/experimental/__stf/internal/async_resources_handle.cuh b/cudax/include/cuda/experimental/__stf/internal/async_resources_handle.cuh index 47baf4739c88..bf418e0f19dc 100644 --- a/cudax/include/cuda/experimental/__stf/internal/async_resources_handle.cuh +++ b/cudax/include/cuda/experimental/__stf/internal/async_resources_handle.cuh @@ -93,7 +93,7 @@ class async_resources_handle // located on stream "from" to stream "dst" (stream dst waits for the // event) // Returned value : boolean indicating if we can skip the synchronization - bool validate_sync_and_update(unsigned long long dst, unsigned long long src, int event_id) + _CCCL_HOST_API bool validate_sync_and_update(unsigned long long dst, unsigned long long src, int event_id) { // If either of the streams has no valid id, do not skip if (dst == k_no_stream_id || src == k_no_stream_id) @@ -141,7 +141,7 @@ class async_resources_handle class impl { public: - impl() + _CCCL_HOST_API impl() { #if _CCCL_CTK_AT_LEAST(12, 4) const int ndevices = cuda_try(); @@ -183,13 +183,13 @@ class async_resources_handle mutable ::std::shared_ptr pimpl; public: - async_resources_handle() + _CCCL_HOST_API async_resources_handle() : pimpl(::std::make_shared()) {} - explicit async_resources_handle(::std::nullptr_t) {} + _CCCL_HOST_API explicit async_resources_handle(::std::nullptr_t) {} - explicit operator bool() const + _CCCL_HOST_API explicit operator bool() const { return pimpl != nullptr; } @@ -209,13 +209,13 @@ public: * the same handle share one registry. The registry itself is internally * mutex-guarded for concurrent lookups. */ - [[nodiscard]] ::cuda::experimental::places::exec_place_resources& get_place_resources() const + [[nodiscard]] _CCCL_HOST_API ::cuda::experimental::places::exec_place_resources& get_place_resources() const { assert(pimpl); return pimpl->place_resources; } - bool validate_sync_and_update(unsigned long long dst, unsigned long long src, int event_id) + _CCCL_HOST_API bool validate_sync_and_update(unsigned long long dst, unsigned long long src, int event_id) { assert(pimpl); return pimpl->cached_syncs.validate_sync_and_update(dst, src, event_id); @@ -223,14 +223,14 @@ public: // The graph is only used during the call (to update or instantiate); it is never stored, so the // caller only needs to keep it valid for the duration of the call. - ::cuda::std::pair<::std::shared_ptr, bool> + _CCCL_HOST_API ::cuda::std::pair<::std::shared_ptr, bool> cached_graphs_query(size_t nnodes, size_t nedges, cudaGraph_t g) { _CCCL_ASSERT(pimpl, "async_resources_handle is not initialized"); return pimpl->cached_graphs.query(nnodes, nedges, g); } - ::cuda::std::pair<::std::shared_ptr, bool> cached_graphs_query(cudaGraph_t g) + _CCCL_HOST_API ::cuda::std::pair<::std::shared_ptr, bool> cached_graphs_query(cudaGraph_t g) { const size_t nnodes = cuda_try(g, nullptr); #if _CCCL_CTK_AT_LEAST(13, 0) @@ -245,7 +245,7 @@ public: #if _CCCL_CTK_AT_LEAST(12, 4) // Get the green context helper cached for this device (or let the user initialize it) - auto& gc_helper(int dev_id) + _CCCL_HOST_API auto& gc_helper(int dev_id) { assert(pimpl); assert(dev_id < int(pimpl->per_device_gc_helper.size())); @@ -253,7 +253,7 @@ public: } // Get green context helper with lazy initialization - ::std::shared_ptr get_gc_helper(int dev_id, int sm_count) + _CCCL_HOST_API ::std::shared_ptr get_gc_helper(int dev_id, int sm_count) { assert(pimpl); assert(dev_id < int(pimpl->per_device_gc_helper.size())); @@ -266,7 +266,7 @@ public: } // Register an external green context helper - void register_gc_helper(int dev_id, ::std::shared_ptr helper) + _CCCL_HOST_API void register_gc_helper(int dev_id, ::std::shared_ptr helper) { assert(pimpl); assert(dev_id < int(pimpl->per_device_gc_helper.size())); @@ -274,43 +274,43 @@ public: } #endif // _CCCL_CTK_AT_LEAST(12, 4) - exec_affinity& get_affinity() + _CCCL_HOST_API exec_affinity& get_affinity() { assert(pimpl); return pimpl->affinity; } - const exec_affinity& get_affinity() const + _CCCL_HOST_API const exec_affinity& get_affinity() const { assert(pimpl); return pimpl->affinity; } - bool has_affinity() const + _CCCL_HOST_API bool has_affinity() const { assert(pimpl); return pimpl->affinity.has_affinity(); } - void push_affinity(::std::vector<::std::shared_ptr> p) const + _CCCL_HOST_API void push_affinity(::std::vector<::std::shared_ptr> p) const { assert(pimpl); pimpl->affinity.push(mv(p)); } - void push_affinity(::std::shared_ptr p) const + _CCCL_HOST_API void push_affinity(::std::shared_ptr p) const { assert(pimpl); pimpl->affinity.push(mv(p)); } - void pop_affinity() const + _CCCL_HOST_API void pop_affinity() const { assert(pimpl); pimpl->affinity.pop(); } - const ::std::vector<::std::shared_ptr>& current_affinity() const + _CCCL_HOST_API const ::std::vector<::std::shared_ptr>& current_affinity() const { assert(pimpl); return pimpl->affinity.top(); @@ -339,30 +339,31 @@ namespace cuda::experimental::places // layer; they are only available to code that already includes // async_resources_handle.cuh. -[[nodiscard]] inline stream_pool& +[[nodiscard]] _CCCL_HOST_API inline stream_pool& exec_place::get_stream_pool(bool for_computation, ::cuda::experimental::stf::async_resources_handle& h) const { return get_stream_pool(for_computation, h.get_place_resources()); } -[[nodiscard]] inline augmented_stream +[[nodiscard]] _CCCL_HOST_API inline augmented_stream exec_place::getStream(::cuda::experimental::stf::async_resources_handle& h, bool for_computation) const { return getStream(h.get_place_resources(), for_computation); } -[[nodiscard]] inline cudaStream_t +[[nodiscard]] _CCCL_HOST_API inline cudaStream_t exec_place::pick_stream(::cuda::experimental::stf::async_resources_handle& h, bool for_computation) const { return pick_stream(h.get_place_resources(), for_computation); } -[[nodiscard]] inline size_t exec_place::stream_pool_size(::cuda::experimental::stf::async_resources_handle& h) const +[[nodiscard]] _CCCL_HOST_API inline size_t +exec_place::stream_pool_size(::cuda::experimental::stf::async_resources_handle& h) const { return stream_pool_size(h.get_place_resources()); } -[[nodiscard]] inline ::std::vector +[[nodiscard]] _CCCL_HOST_API inline ::std::vector exec_place::pick_all_streams(::cuda::experimental::stf::async_resources_handle& h) const { return pick_all_streams(h.get_place_resources()); diff --git a/cudax/include/cuda/experimental/__stf/internal/context.cuh b/cudax/include/cuda/experimental/__stf/internal/context.cuh index d587a0cbc8d7..54da9508bd03 100644 --- a/cudax/include/cuda/experimental/__stf/internal/context.cuh +++ b/cudax/include/cuda/experimental/__stf/internal/context.cuh @@ -59,7 +59,7 @@ namespace cuda::experimental::stf * @overload */ template -decltype(auto) operator->*(::cuda::std::variant& v, F&& f) +_CCCL_HOST_API decltype(auto) operator->*(::cuda::std::variant& v, F&& f) { // Hand-rolled dispatch rather than cuda::std::visit: visit is host-device, and returning the // callable's result through its frames drags implicitly-generated move constructors into @@ -77,7 +77,7 @@ decltype(auto) operator->*(::cuda::std::variant& v, F&& f) * @overload */ template -decltype(auto) operator->*(const ::cuda::std::variant& v, F&& f) +_CCCL_HOST_API decltype(auto) operator->*(const ::cuda::std::variant& v, F&& f) { static_assert(sizeof...(Ts) == 2, "this dispatch is written for the binary payload variants"); if (v.index() == 0) @@ -103,22 +103,22 @@ private: class unified_scope { public: - unified_scope(T1 arg) + _CCCL_HOST_API unified_scope(T1 arg) : payload(mv(arg)) {} - unified_scope(T2 arg) + _CCCL_HOST_API unified_scope(T2 arg) : payload(mv(arg)) {} /// Get the string attached to the task for debugging purposes - const ::std::string& get_symbol() const + _CCCL_HOST_API const ::std::string& get_symbol() const { return payload->*[&](auto& self) { return self.get_symbol(); }; } - auto&& set_exec_place(exec_place e_place) & + _CCCL_HOST_API auto&& set_exec_place(exec_place e_place) & { payload->*[&](auto& self) { self.set_exec_place(mv(e_place)); @@ -126,7 +126,7 @@ private: return *this; } - auto&& set_exec_place(exec_place e_place) && + _CCCL_HOST_API auto&& set_exec_place(exec_place e_place) && { payload->*[&](auto& self) { self.set_exec_place(mv(e_place)); @@ -134,7 +134,7 @@ private: return mv(*this); } - auto& set_symbol(::std::string s) & + _CCCL_HOST_API auto& set_symbol(::std::string s) & { payload->*[&](auto& self) { self.set_symbol(mv(s)); @@ -142,7 +142,7 @@ private: return *this; } - auto&& set_symbol(::std::string s) && + _CCCL_HOST_API auto&& set_symbol(::std::string s) && { payload->*[&](auto& self) { self.set_symbol(mv(s)); @@ -151,7 +151,7 @@ private: } template - void operator->*(Fun&& f) + _CCCL_HOST_API void operator->*(Fun&& f) { payload->*[&](auto& self) { self->*::cuda::std::forward(f); @@ -159,7 +159,7 @@ private: } // Attach opaque user data to the scope; an optional destructor is called when the scope is destroyed - auto& set_user_data(const void* data, size_t sz, void (*dtor)(void*) = nullptr) + _CCCL_HOST_API auto& set_user_data(const void* data, size_t sz, void (*dtor)(void*) = nullptr) { payload->*[&](auto& self) { self.set_user_data(data, sz, dtor); @@ -168,7 +168,7 @@ private: } template - auto& add_deps(Args&&... args) + _CCCL_HOST_API auto& add_deps(Args&&... args) { payload->*[&](auto& self) { self.add_deps(::cuda::std::forward(args)...); @@ -177,7 +177,7 @@ private: } template - auto& add_kernel_desc(Args&&... args) + _CCCL_HOST_API auto& add_kernel_desc(Args&&... args) { payload->*[&](auto& self) { self.add_kernel_desc(::cuda::std::forward(args)...); @@ -186,14 +186,14 @@ private: } template - decltype(auto) get(size_t submitted_index) const + _CCCL_HOST_API decltype(auto) get(size_t submitted_index) const { return payload->*[&](auto& self) { return self.template get(submitted_index); }; } - auto& start() + _CCCL_HOST_API auto& start() { payload->*[&](auto& self) { self.start(); @@ -201,7 +201,7 @@ private: return *this; } - auto& end() + _CCCL_HOST_API auto& end() { payload->*[&](auto& self) { self.end(); @@ -229,14 +229,14 @@ public: class unified_task { public: - unified_task(stream_task task) + _CCCL_HOST_API unified_task(stream_task task) : payload(mv(task)) {} - unified_task(graph_task task) + _CCCL_HOST_API unified_task(graph_task task) : payload(mv(task)) {} - auto& set_symbol(::std::string s) & + _CCCL_HOST_API auto& set_symbol(::std::string s) & { payload->*[&](auto& self) { self.set_symbol(mv(s)); @@ -244,7 +244,7 @@ public: return *this; } - auto&& set_symbol(::std::string s) && + _CCCL_HOST_API auto&& set_symbol(::std::string s) && { payload->*[&](auto& self) { self.set_symbol(mv(s)); @@ -252,7 +252,7 @@ public: return mv(*this); } - auto&& set_exec_place(exec_place e_place) & + _CCCL_HOST_API auto&& set_exec_place(exec_place e_place) & { payload->*[&](auto& self) { self.set_exec_place(mv(e_place)); @@ -260,7 +260,7 @@ public: return *this; } - auto&& set_exec_place(exec_place e_place) && + _CCCL_HOST_API auto&& set_exec_place(exec_place e_place) && { payload->*[&](auto& self) { self.set_exec_place(mv(e_place)); @@ -268,7 +268,7 @@ public: return mv(*this); } - auto& start() + _CCCL_HOST_API auto& start() { payload->*[&](auto& self) { self.start(); @@ -276,7 +276,7 @@ public: return *this; } - auto& end() + _CCCL_HOST_API auto& end() { payload->*[&](auto& self) { self.end(); @@ -284,7 +284,7 @@ public: return *this; } - void enable_capture() + _CCCL_HOST_API void enable_capture() { payload->*[&](auto& self) { self.enable_capture(); @@ -299,7 +299,7 @@ public: * @return stream_or_graph_dynamic_task& */ template - unified_task& add_deps(Args&&... args) + _CCCL_HOST_API unified_task& add_deps(Args&&... args) { payload->*[&](auto& self) { self.add_deps(::cuda::std::forward(args)...); @@ -316,7 +316,7 @@ public: * @return slice */ template - decltype(auto) get(size_t submitted_index) const + _CCCL_HOST_API decltype(auto) get(size_t submitted_index) const { return payload->*[&](auto& self) -> decltype(auto) { return self.template get(submitted_index); @@ -324,14 +324,14 @@ public: } template - void operator->*(Fun&& f) + _CCCL_HOST_API void operator->*(Fun&& f) { payload->*[&](auto& self) { self->*f; }; } - cudaStream_t get_stream() const + _CCCL_HOST_API cudaStream_t get_stream() const { return payload->*[&](auto& self) { return self.get_stream(); @@ -342,7 +342,7 @@ public: * valid for tasks running in a graph context that have NOT enabled stream * capture; aborts for stream-context tasks. Mutually exclusive with the * capture path (get_stream()): use one mechanism or the other. */ - cudaGraph_t get_graph() + _CCCL_HOST_API cudaGraph_t get_graph() { return payload->*[&](auto& self) -> cudaGraph_t { using task_t = ::cuda::std::decay_t; @@ -361,7 +361,7 @@ public: /** When the task's exec place is a grid (size > 1), get the stream for the place at \p place_index * (linear index). Returns nullptr for graph_task (no per-place streams), for non-grid exec places, or * when \p place_index is out of range. */ - cudaStream_t get_stream(size_t place_index) const + _CCCL_HOST_API cudaStream_t get_stream(size_t place_index) const { return payload->*[&](auto& self) -> cudaStream_t { if constexpr (::cuda::std::is_same_v, ::cuda::std::decay_t>) @@ -385,7 +385,7 @@ public: /** When the task's exec place is a grid (size > 1), write its shape to \p out_dims and return true; else return * false. */ - bool get_grid_dims(dim4* out_dims) const + _CCCL_HOST_API bool get_grid_dims(dim4* out_dims) const { if (out_dims == nullptr) { @@ -404,14 +404,14 @@ public: // Get the underlying task base class - both stream_task and graph_task inherit from task. This is convenient when // we do not need the "typed" task, for example when using the "low-level" add_deps method. - ::cuda::experimental::stf::task& get_base_task() + _CCCL_HOST_API ::cuda::experimental::stf::task& get_base_task() { return payload->*[](auto& self) -> ::cuda::experimental::stf::task& { return self.get_base_task(); }; } - const ::cuda::experimental::stf::task& get_base_task() const + _CCCL_HOST_API const ::cuda::experimental::stf::task& get_base_task() const { return payload->*[](auto& self) -> const ::cuda::experimental::stf::task& { return self.get_base_task(); @@ -425,7 +425,7 @@ public: /** * @brief Default constructor for the context class. */ - context() = default; + _CCCL_HIDE_FROM_ABI context() = default; /** * @brief Constructs a stream context with a CUDA stream and an optional asynchronous resource handle. @@ -433,7 +433,7 @@ public: * @param stream The CUDA stream to be used in the context. * @param handle Optional asynchronous resource handle. */ - context(cudaStream_t stream, async_resources_handle handle = async_resources_handle(nullptr)) + _CCCL_HOST_API context(cudaStream_t stream, async_resources_handle handle = async_resources_handle(nullptr)) : payload(stream_ctx(stream, handle)) { // The default choice is stream_ctx, otherwise we should assign a graph_ctx with the appropriate parameters @@ -444,7 +444,7 @@ public: * * @param handle The asynchronous resource handle. */ - context(async_resources_handle handle) + _CCCL_HOST_API context(async_resources_handle handle) : payload(stream_ctx(handle)) { // The default choice is stream_ctx, otherwise we should assign a graph_ctx with the appropriate parameters @@ -455,7 +455,7 @@ public: * * @param ctx The context to be assigned. */ - context(stream_ctx ctx) + _CCCL_HOST_API context(stream_ctx ctx) : payload(mv(ctx)) {} @@ -464,7 +464,7 @@ public: * * @param ctx The context to be assigned. */ - context(graph_ctx ctx) + _CCCL_HOST_API context(graph_ctx ctx) : payload(mv(ctx)) {} @@ -476,7 +476,7 @@ public: * @return Reference to the updated context. */ template - context& operator=(Ctx ctx) + _CCCL_HOST_API context& operator=(Ctx ctx) { payload = mv(ctx); return *this; @@ -487,7 +487,7 @@ public: * * @return A string representation of the context. */ - ::std::string to_string() const + _CCCL_HOST_API ::std::string to_string() const { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); return payload->*[&](auto& self) { @@ -498,14 +498,14 @@ public: /** * @brief Returns an event list which depends on the completion of work in the stream */ - auto stream_to_event_list(cudaStream_t stream, ::std::string str) const + _CCCL_HOST_API auto stream_to_event_list(cudaStream_t stream, ::std::string str) const { return payload->*[&](auto& self) { return self.stream_to_event_list(stream, mv(str)); }; } - void set_graph_cache_policy(::std::function policy) + _CCCL_HOST_API void set_graph_cache_policy(::std::function policy) { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); payload->*[&](auto& self) { @@ -513,7 +513,7 @@ public: }; } - auto get_graph_cache_policy() const + _CCCL_HOST_API auto get_graph_cache_policy() const { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); return payload->*[&](auto& self) { @@ -521,7 +521,7 @@ public: }; } - executable_graph_cache_stat* graph_get_cache_stat() + _CCCL_HOST_API executable_graph_cache_stat* graph_get_cache_stat() { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); return payload->*[&](auto& self) { @@ -529,7 +529,7 @@ public: }; } - cudaGraph_t graph() const + _CCCL_HOST_API cudaGraph_t graph() const { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); return payload->*[&](auto& self) { @@ -537,7 +537,7 @@ public: }; } - size_t stage() const + _CCCL_HOST_API size_t stage() const { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); return payload->*[&](auto& self) { @@ -548,7 +548,7 @@ public: /** * @brief Returns the number of tasks created since the context was created or since the last fence (if any) */ - size_t task_count() const + _CCCL_HOST_API size_t task_count() const { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); return payload->*[&](auto& self) { @@ -565,7 +565,7 @@ public: * @param othersizes The sizes of other dimensions. */ template - auto logical_data(size_t elements, Sizes... othersizes) + _CCCL_HOST_API auto logical_data(size_t elements, Sizes... othersizes) { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); return payload->*[&](auto& self) { @@ -582,7 +582,7 @@ public: * @param ps The other parameters. */ template - auto logical_data(P0&& p0, Ps&&... ps) + _CCCL_HOST_API auto logical_data(P0&& p0, Ps&&... ps) { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); using T0 = ::cuda::std::remove_reference_t; @@ -600,7 +600,7 @@ public: } } - auto token() + _CCCL_HOST_API auto token() { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); return payload->*[&](auto& self) { @@ -609,7 +609,7 @@ public: } template - frozen_logical_data + _CCCL_HOST_API frozen_logical_data freeze(::cuda::experimental::stf::logical_data d, access_mode m = access_mode::read, data_place where = data_place::invalid(), @@ -620,11 +620,11 @@ public: }; } - frozen_logical_data_untyped - freeze(::cuda::experimental::stf::logical_data_untyped d, - access_mode m = access_mode::read, - data_place where = data_place::invalid(), - bool user_freeze = true) + _CCCL_HOST_API frozen_logical_data_untyped freeze( + ::cuda::experimental::stf::logical_data_untyped d, + access_mode m = access_mode::read, + data_place where = data_place::invalid(), + bool user_freeze = true) { return payload->*[&](auto& self) { return self.freeze(mv(d), m, mv(where), user_freeze); @@ -641,7 +641,7 @@ public: * @return The created logical data. */ template - auto logical_data(T* p, size_t n, data_place dplace = data_place::host()) + _CCCL_HOST_API auto logical_data(T* p, size_t n, data_place dplace = data_place::host()) { _CCCL_ASSERT(!dplace.is_invalid(), ""); _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); @@ -651,7 +651,7 @@ public: } template - unified_task task(exec_place e_place, task_dep... deps) + _CCCL_HOST_API unified_task task(exec_place e_place, task_dep... deps) { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); // Workaround: For some obscure reason `mv(deps)...` fails to compile @@ -661,7 +661,7 @@ public: } template - unified_task task(task_dep... deps) + _CCCL_HOST_API unified_task task(task_dep... deps) { return task(default_exec_place(), mv(deps)...); } @@ -674,7 +674,7 @@ public: typename S, typename... Deps, typename = ::cuda::std::enable_if_t<::cuda::std::is_base_of_v>> - auto parallel_for(exec_place_t e_place, S shape, Deps... deps) + _CCCL_HOST_API auto parallel_for(exec_place_t e_place, S shape, Deps... deps) { if constexpr (::cuda::std::is_integral_v) { @@ -696,7 +696,7 @@ public: typename S, typename... Deps, typename = ::cuda::std::enable_if_t<::cuda::std::is_base_of_v>> - auto parallel_for(partitioner_t p, exec_place_t e_place, S shape, Deps... deps) + _CCCL_HOST_API auto parallel_for(partitioner_t p, exec_place_t e_place, S shape, Deps... deps) { EXPECT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized."); using result_t = unified_scope, @@ -707,14 +707,14 @@ public: } template - auto parallel_for(S shape, Deps... deps) + _CCCL_HOST_API auto parallel_for(S shape, Deps... deps) { return parallel_for(default_exec_place(), mv(shape), mv(deps)...); } #endif // !defined(CUDASTF_DISABLE_CODE_GENERATION) && _CCCL_CUDA_COMPILATION() template - auto host_launch(task_dep... deps) + _CCCL_HOST_API auto host_launch(task_dep... deps) { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); using result_t = unified_scope, @@ -725,7 +725,7 @@ public: } template - auto cuda_kernel(task_dep... deps) + _CCCL_HOST_API auto cuda_kernel(task_dep... deps) { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); // false : we expect a single kernel descriptor in the lambda function return type @@ -737,7 +737,7 @@ public: } template - auto cuda_kernel(exec_place e_place, task_dep... deps) + _CCCL_HOST_API auto cuda_kernel(exec_place e_place, task_dep... deps) { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); // false : we expect a single kernel descriptor in the lambda function return type @@ -749,7 +749,7 @@ public: } template - auto cuda_kernel_chain(task_dep... deps) + _CCCL_HOST_API auto cuda_kernel_chain(task_dep... deps) { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); // true : we expect a vector of cuda kernel descriptors in the lambda function return type @@ -761,7 +761,7 @@ public: } template - auto cuda_kernel_chain(exec_place e_place, task_dep... deps) + _CCCL_HOST_API auto cuda_kernel_chain(exec_place e_place, task_dep... deps) { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); // true : we expect a vector of cuda kernel descriptors in the lambda function return type @@ -774,7 +774,7 @@ public: #if !defined(CUDASTF_DISABLE_CODE_GENERATION) && _CCCL_CUDA_COMPILATION() template - auto launch(thread_hierarchy_spec_t spec, exec_place e_place, task_dep... deps) + _CCCL_HOST_API auto launch(thread_hierarchy_spec_t spec, exec_place e_place, task_dep... deps) { using result_t = unified_scope, reserved::launch_scope>; @@ -786,26 +786,26 @@ public: // /* Default execution policy, explicit place */ // default depth to avoid breaking all codes (XXX temporary) template - auto launch(exec_place e_place, task_dep... deps) + _CCCL_HOST_API auto launch(exec_place e_place, task_dep... deps) { return launch(par(par()), mv(e_place), deps...); } // /* Default execution policy, on automatically selected device */ template - auto launch(task_dep... deps) + _CCCL_HOST_API auto launch(task_dep... deps) { return launch(default_exec_place(), mv(deps)...); } template - auto launch(thread_hierarchy_spec ths, task_dep... deps) + _CCCL_HOST_API auto launch(thread_hierarchy_spec ths, task_dep... deps) { return launch(mv(ths), default_exec_place(), mv(deps)...); } #endif // !defined(CUDASTF_DISABLE_CODE_GENERATION) && _CCCL_CUDA_COMPILATION() - auto repeat(size_t count) + _CCCL_HOST_API auto repeat(size_t count) { using result_t = unified_scope, reserved::repeat_scope>; return payload->*[&](auto& self) { @@ -813,7 +813,7 @@ public: }; } - auto repeat(::std::function condition) + _CCCL_HOST_API auto repeat(::std::function condition) { using result_t = unified_scope, reserved::repeat_scope>; return payload->*[&](auto& self) { @@ -821,7 +821,7 @@ public: }; } - cudaStream_t fence() + _CCCL_HOST_API cudaStream_t fence() { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); return payload->*[&](auto& self) { @@ -829,7 +829,7 @@ public: }; } - void finalize() + _CCCL_HOST_API void finalize() { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); payload->*[](auto& self) { @@ -838,7 +838,7 @@ public: } //! Add a resource to be managed by this context - void add_resource(::std::shared_ptr resource) + _CCCL_HOST_API void add_resource(::std::shared_ptr resource) { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); payload->*[&resource](auto& self) { @@ -849,7 +849,7 @@ public: //! Release context resources using the provided stream. //! Normally called automatically during finalize(); when using finalize_as_graph() //! for replayable graphs, call once the graph will no longer be used. - void release_resources(cudaStream_t stream) + _CCCL_HOST_API void release_resources(cudaStream_t stream) { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); payload->*[stream](auto& self) { @@ -859,14 +859,14 @@ public: //! Take all resources from \p other (e.g. a nested context) and merge them into this context. //! \p other will have no resources after this call. - void import_resources_from(context& other) + _CCCL_HOST_API void import_resources_from(context& other) { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); import_resources(other.export_resources()); } private: - ctx_resource_set export_resources() + _CCCL_HOST_API ctx_resource_set export_resources() { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); return payload->*[](auto& self) { @@ -874,7 +874,7 @@ private: }; } - void import_resources(ctx_resource_set&& other) + _CCCL_HOST_API void import_resources(ctx_resource_set&& other) { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); payload->*[&other](auto& self) { @@ -883,7 +883,7 @@ private: } public: - void submit() + _CCCL_HOST_API void submit() { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); payload->*[](auto& self) { @@ -891,7 +891,7 @@ public: }; } - void set_allocator(block_allocator_untyped custom_allocator) + _CCCL_HOST_API void set_allocator(block_allocator_untyped custom_allocator) { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); payload->*[&](auto& self) { @@ -899,7 +899,7 @@ public: }; } - void attach_allocator(block_allocator_untyped custom_allocator) + _CCCL_HOST_API void attach_allocator(block_allocator_untyped custom_allocator) { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); payload->*[&](auto& self) { @@ -907,14 +907,14 @@ public: }; } - void update_uncached_allocator(block_allocator_untyped custom) + _CCCL_HOST_API void update_uncached_allocator(block_allocator_untyped custom) { payload->*[&](auto& self) { self.update_uncached_allocator(mv(custom)); }; } - void change_stage() + _CCCL_HOST_API void change_stage() { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); payload->*[](auto& self) { @@ -922,7 +922,7 @@ public: }; } - ::std::shared_ptr get_dot() + _CCCL_HOST_API ::std::shared_ptr get_dot() { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); return payload->*[](auto& self) { @@ -934,7 +934,7 @@ public: // the backend wait() returns void and only blocks on the token's // dependencies, so `auto` deduces void here as well. template - auto wait(::cuda::experimental::stf::logical_data& ldata) + _CCCL_HOST_API auto wait(::cuda::experimental::stf::logical_data& ldata) { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); return payload->*[&ldata](auto& self) { @@ -943,7 +943,7 @@ public: } template - void set_parent_ctx(parent_ctx_t& parent_ctx) + _CCCL_HOST_API void set_parent_ctx(parent_ctx_t& parent_ctx) { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); reserved::per_ctx_dot::set_parent_ctx(parent_ctx.get_dot(), get_dot()); @@ -952,7 +952,7 @@ public: }; } - void enable_logical_data_stats() + _CCCL_HOST_API void enable_logical_data_stats() { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); payload->*[&](auto& self) { @@ -963,7 +963,7 @@ public: /** * @brief RAII-style description of a new section in the DOT file identified by its symbol */ - auto dot_section(::std::string symbol) const + _CCCL_HOST_API auto dot_section(::std::string symbol) const { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); return payload->*[&](auto& self) { @@ -973,7 +973,7 @@ public: /* Indicates whether the underlying context is a graph context, so that we * may specialize code to deal with the specific constraints of CUDA graphs. */ - bool is_graph_ctx() const + _CCCL_HOST_API bool is_graph_ctx() const { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); return payload->*[&](auto& self) { @@ -981,7 +981,7 @@ public: }; } - async_resources_handle& async_resources() const + _CCCL_HOST_API async_resources_handle& async_resources() const { // if (payload.index() == 0) { // return ::cuda::std::get<0>(payload).async_resources(); @@ -994,29 +994,29 @@ public: } // Shortcuts to manipulate the current affinity stored in the async_resources_handle of the ctx - void push_affinity(::std::vector<::std::shared_ptr> p) const + _CCCL_HOST_API void push_affinity(::std::vector<::std::shared_ptr> p) const { async_resources().push_affinity(mv(p)); } - void push_affinity(::std::shared_ptr p) const + _CCCL_HOST_API void push_affinity(::std::shared_ptr p) const { async_resources().push_affinity(mv(p)); } - void pop_affinity() const + _CCCL_HOST_API void pop_affinity() const { async_resources().pop_affinity(); } - const ::std::vector<::std::shared_ptr>& current_affinity() const + _CCCL_HOST_API const ::std::vector<::std::shared_ptr>& current_affinity() const { return async_resources().current_affinity(); } - const exec_place& current_exec_place() const + _CCCL_HOST_API const exec_place& current_exec_place() const { _CCCL_ASSERT(!current_affinity().empty(), "current_exec_place no affinity set"); return *(current_affinity()[0]); } - bool has_affinity() const + _CCCL_HOST_API bool has_affinity() const { return async_resources().has_affinity(); } @@ -1031,12 +1031,12 @@ public: * context, in which case we take the first execution place in the current * places. */ - exec_place default_exec_place() const + _CCCL_HOST_API exec_place default_exec_place() const { return has_affinity() ? current_exec_place() : exec_place::current_device(); } - graph_ctx to_graph_ctx() const + _CCCL_HOST_API graph_ctx to_graph_ctx() const { // Check if payload holds graph_ctx (index == 1) if (auto ctx = ::cuda::std::get_if(&payload)) @@ -1053,7 +1053,7 @@ public: * this stream after the context has been finalized is an undefined * behaviour. */ - cudaStream_t pick_stream() + _CCCL_HOST_API cudaStream_t pick_stream() { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); return payload->*[](auto& self) { @@ -1066,7 +1066,7 @@ public: * * @return Reference to the backend_ctx_untyped base class from the variant payload */ - backend_ctx_untyped& get_backend() + _CCCL_HOST_API backend_ctx_untyped& get_backend() { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); return payload->*[](auto& ctx) -> backend_ctx_untyped& { @@ -1074,7 +1074,7 @@ public: }; } - const backend_ctx_untyped& get_backend() const + _CCCL_HOST_API const backend_ctx_untyped& get_backend() const { _CCCL_ASSERT(payload.index() != ::cuda::std::variant_npos, "Context is not initialized"); return payload->*[](const auto& ctx) -> const backend_ctx_untyped& { @@ -1154,14 +1154,14 @@ namespace reserved struct dummy_released_resource : ctx_resource { bool* released_ = nullptr; - explicit dummy_released_resource(bool* released) + _CCCL_HOST_API explicit dummy_released_resource(bool* released) : released_(released) {} - bool can_release_in_callback() const noexcept override + _CCCL_HOST_API bool can_release_in_callback() const noexcept override { return true; } - void release_in_callback() noexcept override + _CCCL_HOST_API void release_in_callback() noexcept override { if (released_) { @@ -1250,7 +1250,7 @@ UNITTEST("context with arguments") # if !defined(CUDASTF_DISABLE_CODE_GENERATION) && _CCCL_CUDA_COMPILATION() namespace reserved { -inline void unit_test_context_pfor_integral() +_CCCL_HOST_API inline void unit_test_context_pfor_integral() { context ctx; auto lA = ctx.logical_data(shape_of>(64)); @@ -1273,7 +1273,7 @@ UNITTEST("context parallel_for integral shape") unit_test_context_pfor_integral(); }; -inline void unit_test_context_pfor() +_CCCL_HOST_API inline void unit_test_context_pfor() { context ctx; SCOPE(exit) @@ -1298,7 +1298,7 @@ UNITTEST("context parallel_for") }; template -inline void unit_test_context_launch() +_CCCL_HOST_API inline void unit_test_context_launch() { context ctx; if constexpr (use_graph) @@ -1349,7 +1349,7 @@ UNITTEST("context launch") }; /* Do not provide an exec_place, but put a spec */ -inline void unit_test_context_launch_spec_noplace() +_CCCL_HOST_API inline void unit_test_context_launch_spec_noplace() { context ctx; SCOPE(exit) @@ -1376,7 +1376,7 @@ UNITTEST("context launch spec noplace") unit_test_context_launch_spec_noplace(); }; -inline void unit_test_context_launch_generic() +_CCCL_HOST_API inline void unit_test_context_launch_generic() { context ctx; SCOPE(exit) @@ -1414,7 +1414,7 @@ UNITTEST("context launch test generic") unit_test_context_launch_generic(); }; -inline void unit_test_context_launch_exec_places() +_CCCL_HOST_API inline void unit_test_context_launch_exec_places() { // OK with this // stream_ctx ctx; @@ -1453,7 +1453,7 @@ UNITTEST("context launch specific exec places") unit_test_context_launch_exec_places(); }; -inline void unit_test_context_launch_sync() +_CCCL_HOST_API inline void unit_test_context_launch_sync() { // OK with this (workaround) stream_ctx ctx; @@ -1496,7 +1496,7 @@ UNITTEST("context launch sync") unit_test_context_launch_sync(); }; -inline void unit_test_context_repeat() +_CCCL_HOST_API inline void unit_test_context_repeat() { context ctx; @@ -1542,7 +1542,7 @@ UNITTEST("context repeat") }; template -inline void unit_test_context_launch_implicit_widths(spec_t spec) +_CCCL_HOST_API inline void unit_test_context_launch_implicit_widths(spec_t spec) { // OK with this (workaround) stream_ctx ctx; @@ -1615,7 +1615,7 @@ UNITTEST("context task") ctx.finalize(); }; -inline void unit_test_recursive_apply() +_CCCL_HOST_API inline void unit_test_recursive_apply() { context ctx; SCOPE(exit) @@ -1680,7 +1680,7 @@ UNITTEST("logical data slice const") ctx.finalize(); }; -inline void unit_test_partitioner_product() +_CCCL_HOST_API inline void unit_test_partitioner_product() { context ctx; SCOPE(exit) diff --git a/cudax/include/cuda/experimental/__stf/internal/execution_policy.cuh b/cudax/include/cuda/experimental/__stf/internal/execution_policy.cuh index 18dcef59431c..2b8d279396d8 100644 --- a/cudax/include/cuda/experimental/__stf/internal/execution_policy.cuh +++ b/cudax/include/cuda/experimental/__stf/internal/execution_policy.cuh @@ -74,7 +74,7 @@ enum class hw_scope : unsigned int * @note The function includes assertions to ensure that lhs and rhs are * within the allowed range. */ -inline hw_scope operator|(const hw_scope& lhs, const hw_scope& rhs) +_CCCL_HOST_API inline hw_scope operator|(const hw_scope& lhs, const hw_scope& rhs) { assert(as_underlying(lhs) <= as_underlying(hw_scope::all)); assert(as_underlying(rhs) <= as_underlying(hw_scope::all)); @@ -94,7 +94,7 @@ inline hw_scope operator|(const hw_scope& lhs, const hw_scope& rhs) * @note The function includes assertions to ensure that lhs and rhs are * within the allowed range. */ -inline hw_scope operator&(const hw_scope& lhs, const hw_scope& rhs) +_CCCL_HOST_API inline hw_scope operator&(const hw_scope& lhs, const hw_scope& rhs) { assert(as_underlying(lhs) <= as_underlying(hw_scope::all)); assert(as_underlying(rhs) <= as_underlying(hw_scope::all)); @@ -112,7 +112,7 @@ inline hw_scope operator&(const hw_scope& lhs, const hw_scope& rhs) //! //! @note The function includes an assertion to ensure that s is within the allowed range. //! Any bits beyond those represented by `hw_scope::all` are cleared in the result. -inline hw_scope operator~(const hw_scope& s) +_CCCL_HOST_API inline hw_scope operator~(const hw_scope& s) { assert(as_underlying(s) <= as_underlying(hw_scope::all)); // Keep the bits beyond `all` zero at all times, and @@ -171,10 +171,10 @@ public: using thread_hierarchy_t = thread_hierarchy; ///@{ @name Usual constructors - thread_hierarchy_spec() = default; - thread_hierarchy_spec(const thread_hierarchy_spec&) = default; - thread_hierarchy_spec(thread_hierarchy_spec&) = default; - thread_hierarchy_spec(thread_hierarchy_spec&&) = default; + _CCCL_HIDE_FROM_ABI thread_hierarchy_spec() = default; + _CCCL_HIDE_FROM_ABI thread_hierarchy_spec(const thread_hierarchy_spec&) = default; + _CCCL_HIDE_FROM_ABI thread_hierarchy_spec(thread_hierarchy_spec&) = default; + _CCCL_HIDE_FROM_ABI thread_hierarchy_spec(thread_hierarchy_spec&&) = default; ///@} /** @@ -193,7 +193,7 @@ public: * @param p Values of parameters */ template - explicit thread_hierarchy_spec(const P&... p) + _CCCL_HOST_API explicit thread_hierarchy_spec(const P&... p) : inner(reserved::only_convertible_or(thread_hierarchy_spec(), p...)) , dynamic_width(reserved::only_convertible_or(decltype(dynamic_width)(), p...)) , sync_scope(reserved::only_convertible_or(hw_scope::all, p...)) @@ -210,7 +210,7 @@ public: } } - constexpr bool operator==(const thread_hierarchy_spec& rhs) const noexcept + _CCCL_HOST_API constexpr bool operator==(const thread_hierarchy_spec& rhs) const noexcept { if (dynamic_width != rhs.dynamic_width || sync_scope != rhs.sync_scope || mem_bytes != rhs.mem_bytes) { @@ -229,33 +229,33 @@ public: // For other types ... template >, int> = 0> - constexpr bool operator==(const thread_hierarchy_spec&) const noexcept + _CCCL_HOST_API constexpr bool operator==(const thread_hierarchy_spec&) const noexcept { return false; } - constexpr bool operator!=(const thread_hierarchy_spec& rhs) const noexcept + _CCCL_HOST_API constexpr bool operator!=(const thread_hierarchy_spec& rhs) const noexcept { return !(*this == rhs); } template >, int> = 0> - constexpr bool operator!=(const thread_hierarchy_spec&) const noexcept + _CCCL_HOST_API constexpr bool operator!=(const thread_hierarchy_spec&) const noexcept { return true; } /// @brief Compute the depth of the thread hierarchy. /// @return The depth. - static constexpr size_t depth() + _CCCL_HOST_API static constexpr size_t depth() { return 1 + (sizeof...(lower_levels) / 2); } /// @brief Check if synchronization is possible. /// @return A boolean indicating if synchronization is possible. - static constexpr bool synchronizable() + _CCCL_HOST_API static constexpr bool synchronizable() { return can_sync; } @@ -266,7 +266,7 @@ public: * @param inner The inner thread_hierarchy_spec object. */ template - void set_inner(const thread_hierarchy_spec& inner) + _CCCL_HOST_API void set_inner(const thread_hierarchy_spec& inner) { this->inner = inner; } @@ -276,7 +276,7 @@ public: * @param level The level. * @return The memory bytes. */ - constexpr mem get_mem(size_t level) const + _CCCL_HOST_API constexpr mem get_mem(size_t level) const { if constexpr (depth() > 1) { @@ -293,7 +293,7 @@ public: * @param level The level. * @param value The memory bytes. */ - constexpr void set_mem(size_t level, mem value) + _CCCL_HOST_API constexpr void set_mem(size_t level, mem value) { if constexpr (depth() > 1) { @@ -332,7 +332,7 @@ public: * @param level The level * @return The width (0 if width is dynamic) */ - static constexpr size_t static_width(size_t level) + _CCCL_HOST_API static constexpr size_t static_width(size_t level) { const size_t data[] = {width, lower_levels...}; return data[2 * level]; @@ -343,7 +343,7 @@ public: * @param level The level. * @return The width. */ - constexpr size_t get_width(size_t level) const + _CCCL_HOST_API constexpr size_t get_width(size_t level) const { if constexpr (depth() > 1) { @@ -359,7 +359,7 @@ public: * @brief Set the width. * @param new_width The new width. */ - void set_width(size_t level, size_t new_width) + _CCCL_HOST_API void set_width(size_t level, size_t new_width) { if constexpr (depth() > 1) { @@ -381,7 +381,7 @@ public: /** * @brief Get the scope at a specific level */ - constexpr hw_scope get_scope(size_t level) const + _CCCL_HOST_API constexpr hw_scope get_scope(size_t level) const { if constexpr (depth() > 1) { @@ -419,7 +419,7 @@ private: * @return A `thread_hierarchy_spec` instantiation with the appropriate arguments. */ template -constexpr auto par(const P&... p) +_CCCL_HOST_API constexpr auto par(const P&... p) { using R = typename reserved::deduce_execution_policy::type; return R(p...); @@ -441,7 +441,7 @@ constexpr auto par(const P&... p) * @return A `thread_hierarchy_spec` instantiation with the appropriate arguments. */ template -constexpr auto par(const P&... p) +_CCCL_HOST_API constexpr auto par(const P&... p) { using R = typename reserved::deduce_execution_policy::type; return R(p...); @@ -462,7 +462,7 @@ constexpr auto par(const P&... p) * @return A `thread_hierarchy_spec` instantiation with the appropriate arguments. */ template -constexpr auto con(const P&... p) +_CCCL_HOST_API constexpr auto con(const P&... p) { using R = typename reserved::deduce_execution_policy::type; return R(p...); @@ -483,7 +483,7 @@ constexpr auto con(const P&... p) * @return A `thread_hierarchy_spec` instantiation with the appropriate arguments. */ template -constexpr auto con(const P&... p) +_CCCL_HOST_API constexpr auto con(const P&... p) { using R = typename reserved::deduce_execution_policy::type; return R(p...); diff --git a/cudax/include/cuda/experimental/__stf/internal/inner_shape.cuh b/cudax/include/cuda/experimental/__stf/internal/inner_shape.cuh index 7f9cac7f51fb..91fb054ac3d8 100644 --- a/cudax/include/cuda/experimental/__stf/internal/inner_shape.cuh +++ b/cudax/include/cuda/experimental/__stf/internal/inner_shape.cuh @@ -46,7 +46,7 @@ namespace cuda::experimental::stf * */ template -_CCCL_HOST_DEVICE box::rank()> inner(const shape_of>& s) +_CCCL_HOST_DEVICE_API box::rank()> inner(const shape_of>& s) { using m = mdspan; constexpr size_t rank = m::rank(); @@ -77,7 +77,7 @@ _CCCL_HOST_DEVICE box::rank()> inner(const shape_of -_CCCL_HOST_DEVICE box inner(const box& s) +_CCCL_HOST_DEVICE_API box inner(const box& s) { ::std::array<::std::pair<::std::ptrdiff_t, ::std::ptrdiff_t>, rank> inner_extents; for (size_t i = 0; i < rank; i++) diff --git a/cudax/include/cuda/experimental/__stf/internal/interpreted_execution_policy.cuh b/cudax/include/cuda/experimental/__stf/internal/interpreted_execution_policy.cuh index c57936b829d7..bd2c1664bcab 100644 --- a/cudax/include/cuda/experimental/__stf/internal/interpreted_execution_policy.cuh +++ b/cudax/include/cuda/experimental/__stf/internal/interpreted_execution_policy.cuh @@ -64,18 +64,18 @@ public: class level { public: - level(::std::pair desc, size_t local_mem = 0) + _CCCL_HOST_API level(::std::pair desc, size_t local_mem = 0) : local_mem(local_mem) { level_desc.push_back(mv(desc)); } - level(::std::initializer_list<::std::pair> desc, size_t local_mem = 0) + _CCCL_HOST_API level(::std::initializer_list<::std::pair> desc, size_t local_mem = 0) : level_desc(desc.begin(), desc.end()) , local_mem(local_mem) {} // Get the total width of the level, which is the product of all sublevel sizes - size_t width() const + _CCCL_HOST_API size_t width() const { size_t res = 1; for (auto& e : level_desc) @@ -85,25 +85,25 @@ public: return res; } - void set_mem(size_t size) + _CCCL_HOST_API void set_mem(size_t size) { local_mem = size; } - size_t get_mem() const + _CCCL_HOST_API size_t get_mem() const { return local_mem; } - void set_sync(bool sync) + _CCCL_HOST_API void set_sync(bool sync) { may_sync = sync; } - bool get_sync() const + _CCCL_HOST_API bool get_sync() const { return may_sync; } - const auto& get_desc() const + _CCCL_HOST_API const auto& get_desc() const { return level_desc; } @@ -114,20 +114,20 @@ public: bool may_sync = false; }; - interpreted_execution_policy() = default; - ~interpreted_execution_policy() = default; + _CCCL_HIDE_FROM_ABI interpreted_execution_policy() = default; + _CCCL_HIDE_FROM_ABI ~interpreted_execution_policy() = default; private: /* A substem that contains sync() related functionality for the thread hierarchy */ reserved::cooperative_group_system cg_system; public: - reserved::cooperative_group_system& get_cg_system() + _CCCL_HOST_API reserved::cooperative_group_system& get_cg_system() { return cg_system; } - const reserved::cooperative_group_system& get_cg_system() const + _CCCL_HOST_API const reserved::cooperative_group_system& get_cg_system() const { return cg_system; } @@ -135,49 +135,49 @@ public: template interpreted_execution_policy(const thread_hierarchy_spec& p, const exec_place& where, const Fun& f); - void add_level(level l) + _CCCL_HOST_API void add_level(level l) { levels.push_back(mv(l)); } - void set_level_mem(int level, size_t size) + _CCCL_HOST_API void set_level_mem(int level, size_t size) { assert(level < int(depth)); levels[level].set_mem(size); } - size_t get_level_mem(int level) const + _CCCL_HOST_API size_t get_level_mem(int level) const { assert(level < depth); return levels[level].get_mem(); } - void set_level_sync(int level, bool sync) + _CCCL_HOST_API void set_level_sync(int level, bool sync) { assert(level < int(depth)); levels[level].set_sync(sync); } - bool get_level_sync(size_t level) const + _CCCL_HOST_API bool get_level_sync(size_t level) const { assert(level < int(depth)); return levels[level].get_sync(); } // Returns the width of a level - size_t width(int l) const + _CCCL_HOST_API size_t width(int l) const { EXPECT(static_cast(l) < depth); return levels[l].width(); } - const auto& get_levels() const + _CCCL_HOST_API const auto& get_levels() const { return levels; } /* Compute the kernel configuration given all levels */ - ::std::array get_config() const + _CCCL_HOST_API ::std::array get_config() const { ::std::array config = {1, 1, 1}; @@ -215,13 +215,13 @@ public: /* Helper function to return the hw scope of the last level in the interpreted thread hierarchy. Probably a more * elegant and safer way to do the below exists. */ - hw_scope last_level_scope() const + _CCCL_HOST_API hw_scope last_level_scope() const { return (levels[0].get_desc())[0].first; } /* Compute overall memory requirements */ - ::std::array get_mem_config() const + _CCCL_HOST_API ::std::array get_mem_config() const { /* We take the kernel configuration : config[2] is the number of * threads per block, config[1] the number of blocks, config[0] the @@ -280,7 +280,7 @@ public: return mem_config; } - bool need_cooperative_kernel_launch() const + _CCCL_HOST_API bool need_cooperative_kernel_launch() const { ::std::array config = get_config(); const size_t block_scope_size = config[2]; @@ -302,11 +302,11 @@ public: return false; } - void set_system_mem(void* addr) + _CCCL_HOST_API void set_system_mem(void* addr) { system_mem = addr; } - void* get_system_mem() const + _CCCL_HOST_API void* get_system_mem() const { return system_mem; } diff --git a/cudax/include/cuda/experimental/__stf/internal/slice.cuh b/cudax/include/cuda/experimental/__stf/internal/slice.cuh index b3267bfcc91c..d453be2d0cbc 100644 --- a/cudax/include/cuda/experimental/__stf/internal/slice.cuh +++ b/cudax/include/cuda/experimental/__stf/internal/slice.cuh @@ -76,15 +76,15 @@ struct layout_stride : ::cuda::std::layout_stride template struct mapping : ::cuda::std::layout_stride::mapping { - constexpr mapping() = default; + _CCCL_HIDE_FROM_ABI constexpr mapping() = default; template - constexpr _CCCL_HOST_DEVICE mapping(A&&... a) + constexpr _CCCL_HOST_DEVICE_API mapping(A&&... a) : ::cuda::std::layout_stride::mapping(::cuda::std::forward(a)...) {} template - constexpr _CCCL_HOST_DEVICE auto operator()(is_t&&... is) const + constexpr _CCCL_HOST_DEVICE_API auto operator()(is_t&&... is) const { #ifdef CUDASTF_BOUNDSCHECK each_in_pack( @@ -116,7 +116,7 @@ using slice = mdspan, layout_stride>; * @return size_t */ template -size_t contiguous_dims(const S& span) +_CCCL_HOST_API size_t contiguous_dims(const S& span) { if (span.rank() < 2) { @@ -149,7 +149,8 @@ size_t contiguous_dims(const S& span) * @return auto `slice` */ template -_CCCL_HOST_DEVICE auto make_slice(ElementType* data, const ::std::tuple& extents, const Strides&... strides) +_CCCL_HOST_DEVICE_API auto +make_slice(ElementType* data, const ::std::tuple& extents, const Strides&... strides) { static_assert(sizeof...(Extents) == sizeof...(Strides) + 1); using Result = slice; @@ -171,7 +172,7 @@ _CCCL_HOST_DEVICE auto make_slice(ElementType* data, const ::std::tuple -_CCCL_HOST_DEVICE auto make_slice(T* data, const Extents&... extents) +_CCCL_HOST_DEVICE_API auto make_slice(T* data, const Extents&... extents) { using Result = slice; static_assert(sizeof...(Extents) == Result::rank()); @@ -201,7 +202,7 @@ _CCCL_HOST_DEVICE auto make_slice(T* data, const Extents&... extents) namespace reserved { template -auto make_mdview(Whatevs&&... whatevs) +_CCCL_HOST_API auto make_mdview(Whatevs&&... whatevs) { return make_slice(::cuda::std::forward(whatevs)...); } @@ -374,13 +375,13 @@ public: * @brief Dimensionality of the slice. * */ - _CCCL_HOST_DEVICE static constexpr size_t rank() + _CCCL_HOST_DEVICE_API static constexpr size_t rank() { return described_type::rank(); } // Functions to create the begin and end iterators - _CCCL_HOST_DEVICE auto begin() + _CCCL_HOST_DEVICE_API auto begin() { ::std::array sizes; unroll([&](auto i) { @@ -389,7 +390,7 @@ public: return box(sizes).begin(); } - _CCCL_HOST_DEVICE auto end() + _CCCL_HOST_DEVICE_API auto end() { ::std::array sizes; unroll([&](auto i) { @@ -403,14 +404,14 @@ public: * * All `shape_of` specializations must define this constructor. */ - shape_of() = default; + _CCCL_HIDE_FROM_ABI shape_of() = default; /** * @brief Copies a shape. * * All `shape_of` specializations must define this constructor. */ - shape_of(const shape_of&) = default; + _CCCL_HIDE_FROM_ABI shape_of(const shape_of&) = default; /** * @brief Extracts the shape from a given slice. @@ -419,7 +420,7 @@ public: * * All `shape_of` specializations must define this constructor. */ - explicit _CCCL_HOST_DEVICE shape_of(const described_type& x) + explicit _CCCL_HOST_DEVICE_API shape_of(const described_type& x) : extents(x.extents()) , strides(x.mapping().strides()) {} @@ -431,7 +432,7 @@ public: * * This constructor is optional. */ - explicit shape_of(const coords_t& sizes) + _CCCL_HOST_API explicit shape_of(const coords_t& sizes) : extents(sizes) { size_t product_sizes = 1; @@ -458,7 +459,7 @@ public: * Initializes dimensions to `size0`, `sizes...`. This constructor is optional. */ template - explicit shape_of(size_t size0, Sizes&&... sizes) + _CCCL_HOST_API explicit shape_of(size_t size0, Sizes&&... sizes) : extents(size0, ::cuda::std::forward(sizes)...) { static_assert(sizeof...(sizes) + 1 == rank(), "Wrong number of arguments passed to shape_of."); @@ -473,7 +474,8 @@ public: } ///@{ @name Constructors - explicit shape_of(const ::cuda::std::array& sizes, const ::cuda::std::array& _strides) + _CCCL_HOST_API explicit shape_of(const ::cuda::std::array& sizes, + const ::cuda::std::array& _strides) : shape_of(sizes) { if constexpr (rank() > 1) @@ -491,7 +493,7 @@ public: } ///@} - bool operator==(const shape_of& other) const + _CCCL_HOST_API bool operator==(const shape_of& other) const { return extents == other.extents && strides == other.strides; } @@ -504,7 +506,7 @@ public: * * This member function is optional. */ - constexpr _CCCL_HOST_DEVICE size_t extent(size_t dim) const + constexpr _CCCL_HOST_DEVICE_API size_t extent(size_t dim) const { assert(dim < rank()); return extents.extent(dim); @@ -516,7 +518,7 @@ public: * @param dim The dimension for which to get the stride. * @return The stride for the specified dimension. */ - constexpr _CCCL_HOST_DEVICE size_t stride(size_t dim) const + constexpr _CCCL_HOST_DEVICE_API size_t stride(size_t dim) const { assert(dim < rank()); return strides[dim]; @@ -529,7 +531,7 @@ public: * * This member function is optional. */ - _CCCL_HOST_DEVICE size_t size() const + _CCCL_HOST_DEVICE_API size_t size() const { size_t result = 1; for (size_t i = 0; i != rank(); ++i) @@ -544,7 +546,7 @@ public: * * @return const std::array& */ - _CCCL_HOST_DEVICE ::std::array get_sizes() const + _CCCL_HOST_DEVICE_API ::std::array get_sizes() const { ::std::array result; for (size_t i = 0; i < rank(); ++i) @@ -559,7 +561,7 @@ public: * * @return const dim4 */ - dim4 get_data_dims() const + _CCCL_HOST_API dim4 get_data_dims() const { static_assert(rank() < 5); @@ -598,13 +600,13 @@ public: * @param base Base pointer to create the described_type object. * @return described_type Newly created described_type object. */ - described_type create(T* base) const + _CCCL_HOST_API described_type create(T* base) const { return described_type(base, typename described_type::mapping_type(extents, strides)); } // This transforms a tuple of (shape, 1D index) into a coordinate - _CCCL_HOST_DEVICE coords_t index_to_coords(size_t index) const + _CCCL_HOST_DEVICE_API coords_t index_to_coords(size_t index) const { coords_t coordinates{}; // for (::std::ptrdiff_t i = _dimensions - 1; i >= 0; i--) @@ -814,7 +816,7 @@ UNITTEST("3D slice should be similar to 3D mdspan", (slice())) namespace reserved { template -void for_each_contiguous_hunk(const mdspan& s, F&& f) +_CCCL_HOST_API void for_each_contiguous_hunk(const mdspan& s, F&& f) { constexpr auto rank = mdspan::extents_type::rank(); const size_t c = contiguous_dims(s); @@ -855,7 +857,7 @@ void for_each_contiguous_hunk(const mdspan& s, F&& f) * failed partway and rolled back, or an already-unpinned slice). */ template -void unpin(mdspan& s) +_CCCL_HOST_API void unpin(mdspan& s) { reserved::for_each_contiguous_hunk(s, [](T* base, size_t /*n*/) { unpin_memory(base); @@ -871,7 +873,7 @@ void unpin(mdspan& s) * @return true if the slice was newly pinned, false if it was already pinned */ template -bool pin(mdspan& s) +_CCCL_HOST_API bool pin(mdspan& s) { if (address_is_pinned(s.data_handle())) { @@ -923,7 +925,7 @@ _CCCL_DIAG_SUPPRESS_MSVC(4702) // unreachable code //! this is checked at compile time. //! @note If the mdspan is empty, the function returns 0. template -size_t data_hash(mdspan s, ::cuda::std::index_sequence = {}) +_CCCL_HOST_API size_t data_hash(mdspan s, ::cuda::std::index_sequence = {}) { static_assert(reserved::has_std_hash_v || reserved::has_cudastf_hash_v, "data_hash requires ::std::hash or a cudastf hash for the element type E"); @@ -980,9 +982,8 @@ _CCCL_DIAG_POP * Write the content of the mdspan into a file */ template -void data_dump([[maybe_unused]] mdspan s, - ::std::ostream& file = ::std::cerr, - ::cuda::std::index_sequence = {}) +_CCCL_HOST_API void data_dump( + [[maybe_unused]] mdspan s, ::std::ostream& file = ::std::cerr, ::cuda::std::index_sequence = {}) { using Slice = mdspan; if constexpr (reserved::has_ostream_operator::value) @@ -1077,7 +1078,7 @@ UNITTEST("data_hash") template struct hash> { - ::std::size_t operator()(mdspan const& s) const noexcept + _CCCL_HOST_API ::std::size_t operator()(mdspan const& s) const noexcept { static constexpr auto _dimensions = mdspan::rank(); // Combine hashes from the ptr, sizes and strides diff --git a/cudax/include/cuda/experimental/__stf/internal/thread_hierarchy.cuh b/cudax/include/cuda/experimental/__stf/internal/thread_hierarchy.cuh index bce8169fe14c..3f57e436dfe6 100644 --- a/cudax/include/cuda/experimental/__stf/internal/thread_hierarchy.cuh +++ b/cudax/include/cuda/experimental/__stf/internal/thread_hierarchy.cuh @@ -76,14 +76,14 @@ class thread_hierarchy public: // For tag-types - thread_hierarchy() = default; + _CCCL_HIDE_FROM_ABI thread_hierarchy() = default; template friend class thread_hierarchy; // Construct from an outer thread hierarchy (peel off one level) template - _CCCL_HOST_DEVICE thread_hierarchy(const thread_hierarchy& outer) + _CCCL_HOST_DEVICE_API thread_hierarchy(const thread_hierarchy& outer) : devid(outer.devid) , launch_config(outer.launch_config) , cg_system(outer.cg_system) @@ -102,7 +102,7 @@ public: * on the hardware, and generates a thread_hierarchy object that can be passed * to kernels as an argument */ - thread_hierarchy(int devid, interpreted_execution_policy& p) + _CCCL_HOST_API thread_hierarchy(int devid, interpreted_execution_policy& p) : devid(devid) , launch_config(p.get_config()) , cg_system(p.get_cg_system()) @@ -122,19 +122,19 @@ public: * @param level The level * @return The width (0 if width is dynamic) */ - static constexpr size_t static_width(size_t level) + _CCCL_HOST_API static constexpr size_t static_width(size_t level) { size_t data[] = {spec...}; return data[1 + (2 * level)]; } - _CCCL_HOST_DEVICE const ::std::array& get_config() const + _CCCL_HOST_DEVICE_API const ::std::array& get_config() const { return launch_config; } // Rank from the root - _CCCL_HOST_DEVICE size_t rank([[maybe_unused]] int level, [[maybe_unused]] int root_level) const + _CCCL_HOST_DEVICE_API size_t rank([[maybe_unused]] int level, [[maybe_unused]] int root_level) const { NV_IF_ELSE_TARGET( NV_IS_DEVICE, @@ -169,7 +169,7 @@ public: _CCCL_UNREACHABLE(); } - _CCCL_HOST_DEVICE size_t size([[maybe_unused]] int level, [[maybe_unused]] int root_level) const + _CCCL_HOST_DEVICE_API size_t size([[maybe_unused]] int level, [[maybe_unused]] int root_level) const { if constexpr (depth == 0) { @@ -189,17 +189,17 @@ public: _CCCL_UNREACHABLE(); } - _CCCL_HOST_DEVICE size_t size(int level = int(depth) - 1) const + _CCCL_HOST_DEVICE_API size_t size(int level = int(depth) - 1) const { return size(level, -1); } - _CCCL_HOST_DEVICE size_t rank(int level = int(depth) - 1) const + _CCCL_HOST_DEVICE_API size_t rank(int level = int(depth) - 1) const { return rank(level, -1); } - _CCCL_HOST_DEVICE void sync([[maybe_unused]] int level = 0) + _CCCL_HOST_DEVICE_API void sync([[maybe_unused]] int level = 0) { assert(level >= 0); assert(level < depth); @@ -250,13 +250,13 @@ public: } template - auto remove_first_tuple_element(const ::std::tuple& t) + _CCCL_HOST_API auto remove_first_tuple_element(const ::std::tuple& t) { return ::std::make_tuple(::std::get(t)...); } template - _CCCL_HOST_DEVICE auto apply_partition(const shape_t& s, const ::std::tuple& t) const + _CCCL_HOST_DEVICE_API auto apply_partition(const shape_t& s, const ::std::tuple& t) const { auto s0 = P::apply(s, pos4(rank(0)), dim4(size(0))); auto sans_first = make_tuple_indexwise([&](auto index) { @@ -274,7 +274,7 @@ public: // Default partitioner tuple template - _CCCL_HOST_DEVICE auto apply_partition(const shape_t& s) const + _CCCL_HOST_DEVICE_API auto apply_partition(const shape_t& s) const { if constexpr (depth == 1) { @@ -292,13 +292,13 @@ public: * * @return `thread_hierarchy` instantiated with `spec` sans the first two arguments */ - _CCCL_HOST_DEVICE auto inner() const + _CCCL_HOST_DEVICE_API auto inner() const { return typename inner_t::type(*this); } template - _CCCL_HOST_DEVICE slice storage(int level) + _CCCL_HOST_DEVICE_API slice storage(int level) { assert(level >= 0); assert(level < depth); @@ -349,17 +349,17 @@ public: return make_slice(static_cast(nullptr), 0); } - void set_device_tmp(void* addr) + _CCCL_HOST_API void set_device_tmp(void* addr) { device_tmp = addr; } - void set_system_tmp(void* addr) + _CCCL_HOST_API void set_system_tmp(void* addr) { system_tmp = addr; } - void set_devid(int d) + _CCCL_HOST_API void set_devid(int d) { devid = d; } @@ -387,20 +387,20 @@ private: /* Special case to work-around spec = <> */ template - _CCCL_HOST_DEVICE bool may_sync_impl(int) const + _CCCL_HOST_DEVICE_API bool may_sync_impl(int) const { return false; } template - _CCCL_HOST_DEVICE bool may_sync_impl(int level) const + _CCCL_HOST_DEVICE_API bool may_sync_impl(int level) const { static_assert(Idx < sizeof...(spec) / 2); return (level == Idx) ? sync : may_sync_impl(level); } /* Evaluate at runtime if we can call sync on the specified level */ - _CCCL_HOST_DEVICE bool may_sync(int level) const + _CCCL_HOST_DEVICE_API bool may_sync(int level) const { return may_sync_impl<0, spec...>(level); } @@ -411,7 +411,7 @@ private: namespace reserved { template -__global__ void unit_test_thread_hierarchy(thread_hierarchy h) +_CCCL_KERNEL_ATTRIBUTES void unit_test_thread_hierarchy(thread_hierarchy h) { assert(h.size(0) == 2); assert(h.size(1) == 2 * 32); @@ -447,7 +447,7 @@ UNITTEST("thread hierarchy indexing") namespace reserved { template -__global__ void unit_test_thread_hierarchy_sync(thread_hierarchy h) +_CCCL_KERNEL_ATTRIBUTES void unit_test_thread_hierarchy_sync(thread_hierarchy h) { assert(h.rank(1, 0) == threadIdx.x); assert(h.rank(0, -1) == blockIdx.x); @@ -486,7 +486,7 @@ UNITTEST("thread hierarchy sync") namespace reserved { template -__global__ void unit_test_thread_hierarchy_inner_sync(thread_hierarchy h) +_CCCL_KERNEL_ATTRIBUTES void unit_test_thread_hierarchy_inner_sync(thread_hierarchy h) { h.inner().sync(); auto h_inner = h.inner(); diff --git a/cudax/include/cuda/experimental/__stf/stackable/stackable_ctx.cuh b/cudax/include/cuda/experimental/__stf/stackable/stackable_ctx.cuh index 8ddf489eea46..312b8688de49 100644 --- a/cudax/include/cuda/experimental/__stf/stackable/stackable_ctx.cuh +++ b/cudax/include/cuda/experimental/__stf/stackable/stackable_ctx.cuh @@ -83,13 +83,13 @@ namespace cuda::experimental::stf * this checks if a subsequent operation requesting a different mode is valid. * For example, data imported as read-only cannot be written. */ -inline bool access_mode_is_mutating(access_mode m) +_CCCL_HOST_API inline bool access_mode_is_mutating(access_mode m) { return m == access_mode::write || m == access_mode::rw || m == access_mode::reduce || m == access_mode::reduce_no_init; } -inline bool access_mode_permits(access_mode granted, access_mode requested) +_CCCL_HOST_API inline bool access_mode_permits(access_mode granted, access_mode requested) { return !access_mode_is_mutating(requested) || access_mode_is_mutating(granted); } @@ -105,16 +105,17 @@ public: /// @brief Alias for `T` - matches logical_data convention using element_type = T; - stackable_logical_data() = default; + _CCCL_HIDE_FROM_ABI stackable_logical_data() = default; - stackable_logical_data(const stackable_logical_data&) = default; - stackable_logical_data& operator=(const stackable_logical_data&) = default; - stackable_logical_data(stackable_logical_data&&) noexcept = default; - stackable_logical_data& operator=(stackable_logical_data&&) noexcept = default; + _CCCL_HIDE_FROM_ABI stackable_logical_data(const stackable_logical_data&) = default; + _CCCL_HIDE_FROM_ABI stackable_logical_data& operator=(const stackable_logical_data&) = default; + _CCCL_HIDE_FROM_ABI stackable_logical_data(stackable_logical_data&&) noexcept = default; + _CCCL_HIDE_FROM_ABI stackable_logical_data& operator=(stackable_logical_data&&) noexcept = default; /* Create a logical data in the stackable ctx : in order to make it possible * to export all the way down to the root context, we create the logical data * in the root, and import them. */ + _CCCL_HOST_API stackable_logical_data(stackable_ctx sctx, int ctx_offset, bool ld_from_shape, logical_data ld, bool can_export) : owner_(::std::make_shared(::std::make_shared(mv(sctx)))) { @@ -152,7 +153,7 @@ public: } } - int get_data_root_offset() const + _CCCL_HOST_API int get_data_root_offset() const { return data().data_root_offset; } @@ -163,110 +164,110 @@ public: // this logical data, so a reference into the vector would dangle as soon as // the shared lock below is released. The handle itself is a cheap shared_ptr // wrapper and stays valid regardless of vector growth. - logical_data get_ld(int offset) const + _CCCL_HOST_API logical_data get_ld(int offset) const { auto lock = data().acquire_shared_lock(); _CCCL_ASSERT(offset >= 0 && data().was_imported(offset), "Failed to find imported data"); return data().get_data_node(offset).ld; } - int get_unique_id() const + _CCCL_HOST_API int get_unique_id() const { return get_ld(get_data_root_offset()).get_unique_id(); } - void push(int ctx_offset, access_mode m, data_place where = data_place::invalid()) const + _CCCL_HOST_API void push(int ctx_offset, access_mode m, data_place where = data_place::invalid()) const { // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) const_cast(this)->push_at(ctx_offset, m, mv(where)); } - void push(access_mode m, data_place where = data_place::invalid()) const + _CCCL_HOST_API void push(access_mode m, data_place where = data_place::invalid()) const { push(data().sctx.get_head_offset(), m, mv(where)); } - stackable_ctx& sctx() + _CCCL_HOST_API stackable_ctx& sctx() { return mut_data().sctx; } - const stackable_ctx& sctx() const + _CCCL_HOST_API const stackable_ctx& sctx() const { return data().sctx; } // Helpers — return lazy stackable_task_dep descriptors. - auto read() const + _CCCL_HOST_API auto read() const { return stackable_task_dep(*this, access_mode::read); } - auto read(data_place dp) const + _CCCL_HOST_API auto read(data_place dp) const { return stackable_task_dep(*this, access_mode::read, mv(dp)); } - auto write() + _CCCL_HOST_API auto write() { return stackable_task_dep(*this, access_mode::write); } - auto write(data_place dp) + _CCCL_HOST_API auto write(data_place dp) { return stackable_task_dep(*this, access_mode::write, mv(dp)); } - auto rw() + _CCCL_HOST_API auto rw() { return stackable_task_dep(*this, access_mode::rw); } - auto rw(data_place dp) + _CCCL_HOST_API auto rw(data_place dp) { return stackable_task_dep(*this, access_mode::rw, mv(dp)); } template - auto reduce(Op) + _CCCL_HOST_API auto reduce(Op) { return stackable_task_dep(*this, access_mode::reduce); } template - auto reduce(Op, data_place dp) + _CCCL_HOST_API auto reduce(Op, data_place dp) { return stackable_task_dep(*this, access_mode::reduce, mv(dp)); } template - auto reduce(Op, no_init) + _CCCL_HOST_API auto reduce(Op, no_init) { return stackable_task_dep(*this, access_mode::reduce_no_init); } template - auto reduce(Op, no_init, data_place dp) + _CCCL_HOST_API auto reduce(Op, no_init, data_place dp) { return stackable_task_dep(*this, access_mode::reduce_no_init, mv(dp)); } - auto dep_with_mode(access_mode m) + _CCCL_HOST_API auto dep_with_mode(access_mode m) { return stackable_task_dep(*this, m); } - auto dep_with_mode(access_mode m, data_place dp) + _CCCL_HOST_API auto dep_with_mode(access_mode m, data_place dp) { return stackable_task_dep(*this, m, mv(dp)); } - auto shape() const + _CCCL_HOST_API auto shape() const { return get_ld(get_data_root_offset()).shape(); } - auto& set_symbol(::std::string symbol) + _CCCL_HOST_API auto& set_symbol(::std::string symbol) { auto& st = mut_data(); auto ctx_lock = st.sctx.acquire_exclusive_lock(); @@ -277,7 +278,7 @@ public: return *this; } - void set_write_back(bool flag) + _CCCL_HOST_API void set_write_back(bool flag) { const auto& st = data(); _CCCL_ASSERT(st.data_root_offset >= 0, "invalid value"); @@ -285,17 +286,17 @@ public: mut_data().get_data_node(st.data_root_offset).ld.set_write_back(flag); } - void set_read_only(bool flag = true) + _CCCL_HOST_API void set_read_only(bool flag = true) { mut_data().read_only = flag; } - bool is_read_only() const + _CCCL_HOST_API bool is_read_only() const { return data().read_only; } - auto get_symbol() const + _CCCL_HOST_API auto get_symbol() const { return data().symbol; } @@ -305,10 +306,11 @@ public: // if necessary. // // Returns true if the task_dep needs an update - bool validate_access(int ctx_offset, - const stackable_ctx& sctx_ref, - access_mode m, - const data_place& dplace_hint = data_place::invalid()) const + _CCCL_HOST_API bool validate_access( + int ctx_offset, + const stackable_ctx& sctx_ref, + access_mode m, + const data_place& dplace_hint = data_place::invalid()) const { auto& self = *const_cast(this); // NOLINT(cppcoreguidelines-pro-type-const-cast) auto lock = self.mut_data().acquire_exclusive_lock(); @@ -378,11 +380,11 @@ private: class state : public stackable_logical_data_impl_state_base { public: - explicit state(stackable_ctx _sctx) + _CCCL_HOST_API explicit state(stackable_ctx _sctx) : sctx(mv(_sctx)) {} - void pop_before_finalize(int ctx_offset) override + _CCCL_HOST_API void pop_before_finalize(int ctx_offset) override { const int parent_offset = sctx.get_parent_offset(ctx_offset); _CCCL_ASSERT(parent_offset >= 0, ""); @@ -419,7 +421,7 @@ private: sctx.get_node(ctx_offset)->ctx.get_dot()->ctx_add_output_id(parent_dnode.frozen_ld.value().unfreeze_fake_task_id()); } - void pop_after_finalize(int parent_offset, const event_list& finalize_prereqs) override + _CCCL_HOST_API void pop_after_finalize(int parent_offset, const event_list& finalize_prereqs) override { const nvtx_range r("stackable_logical_data::pop_after_finalize"); @@ -441,7 +443,7 @@ private: struct data_node { - explicit data_node(logical_data ld) + _CCCL_HOST_API explicit data_node(logical_data ld) : ld(mv(ld)) {} @@ -452,7 +454,7 @@ private: access_mode effective_mode = access_mode::none; }; - auto& get_data_node(int offset) + _CCCL_HOST_API auto& get_data_node(int offset) { _CCCL_ASSERT(offset >= 0, "invalid value"); const auto idx = static_cast(offset); @@ -460,7 +462,7 @@ private: return data_nodes[idx].value(); } - const auto& get_data_node(int offset) const + _CCCL_HOST_API const auto& get_data_node(int offset) const { _CCCL_ASSERT(offset >= 0, "invalid value"); const auto idx = static_cast(offset); @@ -468,9 +470,9 @@ private: return data_nodes[idx].value(); } - void grow_data_nodes(size_t target_size, - size_t factor_numerator = node_hierarchy::default_growth_numerator, - size_t factor_denominator = node_hierarchy::default_growth_denominator) + _CCCL_HOST_API void grow_data_nodes(size_t target_size, + size_t factor_numerator = node_hierarchy::default_growth_numerator, + size_t factor_denominator = node_hierarchy::default_growth_denominator) { if (target_size <= data_nodes.size()) { @@ -481,7 +483,7 @@ private: data_nodes.resize(new_size); } - bool was_imported(int offset) const + _CCCL_HOST_API bool was_imported(int offset) const { if (offset < 0) { @@ -492,32 +494,32 @@ private: return idx < data_nodes.size() && data_nodes[idx].has_value(); } - void mark_access(int offset, access_mode m) + _CCCL_HOST_API void mark_access(int offset, access_mode m) { _CCCL_ASSERT(offset >= 0, "Failed to find data node for mark_access"); _CCCL_ASSERT(data_nodes[static_cast(offset)].has_value(), "Failed to find data node for mark_access"); data_nodes[static_cast(offset)].value().effective_mode |= m; } - bool is_frozen(int offset) const + _CCCL_HOST_API bool is_frozen(int offset) const { _CCCL_ASSERT(offset >= 0, ""); _CCCL_ASSERT(data_nodes[static_cast(offset)].has_value(), ""); return data_nodes[static_cast(offset)].value().frozen_ld.has_value(); } - access_mode get_frozen_mode(int offset) const + _CCCL_HOST_API access_mode get_frozen_mode(int offset) const { _CCCL_ASSERT(is_frozen(offset), ""); return data_nodes[static_cast(offset)].value().frozen_ld.value().get_access_mode(); } - ::std::shared_lock<::std::shared_mutex> acquire_shared_lock() const + _CCCL_HOST_API ::std::shared_lock<::std::shared_mutex> acquire_shared_lock() const { return ::std::shared_lock<::std::shared_mutex>(mutex); } - ::std::unique_lock<::std::shared_mutex> acquire_exclusive_lock() + _CCCL_HOST_API ::std::unique_lock<::std::shared_mutex> acquire_exclusive_lock() { return ::std::unique_lock<::std::shared_mutex>(mutex); } @@ -539,11 +541,11 @@ private: { ::std::shared_ptr payload; - explicit owner(::std::shared_ptr st) + _CCCL_HOST_API explicit owner(::std::shared_ptr st) : payload(mv(st)) {} - ~owner() + _CCCL_HOST_API ~owner() { if (payload) { @@ -557,19 +559,19 @@ private: owner& operator=(owner&&) = delete; }; - const state& data() const + _CCCL_HOST_API const state& data() const { _CCCL_ASSERT(owner_ && owner_->payload, "stackable_logical_data used after move or default construction"); return *owner_->payload; } - state& mut_data() + _CCCL_HOST_API state& mut_data() { _CCCL_ASSERT(owner_ && owner_->payload, "stackable_logical_data used after move or default construction"); return *owner_->payload; } - static void adopt_into_context(const ::std::shared_ptr& st) + _CCCL_HOST_API static void adopt_into_context(const ::std::shared_ptr& st) { if (!st) { @@ -597,7 +599,7 @@ private: } template - static void traverse_data_nodes(state& st, Func&& func) + _CCCL_HOST_API static void traverse_data_nodes(state& st, Func&& func) { ::std::stack node_stack; node_stack.push(st.data_root_offset); @@ -620,12 +622,12 @@ private: } } - void mark_access_at(int offset, access_mode m) + _CCCL_HOST_API void mark_access_at(int offset, access_mode m) { mut_data().mark_access(offset, m); } - void push_at(int ctx_offset, access_mode m, data_place where = data_place::invalid()) + _CCCL_HOST_API void push_at(int ctx_offset, access_mode m, data_place where = data_place::invalid()) { auto& st = mut_data(); const int parent_offset = st.sctx.get_parent_offset(ctx_offset); @@ -773,7 +775,7 @@ private: ::std::shared_ptr owner_; }; -inline stackable_logical_data stackable_ctx::token() +_CCCL_HOST_API inline stackable_logical_data stackable_ctx::token() { const int head = pimpl->get_head_offset(); return stackable_logical_data(*this, head, true, get_root_ctx().token(), true); @@ -794,6 +796,7 @@ public: using op_type = reduce_op; static constexpr bool does_work = !::cuda::std::is_same_v; + _CCCL_HOST_API stackable_task_dep(stackable_logical_data _d, access_mode _mode, data_place _dplace = data_place::affine()) : d(mv(_d)) , mode(_mode) @@ -802,7 +805,7 @@ public: // Implicit conversion to task_dep for interop with non-stackable contexts. // Resolves at the current thread's head offset. - operator task_dep() const + _CCCL_HOST_API operator task_dep() const { auto& sctx = d.sctx(); int offset = sctx.get_head_offset(); @@ -810,24 +813,24 @@ public: return resolve(offset); } - const stackable_logical_data& get_d() const + _CCCL_HOST_API const stackable_logical_data& get_d() const { return d; } - access_mode get_access_mode() const + _CCCL_HOST_API access_mode get_access_mode() const { return mode; } - const data_place& get_dplace() const + _CCCL_HOST_API const data_place& get_dplace() const { return dplace; } // Create a concrete task_dep from the logical data at the given context offset. // Callers must ensure validate_access has already been called for this offset. - task_dep resolve(int context_offset) const + _CCCL_HOST_API task_dep resolve(int context_offset) const { auto context_ld = d.get_ld(context_offset); return task_dep(context_ld, mode, dplace); @@ -862,7 +865,7 @@ private: class launchable_graph_handle { public: - launchable_graph_handle() = default; + _CCCL_HIDE_FROM_ABI launchable_graph_handle() = default; //! \brief Underlying executable graph. Aborts if the handle is invalid. //! @@ -876,7 +879,7 @@ public: //! Note: unlike `graph()`, calling `exec()` *does* force instantiation. //! Callers that only want to embed the graph as a child node in another //! graph should use `graph()` and avoid `exec()` entirely. - cudaGraphExec_t exec() const + _CCCL_HOST_API cudaGraphExec_t exec() const { _CCCL_VERIFY(!token_.expired(), "launchable_graph_handle::exec() called after pop_epilogue()"); auto shared_exec = ctx_.prepare_handle_for_exec(node_offset_); @@ -889,7 +892,7 @@ public: //! Does NOT perform any stream synchronization - it is purely observational. //! If you plan to drive the graph manually, call `exec()` first (it will //! lazily sync), or use `launch()` which does both in one step. - cudaStream_t stream() const + _CCCL_HOST_API cudaStream_t stream() const { _CCCL_VERIFY(!token_.expired(), "launchable_graph_handle::stream() called after pop_epilogue()"); return support_stream_; @@ -899,7 +902,7 @@ public: //! //! On the first call, waits for the nested context's freeze/get events //! (dep A). Subsequent calls skip the sync and issue the launch directly. - void launch() + _CCCL_HOST_API void launch() { _CCCL_VERIFY(!token_.expired(), "launchable_graph_handle::launch() called after pop_epilogue()"); ctx_.launch_prepared_graph(node_offset_, support_stream_); @@ -934,7 +937,7 @@ public: //! work has completed (e.g. via `cudaStreamSynchronize` on your launch //! stream) before calling `pop_epilogue()`, otherwise the unfreeze of the //! pushed data will race the child graph. - cudaGraph_t graph() const + _CCCL_HOST_API cudaGraph_t graph() const { _CCCL_VERIFY(!token_.expired(), "launchable_graph_handle::graph() called after pop_epilogue()"); ctx_.prepare_handle_for_graph(node_offset_); @@ -942,12 +945,12 @@ public: } //! \brief True iff the owning stackable_ctx has not yet run pop_epilogue(). - bool valid() const + _CCCL_HOST_API bool valid() const { return !token_.expired(); } - explicit operator bool() const + _CCCL_HOST_API explicit operator bool() const { return valid(); } @@ -968,7 +971,7 @@ private: cudaStream_t support_stream_ = nullptr; }; -inline launchable_graph_handle stackable_ctx::pop_prologue() +_CCCL_HOST_API inline launchable_graph_handle stackable_ctx::pop_prologue() { auto r = pimpl->pop_prologue_impl(); @@ -1007,8 +1010,8 @@ class stackable_ctx::graph_scope_guard public: using context_type = stackable_ctx; - explicit graph_scope_guard(stackable_ctx& ctx, - ::cuda::std::source_location loc = ::cuda::std::source_location::current()) + _CCCL_HOST_API explicit graph_scope_guard(stackable_ctx& ctx, + ::cuda::std::source_location loc = ::cuda::std::source_location::current()) : ctx_(ctx) { ctx_.push(loc); @@ -1016,7 +1019,7 @@ public: // pop() completes the level's teardown whatever its steps report; a destructor cannot pass the // first failure on, so the policy is to report it and end the program. - ~graph_scope_guard() + _CCCL_HOST_API ~graph_scope_guard() { ctx_.pop(sump::abort); } @@ -1030,7 +1033,7 @@ private: stackable_ctx& ctx_; }; -inline stackable_ctx::graph_scope_guard stackable_ctx::graph_scope(::cuda::std::source_location loc) +_CCCL_HOST_API inline stackable_ctx::graph_scope_guard stackable_ctx::graph_scope(::cuda::std::source_location loc) { return graph_scope_guard(*this, loc); } @@ -1057,8 +1060,8 @@ class stackable_ctx::launchable_graph_scope public: using context_type = stackable_ctx; - explicit launchable_graph_scope(stackable_ctx& ctx, - ::cuda::std::source_location loc = ::cuda::std::source_location::current()) + _CCCL_HOST_API explicit launchable_graph_scope( + stackable_ctx& ctx, ::cuda::std::source_location loc = ::cuda::std::source_location::current()) : ctx_(ctx) { ctx_.push(loc); @@ -1066,7 +1069,7 @@ public: // or on release() if none of those were called. } - ~launchable_graph_scope() noexcept + _CCCL_HOST_API ~launchable_graph_scope() noexcept { release(); } @@ -1077,21 +1080,21 @@ public: launchable_graph_scope& operator=(launchable_graph_scope&&) = delete; //! \brief Launch the graph once. The first call triggers pop_prologue(). - void launch() + _CCCL_HOST_API void launch() { ensure_prepared_(); handle_.launch(); } //! \brief Expose the executable graph. Triggers pop_prologue() on demand. - cudaGraphExec_t exec() + _CCCL_HOST_API cudaGraphExec_t exec() { ensure_prepared_(); return handle_.exec(); } //! \brief Expose the support stream. Triggers pop_prologue() on demand. - cudaStream_t stream() + _CCCL_HOST_API cudaStream_t stream() { ensure_prepared_(); return handle_.stream(); @@ -1100,7 +1103,7 @@ public: //! \brief Expose the underlying CUDA graph (for embedding into another //! graph via cudaGraphAddChildGraphNode). Triggers pop_prologue() on //! demand. - cudaGraph_t graph() + _CCCL_HOST_API cudaGraph_t graph() { ensure_prepared_(); return handle_.graph(); @@ -1114,7 +1117,7 @@ public: //! Tearing the graph node down is not something a caller could retry or recover from, so //! a failure here terminates. // NOLINTNEXTLINE(bugprone-exception-escape) - void release() noexcept + _CCCL_HOST_API void release() noexcept { if (released_) { @@ -1135,7 +1138,7 @@ public: } private: - void ensure_prepared_() + _CCCL_HOST_API void ensure_prepared_() { _CCCL_VERIFY(!released_, "launchable_graph_scope used after release()"); if (!handle_) @@ -1207,15 +1210,15 @@ private: class stackable_ctx::launchable_graph { public: - launchable_graph() = default; - launchable_graph(const launchable_graph&) = default; - launchable_graph(launchable_graph&&) noexcept = default; - launchable_graph& operator=(const launchable_graph&) = default; - launchable_graph& operator=(launchable_graph&&) noexcept = default; - ~launchable_graph() = default; + _CCCL_HIDE_FROM_ABI launchable_graph() = default; + _CCCL_HIDE_FROM_ABI launchable_graph(const launchable_graph&) = default; + _CCCL_HIDE_FROM_ABI launchable_graph(launchable_graph&&) noexcept = default; + _CCCL_HIDE_FROM_ABI launchable_graph& operator=(const launchable_graph&) = default; + _CCCL_HIDE_FROM_ABI launchable_graph& operator=(launchable_graph&&) noexcept = default; + _CCCL_HIDE_FROM_ABI ~launchable_graph() = default; //! \brief Launch the graph once on its support stream. - void launch() + _CCCL_HOST_API void launch() { _CCCL_VERIFY(state_, "launchable_graph::launch() called on an empty/moved-from handle"); state_->handle.launch(); @@ -1223,14 +1226,14 @@ public: //! \brief Underlying executable graph. Triggers lazy instantiation + dep-A //! sync on the first call (same contract as `launchable_graph_handle::exec()`). - cudaGraphExec_t exec() const + _CCCL_HOST_API cudaGraphExec_t exec() const { _CCCL_VERIFY(state_, "launchable_graph::exec() called on an empty/moved-from handle"); return state_->handle.exec(); } //! \brief Support stream the graph was prepared against. Purely observational. - cudaStream_t stream() const + _CCCL_HOST_API cudaStream_t stream() const { _CCCL_VERIFY(state_, "launchable_graph::stream() called on an empty/moved-from handle"); return state_->handle.stream(); @@ -1238,7 +1241,7 @@ public: //! \brief Underlying cudaGraph_t topology (for embedding as a child graph). //! Triggers lazy dep-A sync but does NOT call `cudaGraphInstantiate`. - cudaGraph_t graph() const + _CCCL_HOST_API cudaGraph_t graph() const { _CCCL_VERIFY(state_, "launchable_graph::graph() called on an empty/moved-from handle"); return state_->handle.graph(); @@ -1247,12 +1250,12 @@ public: //! \brief True iff this copy still holds a shared reference and the //! underlying pop has not been epilogued (e.g. manually via //! `ctx.pop_epilogue()`). - bool valid() const noexcept + _CCCL_HOST_API bool valid() const noexcept { return state_ && state_->handle.valid(); } - explicit operator bool() const noexcept + _CCCL_HOST_API explicit operator bool() const noexcept { return valid(); } @@ -1260,14 +1263,14 @@ public: //! \brief Number of live shared copies referring to the same graph. Debug //! introspection only. Returns 0 for a default-constructed / moved-from //! instance. - long use_count() const noexcept + _CCCL_HOST_API long use_count() const noexcept { return state_ ? state_.use_count() : 0; } //! \brief Drop this shared reference eagerly. When this was the last copy, //! `pop_epilogue()` runs now instead of at destruction time. Idempotent. - void reset() noexcept + _CCCL_HOST_API void reset() noexcept { state_.reset(); } @@ -1283,7 +1286,7 @@ private: stackable_ctx ctx; launchable_graph_handle handle; - state(stackable_ctx c, launchable_graph_handle h) + _CCCL_HOST_API state(stackable_ctx c, launchable_graph_handle h) : ctx(mv(c)) , handle(mv(h)) {} @@ -1291,7 +1294,7 @@ private: // As in release(), a failing pop_epilogue() is not recoverable, so terminating is the // intended outcome. // NOLINTNEXTLINE(bugprone-exception-escape) - ~state() + _CCCL_HOST_API ~state() { // Guard against users who manually called pop_epilogue() on the ctx // while shared copies were still alive: the handle's token is expired @@ -1309,7 +1312,7 @@ private: ::std::shared_ptr state_; }; -inline stackable_ctx::launchable_graph stackable_ctx::pop_prologue_shared() +_CCCL_HOST_API inline stackable_ctx::launchable_graph stackable_ctx::pop_prologue_shared() { // pop_prologue() already performs the validity / state checks. auto handle = pop_prologue(); @@ -1329,7 +1332,7 @@ class stackable_ctx::while_graph_scope_guard public: using context_type = stackable_ctx; - explicit while_graph_scope_guard( + _CCCL_HOST_API explicit while_graph_scope_guard( stackable_ctx& ctx, unsigned int default_launch_value = 0, unsigned int flags = cudaGraphCondAssignDefault, @@ -1340,12 +1343,12 @@ public: } // As with graph_scope_guard: a destructor can only report the first failure and end the program. - ~while_graph_scope_guard() + _CCCL_HOST_API ~while_graph_scope_guard() { ctx_.pop(sump::abort); } - cudaGraphConditionalHandle cond_handle() const + _CCCL_HOST_API cudaGraphConditionalHandle cond_handle() const { return conditional_handle_; } @@ -1354,7 +1357,7 @@ public: class condition_update_scope { public: - condition_update_scope(stackable_ctx& ctx, cudaGraphConditionalHandle handle, Deps... deps) + _CCCL_HOST_API condition_update_scope(stackable_ctx& ctx, cudaGraphConditionalHandle handle, Deps... deps) : ctx_(ctx) , handle_(handle) , tdeps(mv(deps)...) @@ -1385,7 +1388,7 @@ public: using kernel_tuple_t = typename to_cuda_tuple::type; template - void operator->*(CondFunc&& cond_func) + _CCCL_HOST_API void operator->*(CondFunc&& cond_func) { ::std::apply( [this](auto&&... deps) { @@ -1414,7 +1417,7 @@ public: //! //! The lambda should return true to continue the loop, false to exit. template - auto update_cond(Args&&... args) + _CCCL_HOST_API auto update_cond(Args&&... args) { return condition_update_scope(ctx_, cond_handle(), args...); } @@ -1429,7 +1432,7 @@ private: cudaGraphConditionalHandle conditional_handle_{}; }; -inline stackable_ctx::while_graph_scope_guard stackable_ctx::while_graph_scope( +_CCCL_HOST_API inline stackable_ctx::while_graph_scope_guard stackable_ctx::while_graph_scope( unsigned int default_launch_value, unsigned int flags, ::cuda::std::source_location loc) { return while_graph_scope_guard(*this, default_launch_value, flags, loc); @@ -1464,7 +1467,7 @@ class repeat_graph_scope_guard { public: template - static void init_counter_value(stackable_ctx& ctx, CounterType counter, size_t count) + _CCCL_HOST_API static void init_counter_value(stackable_ctx& ctx, CounterType counter, size_t count) { ctx.parallel_for(box(1), counter.write())->*[count] __device__(size_t, auto counter) { *counter = count; @@ -1472,7 +1475,8 @@ public: } template - static void setup_condition_update(stackable_ctx::while_graph_scope_guard& while_guard, CounterType counter) + _CCCL_HOST_API static void + setup_condition_update(stackable_ctx::while_graph_scope_guard& while_guard, CounterType counter) { while_guard.update_cond(counter.read())->*[] __device__(auto counter) { (*counter)--; @@ -1480,7 +1484,7 @@ public: }; } - explicit repeat_graph_scope_guard( + _CCCL_HOST_API explicit repeat_graph_scope_guard( stackable_ctx& ctx, size_t count, unsigned int default_launch_value = 1, @@ -1512,7 +1516,7 @@ private: ::cuda::std::optional while_guard_; }; -inline auto stackable_ctx::repeat_graph_scope( +_CCCL_HOST_API inline auto stackable_ctx::repeat_graph_scope( size_t count, unsigned int default_launch_value, unsigned int flags, ::cuda::std::source_location loc) { return repeat_graph_scope_guard(*this, count, default_launch_value, flags, loc); @@ -1524,20 +1528,20 @@ inline auto stackable_ctx::repeat_graph_scope( namespace reserved { template -static __global__ void kernel_set(T* addr, T val) +_CCCL_KERNEL_ATTRIBUTES void kernel_set(T* addr, T val) { printf("SETTING ADDR %p at %d\n", addr, val); *addr = val; } template -static __global__ void kernel_add(T* addr, T val) +_CCCL_KERNEL_ATTRIBUTES void kernel_add(T* addr, T val) { *addr += val; } template -static __global__ void kernel_check_value(T* addr, T val) +_CCCL_KERNEL_ATTRIBUTES void kernel_check_value(T* addr, T val) { printf("CHECK %d EXPECTED %d\n", *addr, val); if (*addr != val) @@ -1668,7 +1672,7 @@ UNITTEST("graph_scope multiple sequential scopes") ctx.finalize(); }; -inline void test_graph_scope_with_tmp() +_CCCL_HOST_API inline void test_graph_scope_with_tmp() { stackable_ctx ctx; auto lA = ctx.logical_data(shape_of>(1024)); @@ -1704,7 +1708,7 @@ UNITTEST("graph_scope with temporary data") test_graph_scope_with_tmp(); }; -inline void test_graph_scope() +_CCCL_HOST_API inline void test_graph_scope() { stackable_ctx ctx; @@ -1784,7 +1788,7 @@ UNITTEST("stackable task with set_symbol and set_exec_place") }; }; -inline void test_pop_prologue_repeated_launch() +_CCCL_HOST_API inline void test_pop_prologue_repeated_launch() { constexpr int N = 16; @@ -1830,7 +1834,7 @@ UNITTEST("pop_prologue + repeated launch accumulates N times") test_pop_prologue_repeated_launch(); }; -inline void test_pop_prologue_manual_exec_launch() +_CCCL_HOST_API inline void test_pop_prologue_manual_exec_launch() { // Same setup as test_pop_prologue_repeated_launch, but drive the graph // manually via cudaGraphLaunch(handle.exec(), handle.stream()) as the @@ -1881,7 +1885,7 @@ UNITTEST("pop_prologue + manual cudaGraphLaunch via exec()/stream()") test_pop_prologue_manual_exec_launch(); }; -inline void test_pop_prologue_zero_launches() +_CCCL_HOST_API inline void test_pop_prologue_zero_launches() { stackable_ctx ctx; @@ -1928,7 +1932,7 @@ UNITTEST("pop_prologue + pop_epilogue with zero launches") test_pop_prologue_zero_launches(); }; -inline void test_pop_prologue_handle_invalidation() +_CCCL_HOST_API inline void test_pop_prologue_handle_invalidation() { stackable_ctx ctx; @@ -1965,7 +1969,7 @@ UNITTEST("pop_prologue invalidates handle after pop_epilogue") test_pop_prologue_handle_invalidation(); }; -inline void test_pop_prologue_graph_child_embed() +_CCCL_HOST_API inline void test_pop_prologue_graph_child_embed() { // Exercise launchable_graph_handle::graph(): embed the popped graph as a // child node in an outer, user-built graph instead of launching it through @@ -2052,7 +2056,7 @@ UNITTEST("pop_prologue graph() embedded as a child graph node") test_pop_prologue_graph_child_embed(); }; -inline void test_launchable_graph_scope_raii() +_CCCL_HOST_API inline void test_launchable_graph_scope_raii() { constexpr int N = 5; @@ -2094,7 +2098,7 @@ UNITTEST("launchable_graph_scope RAII") test_launchable_graph_scope_raii(); }; -inline void test_pop_prologue_shared_basic() +_CCCL_HOST_API inline void test_pop_prologue_shared_basic() { constexpr int N = 5; @@ -2154,7 +2158,7 @@ UNITTEST("pop_prologue_shared last copy triggers pop_epilogue") test_pop_prologue_shared_basic(); }; -inline void test_pop_prologue_shared_copies() +_CCCL_HOST_API inline void test_pop_prologue_shared_copies() { stackable_ctx ctx; @@ -2205,7 +2209,7 @@ UNITTEST("pop_prologue_shared multiple copies share a single graph") test_pop_prologue_shared_copies(); }; -inline void test_pop_prologue_shared_stored_in_container() +_CCCL_HOST_API inline void test_pop_prologue_shared_stored_in_container() { stackable_ctx ctx; @@ -2255,7 +2259,7 @@ UNITTEST("pop_prologue_shared storable across scopes / in containers") test_pop_prologue_shared_stored_in_container(); }; -inline void test_pop_prologue_shared_manual_epilogue_meh() +_CCCL_HOST_API inline void test_pop_prologue_shared_manual_epilogue_meh() { // If the user manually calls ctx.pop_epilogue() after creating shared // copies, outstanding copies must become invalid and the shared state @@ -2293,7 +2297,7 @@ UNITTEST("pop_prologue_shared tolerates manual pop_epilogue") }; # if _CCCL_CTK_AT_LEAST(12, 4) && !defined(CUDASTF_DISABLE_CODE_GENERATION) -inline void test_pop_prologue_with_while_graph_scope_meh() +_CCCL_HOST_API inline void test_pop_prologue_with_while_graph_scope_meh() { constexpr int N = 3; // re-launch the whole while-graph 3 times constexpr size_t inner_iters = 4; // each launch runs the body 4 times diff --git a/cudax/include/cuda/experimental/__stf/stream/stream_ctx.cuh b/cudax/include/cuda/experimental/__stf/stream/stream_ctx.cuh index b209dbf27e50..a517cc3c67ea 100644 --- a/cudax/include/cuda/experimental/__stf/stream/stream_ctx.cuh +++ b/cudax/include/cuda/experimental/__stf/stream/stream_ctx.cuh @@ -60,9 +60,9 @@ struct streamed_interface_of; class uncached_stream_allocator : public block_allocator_interface { public: - uncached_stream_allocator() = default; + _CCCL_HIDE_FROM_ABI uncached_stream_allocator() = default; - void* + _CCCL_HOST_API void* allocate(backend_ctx_untyped& ctx, const data_place& memory_node, ::std::ptrdiff_t& s, event_list& prereqs) override { auto dstream = memory_node.getDataStream(ctx.async_resources().get_place_resources()); @@ -84,7 +84,7 @@ public: return result; } - void deallocate( + _CCCL_HOST_API void deallocate( backend_ctx_untyped& ctx, const data_place& memory_node, event_list& prereqs, void* ptr, size_t sz) override { auto dstream = memory_node.getDataStream(ctx.async_resources().get_place_resources()); @@ -108,12 +108,12 @@ public: } // Nothing is done as all deallocation are done immediately - event_list deinit(backend_ctx_untyped& /* ctx */) override + _CCCL_HOST_API event_list deinit(backend_ctx_untyped& /* ctx */) override { return event_list(); } - ::std::string to_string() const override + _CCCL_HOST_API ::std::string to_string() const override { return "uncached stream allocator"; } @@ -155,10 +155,10 @@ public: /// @brief This type is copyable, assignable, and movable. However, copies have reference semantics. ///@{ - stream_ctx(async_resources_handle handle = async_resources_handle(nullptr)) + _CCCL_HOST_API stream_ctx(async_resources_handle handle = async_resources_handle(nullptr)) : backend_ctx(::std::make_shared(mv(handle))) {} - stream_ctx(cudaStream_t user_stream, async_resources_handle handle = async_resources_handle(nullptr)) + _CCCL_HOST_API stream_ctx(cudaStream_t user_stream, async_resources_handle handle = async_resources_handle(nullptr)) : backend_ctx(::std::make_shared(mv(handle), !is_stream_capturing(user_stream))) { // If ``user_stream`` is currently capturing a CUDA graph, avoid issuing @@ -197,7 +197,7 @@ public: ///@} - void set_user_stream(cudaStream_t user_stream) + _CCCL_HOST_API void set_user_stream(cudaStream_t user_stream) { // TODO first introduce the user stream in our pool auto dstream = augmented_stream(user_stream); @@ -222,14 +222,14 @@ public: * @brief Creates a task on the specified execution place */ template - stream_task task(exec_place e_place, task_dep... deps) + _CCCL_HOST_API stream_task task(exec_place e_place, task_dep... deps) { EXPECT(state().deferred_tasks.empty(), "Mixing deferred and immediate tasks is not supported yet."); return stream_task(*this, mv(e_place), mv(deps)...); // NOLINT(cppcoreguidelines-slicing) } template - deferred_stream_task deferred_task(exec_place e_place, task_dep... deps) + _CCCL_HOST_API deferred_stream_task deferred_task(exec_place e_place, task_dep... deps) { auto result = deferred_stream_task(*this, mv(e_place), mv(deps)...); @@ -241,13 +241,13 @@ public: } template - deferred_stream_task deferred_task(task_dep... deps) + _CCCL_HOST_API deferred_stream_task deferred_task(task_dep... deps) { return deferred_task(exec_place::current_device(), mv(deps)...); } template - auto deferred_host_launch(task_dep... deps) + _CCCL_HOST_API auto deferred_host_launch(task_dep... deps) { auto result = deferred_host_launch_scope(this, mv(deps)...); int id = result.get_mapping_id(); @@ -258,7 +258,7 @@ public: return result; } - cudaStream_t fence() + _CCCL_HOST_API cudaStream_t fence() { const auto& user_dstream = state().user_dstream; // We either use the user-provided stream, or we get one stream from the pool @@ -287,34 +287,34 @@ public: { struct payload_t : public deferred_stream_task<>::payload_t { - payload_t(stream_ctx& ctx, exec_place e_place, shape_t shape, task_dep... deps) + _CCCL_HOST_API payload_t(stream_ctx& ctx, exec_place e_place, shape_t shape, task_dep... deps) : task(ctx, mv(e_place), mv(shape), mv(deps)...) {} reserved::parallel_for_scope task; ::std::function&)> todo; - void set_symbol(::std::string s) override + _CCCL_HOST_API void set_symbol(::std::string s) override { task.set_symbol(mv(s)); } - const ::std::string& get_symbol() const override + _CCCL_HOST_API const ::std::string& get_symbol() const override { return task.get_symbol(); } - int get_mapping_id() const override + _CCCL_HOST_API int get_mapping_id() const override { return task.get_mapping_id(); } - void run() override + _CCCL_HOST_API void run() override { todo(task); } - void populate_deps_scheduling_info() override + _CCCL_HOST_API void populate_deps_scheduling_info() override { const auto& deps = get_task_deps(); reserved::ensure_task_deps_initialized(deps); @@ -325,24 +325,25 @@ public: } } - const task_dep_vector_untyped& get_task_deps() const override + _CCCL_HOST_API const task_dep_vector_untyped& get_task_deps() const override { return task.get_task_deps(); } - void set_exec_place(exec_place e_place) override + _CCCL_HOST_API void set_exec_place(exec_place e_place) override { task.set_exec_place(mv(e_place)); } }; - payload_t& my_payload() const + _CCCL_HOST_API payload_t& my_payload() const { // Safe to do the cast because we've set the pointer earlier ourselves return *static_cast(payload.get()); } public: + _CCCL_HOST_API deferred_parallel_for_scope(stream_ctx& ctx, exec_place e_place, shape_t shape, task_dep... deps) : deferred_stream_task<>(::std::make_shared(ctx, mv(e_place), mv(shape), mv(deps)...)) {} @@ -354,26 +355,26 @@ public: * @param s * @return deferred_parallel_for_scope& */ - deferred_parallel_for_scope& set_symbol(::std::string s) & + _CCCL_HOST_API deferred_parallel_for_scope& set_symbol(::std::string s) & { payload->set_symbol(mv(s)); return *this; } - deferred_parallel_for_scope&& set_symbol(::std::string s) && + _CCCL_HOST_API deferred_parallel_for_scope&& set_symbol(::std::string s) && { set_symbol(mv(s)); return mv(*this); } ///@} - void populate_deps_scheduling_info() + _CCCL_HOST_API void populate_deps_scheduling_info() { payload->populate_deps_scheduling_info(); } template - void operator->*(Fun fun) + _CCCL_HOST_API void operator->*(Fun fun) { my_payload().todo = [f = mv(fun)](reserved::parallel_for_scope& task) { task->*f; @@ -386,34 +387,34 @@ public: { struct payload_t : public deferred_stream_task<>::payload_t { - payload_t(stream_ctx& ctx, task_dep... deps) + _CCCL_HOST_API payload_t(stream_ctx& ctx, task_dep... deps) : task(ctx, mv(deps)...) {} reserved::host_launch_scope task; ::std::function&)> todo; - void set_symbol(::std::string s) override + _CCCL_HOST_API void set_symbol(::std::string s) override { task.set_symbol(s); } - const ::std::string& get_symbol() const override + _CCCL_HOST_API const ::std::string& get_symbol() const override { return task.get_symbol(); } - int get_mapping_id() const override + _CCCL_HOST_API int get_mapping_id() const override { return task.get_mapping_id(); } - void run() override + _CCCL_HOST_API void run() override { todo(task); } - void populate_deps_scheduling_info() override + _CCCL_HOST_API void populate_deps_scheduling_info() override { const auto& deps = get_task_deps(); reserved::ensure_task_deps_initialized(deps); @@ -424,22 +425,22 @@ public: } } - const task_dep_vector_untyped& get_task_deps() const override + _CCCL_HOST_API const task_dep_vector_untyped& get_task_deps() const override { return task.get_task_deps(); } - void set_exec_place(exec_place) override {} + _CCCL_HOST_API void set_exec_place(exec_place) override {} }; - payload_t& my_payload() const + _CCCL_HOST_API payload_t& my_payload() const { // Safe to do the cast because we've set the pointer earlier ourselves return *static_cast(payload.get()); } public: - deferred_host_launch_scope(stream_ctx& ctx, task_dep... deps) + _CCCL_HOST_API deferred_host_launch_scope(stream_ctx& ctx, task_dep... deps) : deferred_stream_task<>(::std::make_shared(ctx, mv(deps)...)) {} @@ -450,26 +451,26 @@ public: * @param s * @return deferred_host_launch_scope& */ - deferred_host_launch_scope& set_symbol(::std::string s) & + _CCCL_HOST_API deferred_host_launch_scope& set_symbol(::std::string s) & { payload->set_symbol(mv(s)); return *this; } - deferred_host_launch_scope&& set_symbol(::std::string s) && + _CCCL_HOST_API deferred_host_launch_scope&& set_symbol(::std::string s) && { set_symbol(mv(s)); return mv(*this); } ///@} - void populate_deps_scheduling_info() + _CCCL_HOST_API void populate_deps_scheduling_info() { payload->populate_deps_scheduling_info(); } template - void operator->*(Fun fun) + _CCCL_HOST_API void operator->*(Fun fun) { my_payload().todo = [f = mv(fun)](reserved::host_launch_scope& task) { task->*f; @@ -477,7 +478,7 @@ public: } }; - void finalize() + _CCCL_HOST_API void finalize() { _CCCL_ASSERT(get_phase() < backend_ctx_untyped::phase::finalized, ""); auto& state = this->state(); @@ -512,13 +513,13 @@ public: } } - float get_submission_time_ms() const + _CCCL_HOST_API float get_submission_time_ms() const { assert(state().submitted_stream); return state().submission_time; } - void submit() + _CCCL_HOST_API void submit() { auto& state = this->state(); _CCCL_ASSERT(!state.submitted_stream, ""); @@ -612,10 +613,10 @@ public: } // no-op : so that we can use the same code with stream_ctx and graph_ctx - void change_stage() {} + _CCCL_HOST_API void change_stage() {} template - auto deferred_parallel_for(exec_place e_place, S shape, task_dep... deps) + _CCCL_HOST_API auto deferred_parallel_for(exec_place e_place, S shape, task_dep... deps) { auto result = deferred_parallel_for_scope(*this, mv(e_place), mv(shape), mv(deps)...); int id = result.get_mapping_id(); @@ -626,13 +627,13 @@ public: } template - auto deferred_parallel_for(S shape, task_dep... deps) + _CCCL_HOST_API auto deferred_parallel_for(S shape, task_dep... deps) { return deferred_parallel_for(exec_place::current_device(), mv(shape), mv(deps)...); } template - auto wait(cuda::experimental::stf::logical_data& ldata) + _CCCL_HOST_API auto wait(cuda::experimental::stf::logical_data& ldata) { if constexpr (::cuda::std::is_same_v) { @@ -663,13 +664,14 @@ private: class impl : public base_impl { public: - impl(async_resources_handle _async_resources = async_resources_handle(nullptr), bool initialize_cuda_runtime = true) + _CCCL_HOST_API impl(async_resources_handle _async_resources = async_resources_handle(nullptr), + bool initialize_cuda_runtime = true) : base_impl(mv(_async_resources), initialize_cuda_runtime) { reserved::backend_ctx_setup_allocators(*this); } - void cleanup() + _CCCL_HOST_API void cleanup() { // Reset this object deferred_tasks.clear(); @@ -679,25 +681,25 @@ private: } // Due to circular dependencies, we need to define it here, and not in backend_ctx_untyped - void update_uncached_allocator(block_allocator_untyped custom) override + _CCCL_HOST_API void update_uncached_allocator(block_allocator_untyped custom) override { reserved::backend_ctx_update_uncached_allocator(*this, mv(custom)); } - event_list stream_to_event_list(cudaStream_t stream, ::std::string symbol) const override + _CCCL_HOST_API event_list stream_to_event_list(cudaStream_t stream, ::std::string symbol) const override { auto e = reserved::record_event_in_stream(augmented_stream(stream), *get_dot(), mv(symbol)); return event_list(mv(e)); } - ::std::string to_string() const override + _CCCL_HOST_API ::std::string to_string() const override { return "stream backend context"; } // We need to ensure all dangling events have been completed (eg. by having // the CUDA stream used in the finalization wait on these events) - bool track_dangling_events() const override + _CCCL_HOST_API bool track_dangling_events() const override { return true; } @@ -716,11 +718,11 @@ private: bool blocking_finalize = true; }; - impl& state() + _CCCL_HOST_API impl& state() { return dynamic_cast(get_state()); } - const impl& state() const + _CCCL_HOST_API const impl& state() const { return dynamic_cast(get_state()); } @@ -728,7 +730,7 @@ private: /// @brief Build the task graph by populating the predecessor and successor lists. /// The logic here is copied from acquire_release.h notify_access(), although this doesn't handle redux at the /// moment - void build_task_graph() + _CCCL_HOST_API void build_task_graph() { auto& state = this->state(); @@ -831,7 +833,7 @@ UNITTEST("logical_data_untyped moveable") class scalar { public: - scalar(stream_ctx& ctx) + _CCCL_HOST scalar(stream_ctx& ctx) { const size_t s = sizeof(double); auto owner = ::cuda::std::make_unique(); @@ -849,7 +851,7 @@ UNITTEST("logical_data_untyped moveable") owner.release(); } - scalar& operator=(scalar&& rhs) noexcept + _CCCL_HOST scalar& operator=(scalar&& rhs) noexcept { handle = mv(rhs.handle); return *this; @@ -881,7 +883,7 @@ UNITTEST("logical_data_untyped moveable") # if _CCCL_CUDA_COMPILATION() namespace reserved { -static __global__ void dummy() {} +_CCCL_KERNEL_ATTRIBUTES void dummy() {} } // namespace reserved UNITTEST("copyable stream_ctx") @@ -1083,7 +1085,7 @@ UNITTEST("const class containing a stream_ctx") struct foo { foo() = default; - stream_ctx& get_ctx() const + _CCCL_HOST stream_ctx& get_ctx() const { return ctx; } @@ -1154,7 +1156,7 @@ UNITTEST("get logical_data from a task_dep") # if !defined(CUDASTF_DISABLE_CODE_GENERATION) && _CCCL_CUDA_COMPILATION() namespace reserved { -inline void unit_test_pfor() +_CCCL_HOST_API inline void unit_test_pfor() { stream_ctx ctx; // finalize() submits pending work and synchronizes, so it belongs on the normal path only: @@ -1181,7 +1183,7 @@ UNITTEST("basic parallel_for test") unit_test_pfor(); }; -inline void unit_test_pfor_integral_shape() +_CCCL_HOST_API inline void unit_test_pfor_integral_shape() { stream_ctx ctx; auto lA = ctx.logical_data(shape_of>(64)); @@ -1204,7 +1206,7 @@ UNITTEST("parallel_for with integral shape") unit_test_pfor_integral_shape(); }; -inline void unit_test_host_pfor() +_CCCL_HOST_API inline void unit_test_host_pfor() { stream_ctx ctx; auto lA = ctx.logical_data(shape_of>(64)); @@ -1225,7 +1227,7 @@ UNITTEST("basic parallel_for test on host") unit_test_host_pfor(); }; -inline void unit_test_pfor_mix_host_dev() +_CCCL_HOST_API inline void unit_test_pfor_mix_host_dev() { stream_ctx ctx; @@ -1261,7 +1263,7 @@ UNITTEST("parallel_for host and device") unit_test_pfor_mix_host_dev(); }; -inline void unit_test_untyped_place_pfor() +_CCCL_HOST_API inline void unit_test_untyped_place_pfor() { stream_ctx ctx; @@ -1289,7 +1291,7 @@ UNITTEST("basic parallel_for test on host (untyped execution place)") unit_test_untyped_place_pfor(); }; -inline void unit_test_pfor_grid() +_CCCL_HOST_API inline void unit_test_pfor_grid() { stream_ctx ctx; auto where = exec_place::all_devices(); @@ -1311,7 +1313,7 @@ UNITTEST("basic parallel_for test on grid") unit_test_pfor_grid(); }; -inline void unit_test_pfor_untyped_grid() +_CCCL_HOST_API inline void unit_test_pfor_untyped_grid() { stream_ctx ctx; @@ -1335,7 +1337,7 @@ UNITTEST("basic parallel_for test on grid") unit_test_pfor_untyped_grid(); }; -inline void unit_test_launch() +_CCCL_HOST_API inline void unit_test_launch() { stream_ctx ctx; // finalize() submits pending work and synchronizes, so it belongs on the normal path only: diff --git a/cudax/include/cuda/experimental/__stf/utility/cartesian_iterator.cuh b/cudax/include/cuda/experimental/__stf/utility/cartesian_iterator.cuh index 5ea54b3c9126..42c6bda72e85 100644 --- a/cudax/include/cuda/experimental/__stf/utility/cartesian_iterator.cuh +++ b/cudax/include/cuda/experimental/__stf/utility/cartesian_iterator.cuh @@ -138,103 +138,103 @@ public: using reference = const value_type&; using iterator_category = ::std::random_access_iterator_tag; - _CCCL_HOST_DEVICE explicit RangeIterator(int value) + _CCCL_HOST_DEVICE_API explicit RangeIterator(int value) : currentValue(value) {} - _CCCL_HOST_DEVICE value_type operator*() const + _CCCL_HOST_DEVICE_API value_type operator*() const { return currentValue; } - _CCCL_HOST_DEVICE RangeIterator& operator++() + _CCCL_HOST_DEVICE_API RangeIterator& operator++() { ++currentValue; return *this; } - _CCCL_HOST_DEVICE RangeIterator operator++(int) + _CCCL_HOST_DEVICE_API RangeIterator operator++(int) { RangeIterator temp = *this; ++(*this); return temp; } - _CCCL_HOST_DEVICE RangeIterator& operator--() + _CCCL_HOST_DEVICE_API RangeIterator& operator--() { --currentValue; return *this; } - _CCCL_HOST_DEVICE RangeIterator operator--(int) + _CCCL_HOST_DEVICE_API RangeIterator operator--(int) { RangeIterator temp = *this; --(*this); return temp; } - _CCCL_HOST_DEVICE RangeIterator& operator+=(difference_type n) + _CCCL_HOST_DEVICE_API RangeIterator& operator+=(difference_type n) { currentValue += static_cast(n); return *this; } - _CCCL_HOST_DEVICE RangeIterator operator+(difference_type n) const + _CCCL_HOST_DEVICE_API RangeIterator operator+(difference_type n) const { RangeIterator result = *this; result += n; return result; } - _CCCL_HOST_DEVICE RangeIterator& operator-=(difference_type n) + _CCCL_HOST_DEVICE_API RangeIterator& operator-=(difference_type n) { currentValue -= static_cast(n); return *this; } - _CCCL_HOST_DEVICE RangeIterator operator-(difference_type n) const + _CCCL_HOST_DEVICE_API RangeIterator operator-(difference_type n) const { RangeIterator result = *this; result -= n; return result; } - _CCCL_HOST_DEVICE difference_type operator-(const RangeIterator& other) const + _CCCL_HOST_DEVICE_API difference_type operator-(const RangeIterator& other) const { return currentValue - other.currentValue; } - _CCCL_HOST_DEVICE value_type operator[](difference_type n) const + _CCCL_HOST_DEVICE_API value_type operator[](difference_type n) const { return *(*this + n); } - _CCCL_HOST_DEVICE bool operator==(const RangeIterator& other) const + _CCCL_HOST_DEVICE_API bool operator==(const RangeIterator& other) const { return currentValue == other.currentValue; } - _CCCL_HOST_DEVICE bool operator!=(const RangeIterator& other) const + _CCCL_HOST_DEVICE_API bool operator!=(const RangeIterator& other) const { return !(*this == other); } - _CCCL_HOST_DEVICE bool operator<(const RangeIterator& other) const + _CCCL_HOST_DEVICE_API bool operator<(const RangeIterator& other) const { return currentValue < other.currentValue; } - _CCCL_HOST_DEVICE bool operator>(const RangeIterator& other) const + _CCCL_HOST_DEVICE_API bool operator>(const RangeIterator& other) const { return currentValue > other.currentValue; } - _CCCL_HOST_DEVICE bool operator<=(const RangeIterator& other) const + _CCCL_HOST_DEVICE_API bool operator<=(const RangeIterator& other) const { return currentValue <= other.currentValue; } - _CCCL_HOST_DEVICE bool operator>=(const RangeIterator& other) const + _CCCL_HOST_DEVICE_API bool operator>=(const RangeIterator& other) const { return currentValue >= other.currentValue; } @@ -250,27 +250,27 @@ public: using iterator = RangeIterator; using difference_type = iterator::difference_type; - _CCCL_HOST_DEVICE explicit Range(int n) + _CCCL_HOST_DEVICE_API explicit Range(int n) : beginIterator(0) , endIterator(n) {} - _CCCL_HOST_DEVICE iterator begin() const + _CCCL_HOST_DEVICE_API iterator begin() const { return beginIterator; } - _CCCL_HOST_DEVICE iterator end() const + _CCCL_HOST_DEVICE_API iterator end() const { return endIterator; } - _CCCL_HOST_DEVICE value_type size() const + _CCCL_HOST_DEVICE_API value_type size() const { return endIterator - beginIterator; } - _CCCL_HOST_DEVICE value_type operator[](difference_type index) const + _CCCL_HOST_DEVICE_API value_type operator[](difference_type index) const { return *(beginIterator + index); } @@ -284,7 +284,7 @@ private: namespace reserved { template -__global__ void unit_test_range_func(T n) +_CCCL_KERNEL_ATTRIBUTES void unit_test_range_func(T n) { const Range range(n); int sum = 0; @@ -329,95 +329,95 @@ public: using reference = const value_type&; using iterator_category = ::std::random_access_iterator_tag; - _CCCL_HOST_DEVICE explicit StridedRangeIterator(value_type value, difference_type stride) + _CCCL_HOST_DEVICE_API explicit StridedRangeIterator(value_type value, difference_type stride) : currentValue(value) , stride(stride) {} - _CCCL_HOST_DEVICE value_type operator*() const + _CCCL_HOST_DEVICE_API value_type operator*() const { return currentValue; } - _CCCL_HOST_DEVICE StridedRangeIterator& operator++() + _CCCL_HOST_DEVICE_API StridedRangeIterator& operator++() { currentValue += stride; return *this; } - _CCCL_HOST_DEVICE StridedRangeIterator operator++(int) + _CCCL_HOST_DEVICE_API StridedRangeIterator operator++(int) { StridedRangeIterator temp = *this; ++(*this); return temp; } - _CCCL_HOST_DEVICE StridedRangeIterator& operator--() + _CCCL_HOST_DEVICE_API StridedRangeIterator& operator--() { currentValue -= stride; return *this; } - _CCCL_HOST_DEVICE StridedRangeIterator operator--(int) + _CCCL_HOST_DEVICE_API StridedRangeIterator operator--(int) { StridedRangeIterator temp = *this; --(*this); return temp; } - _CCCL_HOST_DEVICE StridedRangeIterator& operator+=(difference_type n) + _CCCL_HOST_DEVICE_API StridedRangeIterator& operator+=(difference_type n) { currentValue += stride * n; return *this; } - _CCCL_HOST_DEVICE StridedRangeIterator& operator-=(difference_type n) + _CCCL_HOST_DEVICE_API StridedRangeIterator& operator-=(difference_type n) { currentValue -= stride * n; return *this; } - _CCCL_HOST_DEVICE StridedRangeIterator operator+(difference_type n) const + _CCCL_HOST_DEVICE_API StridedRangeIterator operator+(difference_type n) const { return StridedRangeIterator(currentValue + (stride * n), stride); } - _CCCL_HOST_DEVICE StridedRangeIterator operator-(difference_type n) const + _CCCL_HOST_DEVICE_API StridedRangeIterator operator-(difference_type n) const { return StridedRangeIterator(currentValue - (stride * n), stride); } - _CCCL_HOST_DEVICE difference_type operator-(const StridedRangeIterator& other) const + _CCCL_HOST_DEVICE_API difference_type operator-(const StridedRangeIterator& other) const { return static_cast((currentValue - other.currentValue) / stride); } - _CCCL_HOST_DEVICE bool operator==(const StridedRangeIterator& other) const + _CCCL_HOST_DEVICE_API bool operator==(const StridedRangeIterator& other) const { return currentValue == other.currentValue; } - _CCCL_HOST_DEVICE bool operator!=(const StridedRangeIterator& other) const + _CCCL_HOST_DEVICE_API bool operator!=(const StridedRangeIterator& other) const { return currentValue != other.currentValue; } - _CCCL_HOST_DEVICE bool operator<(const StridedRangeIterator& other) const + _CCCL_HOST_DEVICE_API bool operator<(const StridedRangeIterator& other) const { return currentValue < other.currentValue; } - _CCCL_HOST_DEVICE bool operator>(const StridedRangeIterator& other) const + _CCCL_HOST_DEVICE_API bool operator>(const StridedRangeIterator& other) const { return currentValue > other.currentValue; } - _CCCL_HOST_DEVICE bool operator<=(const StridedRangeIterator& other) const + _CCCL_HOST_DEVICE_API bool operator<=(const StridedRangeIterator& other) const { return currentValue <= other.currentValue; } - _CCCL_HOST_DEVICE bool operator>=(const StridedRangeIterator& other) const + _CCCL_HOST_DEVICE_API bool operator>=(const StridedRangeIterator& other) const { return currentValue >= other.currentValue; } @@ -434,7 +434,7 @@ public: using iterator = StridedRangeIterator; using difference_type = iterator::difference_type; - _CCCL_HOST_DEVICE explicit StridedRange(value_type start, value_type end, difference_type stride) + _CCCL_HOST_DEVICE_API explicit StridedRange(value_type start, value_type end, difference_type stride) : start_(start) , end_(end) , stride_(stride) @@ -450,12 +450,12 @@ public: } } - _CCCL_HOST_DEVICE iterator begin() const + _CCCL_HOST_DEVICE_API iterator begin() const { return iterator(start_, stride_); } - _CCCL_HOST_DEVICE iterator end() const + _CCCL_HOST_DEVICE_API iterator end() const { return iterator(end_, stride_); } @@ -507,7 +507,7 @@ UNITTEST("StridedRange loop") }; template -__global__ void unit_test_strided_range_func(T n) +_CCCL_KERNEL_ATTRIBUTES void unit_test_strided_range_func(T n) { const int tid = static_cast((blockIdx.x * blockDim.x) + threadIdx.x); const int nthreads = static_cast(gridDim.x * blockDim.x); diff --git a/cudax/include/cuda/experimental/__stf/utility/cuda_safe_call.cuh b/cudax/include/cuda/experimental/__stf/utility/cuda_safe_call.cuh index 7099be73be7a..cd0c650c226b 100644 --- a/cudax/include/cuda/experimental/__stf/utility/cuda_safe_call.cuh +++ b/cudax/include/cuda/experimental/__stf/utility/cuda_safe_call.cuh @@ -125,7 +125,7 @@ struct cuda_status_traits_base { //! @brief The success value, the zero enumerator. //! @return `T{}` - static constexpr T success() noexcept + _CCCL_HOST_API static constexpr T success() noexcept { return T{}; } @@ -148,7 +148,7 @@ struct status_traits : cuda_status_traits_base //! Builds no exception, so it does not query the current device. //! @param[in] status the status //! @return a string such as "cudaErrorInvalidValue: invalid argument" - static ::std::string name(const cudaError_t status) + _CCCL_HOST_API static ::std::string name(const cudaError_t status) { const char* const error_name = ::cudaGetErrorName(status); const char* const error_string = ::cudaGetErrorString(status); @@ -170,7 +170,7 @@ struct status_traits : cuda_status_traits_base //! with fixed text for a status the driver does not recognize. //! @param[in] status the status //! @return a string such as "CUDA_ERROR_INVALID_VALUE: invalid argument" - static ::std::string name(const CUresult status) + _CCCL_HOST_API static ::std::string name(const CUresult status) { const char* error_name = nullptr; if (::cuGetErrorName(status, &error_name) != CUDA_SUCCESS || error_name == nullptr) @@ -199,7 +199,7 @@ struct status_traits : cuda_status_traits_base //! @brief The status's enumerator name. //! @param[in] status the status //! @return a string such as "CUDA_OCC_ERROR_INVALID_INPUT" - static ::std::string name(const cudaOccError status) + _CCCL_HOST_API static ::std::string name(const cudaOccError status) { switch (status) { @@ -280,6 +280,7 @@ public: * @param loc location of the call, defaulted */ template + _CCCL_HOST_API cuda_exception(const T status, const ::cuda::std::source_location loc = ::cuda::std::source_location::current()) : stored_status(status) { @@ -304,7 +305,7 @@ public: * * @return the error message */ - const char* what() const noexcept override + _CCCL_HOST_API const char* what() const noexcept override { return msg.c_str(); } @@ -319,7 +320,7 @@ public: * @return the status */ template - S status() const noexcept + _CCCL_HOST_API S status() const noexcept { const S* const result = ::cuda::std::get_if(&stored_status); _CCCL_VERIFY(result != nullptr, @@ -329,7 +330,7 @@ public: private: template - void format(const char* fmt, Ps&&... ps) + _CCCL_HOST_API void format(const char* fmt, Ps&&... ps) { // Compute the bytes to be written. auto needed = ::std::snprintf(nullptr, 0, fmt, ps...); @@ -358,7 +359,8 @@ private: }; template -cuda_exception cuda_status_traits_base::to_exception(const T status, const ::cuda::std::source_location loc) +_CCCL_HOST_API cuda_exception +cuda_status_traits_base::to_exception(const T status, const ::cuda::std::source_location loc) { return cuda_exception(status, loc); } @@ -534,7 +536,8 @@ UNITTEST("first_last_param") * @snippet this cuda_safe_call */ template -void cuda_safe_call(const T status, const ::cuda::std::source_location loc = ::cuda::std::source_location::current()) +_CCCL_HOST_API void +cuda_safe_call(const T status, const ::cuda::std::source_location loc = ::cuda::std::source_location::current()) { // Common early exit test for all cases if (status == 0) @@ -578,7 +581,8 @@ UNITTEST("cuda_safe_call") * @snippet this cuda_try1 */ template -void cuda_try(Status status, const ::cuda::std::source_location loc = ::cuda::std::source_location::current()) +_CCCL_HOST_API void +cuda_try(Status status, const ::cuda::std::source_location loc = ::cuda::std::source_location::current()) { if (status) { @@ -676,7 +680,7 @@ namespace reserved // raw status to `check` (a stateless callable: throw for cuda_try, report-and-abort for // cuda_safe_call). `loc` is the user's call site, captured by the public fronts (see below). template -auto checked_api_call(Check check, const ::cuda::std::source_location loc, Ps&&... ps) +_CCCL_HOST_API auto checked_api_call(Check check, const ::cuda::std::source_location loc, Ps&&... ps) { constexpr bool direct_form = ::cuda::std::is_invocable_v; @@ -733,7 +737,7 @@ auto checked_api_call(Check check, const ::cuda::std::source_location loc, Ps&&. // output pointer). `forward` preserves the parameter's value category, so // reference parameters pass through unchanged. Call syntax is unchanged. template -auto cuda_try(const ::cuda::std::source_location loc = ::cuda::std::source_location::current()) +_CCCL_HOST_API auto cuda_try(const ::cuda::std::source_location loc = ::cuda::std::source_location::current()) { return reserved::checked_api_call( [](auto status, auto l) { @@ -750,7 +754,7 @@ _CCCL_REQUIRES((!::cuda::std::is_void_v>) _CCCL_AND( reserved::first_param, Ps..., reserved::out_param>*>)) -auto cuda_try(with_location> p0, Ps&&... rest) +_CCCL_HOST_API auto cuda_try(with_location> p0, Ps&&... rest) { return reserved::checked_api_call( [](auto status, auto l) { @@ -772,7 +776,7 @@ _CCCL_REQUIRES( !::cuda::std::is_void_v>>) _CCCL_AND(!::cuda::std::is_const_v>>) _CCCL_AND ::cuda::std:: is_invocable_v>*, reserved::second_param, Ps...>) -auto cuda_try(with_location> p0, Ps&&... rest) +_CCCL_HOST_API auto cuda_try(with_location> p0, Ps&&... rest) { reserved::out_param> result{}; cuda_try( @@ -786,7 +790,7 @@ auto cuda_try(with_location> p0, Ps&&... rest) // of reporting an overload race. _CCCL_TEMPLATE(auto fun, typename... Ps) _CCCL_REQUIRES(reserved::ambiguous_output_forms) -void cuda_try(Ps&&...) +_CCCL_HOST_API void cuda_try(Ps&&...) { static_assert(reserved::dependent_false, "Ambiguous cuda_try: both first- and last-output forms apply; " @@ -803,7 +807,7 @@ void cuda_try(Ps&&...) * @snippet this cuda_safe_call2 */ template -auto cuda_safe_call(const ::cuda::std::source_location loc = ::cuda::std::source_location::current()) +_CCCL_HOST_API auto cuda_safe_call(const ::cuda::std::source_location loc = ::cuda::std::source_location::current()) { return reserved::checked_api_call( [](auto status, auto l) { @@ -820,7 +824,7 @@ _CCCL_REQUIRES((!::cuda::std::is_void_v>) _CCCL_AND( reserved::first_param, Ps..., reserved::out_param>*>)) -auto cuda_safe_call(with_location> p0, Ps&&... rest) +_CCCL_HOST_API auto cuda_safe_call(with_location> p0, Ps&&... rest) { return reserved::checked_api_call( [](auto status, auto l) { @@ -842,7 +846,7 @@ _CCCL_REQUIRES( !::cuda::std::is_void_v>>) _CCCL_AND(!::cuda::std::is_const_v>>) _CCCL_AND ::cuda::std:: is_invocable_v>*, reserved::second_param, Ps...>) -auto cuda_safe_call(with_location> p0, Ps&&... rest) +_CCCL_HOST_API auto cuda_safe_call(with_location> p0, Ps&&... rest) { reserved::out_param> result{}; cuda_safe_call( @@ -856,7 +860,7 @@ auto cuda_safe_call(with_location> p0, Ps&&... rest) // of reporting an overload race. _CCCL_TEMPLATE(auto fun, typename... Ps) _CCCL_REQUIRES(reserved::ambiguous_output_forms) -void cuda_safe_call(Ps&&...) +_CCCL_HOST_API void cuda_safe_call(Ps&&...) { static_assert(reserved::dependent_false, "Ambiguous cuda_safe_call: both first- and last-output forms apply; " @@ -895,13 +899,13 @@ auto cuda_safe_call(Ps&&... ps) #endif // !(_CCCL_CUDA_COMPILER(NVCC) && _CCCL_CTK_BELOW(12, 1)) #ifdef UNITTESTED_FILE -inline cudaError_t test_first_output_param(int* out) +_CCCL_HOST_API inline cudaError_t test_first_output_param(int* out) { *out = 1; return cudaSuccess; } -inline cudaError_t test_last_output_param(double in, int* out) +_CCCL_HOST_API inline cudaError_t test_last_output_param(double in, int* out) { *out = static_cast(in); return cudaSuccess; @@ -918,12 +922,12 @@ UNITTEST("cuda_try2") }; # if !(_CCCL_CUDA_COMPILER(NVCC) && _CCCL_CTK_BELOW(12, 1)) -inline cudaError_t test_failing_direct(int) +_CCCL_HOST_API inline cudaError_t test_failing_direct(int) { return cudaErrorInvalidValue; } -inline cudaError_t test_lvalue_ref_param(int& x) +_CCCL_HOST_API inline cudaError_t test_lvalue_ref_param(int& x) { x = 5; return cudaSuccess; diff --git a/cudax/include/cuda/experimental/__stf/utility/dimensions.cuh b/cudax/include/cuda/experimental/__stf/utility/dimensions.cuh index ff805b54ee16..7c38cad9e3fe 100644 --- a/cudax/include/cuda/experimental/__stf/utility/dimensions.cuh +++ b/cudax/include/cuda/experimental/__stf/utility/dimensions.cuh @@ -41,11 +41,11 @@ namespace cuda::experimental::stf class pos4 { public: - constexpr pos4() = default; + _CCCL_HIDE_FROM_ABI constexpr pos4() = default; /// Create a pos4 from its coordinates template - _CCCL_HOST_DEVICE constexpr explicit pos4(IntegralX x, IntegralY y = 0, IntegralZ z = 0, IntegralT t = 0) + _CCCL_HOST_DEVICE_API constexpr explicit pos4(IntegralX x, IntegralY y = 0, IntegralZ z = 0, IntegralT t = 0) : x(static_cast(x)) , y(static_cast(y)) , z(static_cast(z)) @@ -53,7 +53,7 @@ public: {} /// Get the position along a specific axis - _CCCL_HOST_DEVICE constexpr ssize_t get(size_t axis_id) const + _CCCL_HOST_DEVICE_API constexpr ssize_t get(size_t axis_id) const { switch (axis_id) { @@ -70,13 +70,13 @@ public: } /// Get the position along a specific axis - _CCCL_HOST_DEVICE constexpr ssize_t operator()(int axis_id) const + _CCCL_HOST_DEVICE_API constexpr ssize_t operator()(int axis_id) const { return get(axis_id); } /// Comparison of two pos4 in lexicographical order - _CCCL_HOST_DEVICE constexpr bool operator<(const pos4& rhs) const + _CCCL_HOST_DEVICE_API constexpr bool operator<(const pos4& rhs) const { if (x != rhs.x) { @@ -94,13 +94,13 @@ public: } /// Equality test between two pos4 - _CCCL_HOST_DEVICE constexpr bool operator==(const pos4& rhs) const + _CCCL_HOST_DEVICE_API constexpr bool operator==(const pos4& rhs) const { return x == rhs.x && y == rhs.y && z == rhs.z && t == rhs.t; } /// Convert the pos4 to a string - ::std::string to_string() const + _CCCL_HOST_API ::std::string to_string() const { return ::std::string("pos4(" + ::std::to_string(x) + "," + ::std::to_string(y) + "," + ::std::to_string(z) + "," + ::std::to_string(t) + ")"); @@ -118,11 +118,11 @@ public: class dim4 { public: - dim4() = default; + _CCCL_HIDE_FROM_ABI dim4() = default; /// Create a dim4 from its extents template - _CCCL_HOST_DEVICE constexpr explicit dim4(IntegralX x, IntegralY y = 1, IntegralZ z = 1, IntegralT t = 1) + _CCCL_HOST_DEVICE_API constexpr explicit dim4(IntegralX x, IntegralY y = 1, IntegralZ z = 1, IntegralT t = 1) : x(static_cast(x)) , y(static_cast(y)) , z(static_cast(z)) @@ -130,7 +130,7 @@ public: {} /// Get the dimension along a specific axis - _CCCL_HOST_DEVICE constexpr size_t get(size_t axis_id) const + _CCCL_HOST_DEVICE_API constexpr size_t get(size_t axis_id) const { switch (axis_id) { @@ -147,20 +147,20 @@ public: } /// Get the dimension along a specific axis - _CCCL_HOST_DEVICE constexpr size_t operator()(int axis_id) const + _CCCL_HOST_DEVICE_API constexpr size_t operator()(int axis_id) const { return get(axis_id); } /// Get the total size (multiply all dimensions) - _CCCL_HOST_DEVICE constexpr size_t size() const + _CCCL_HOST_DEVICE_API constexpr size_t size() const { const size_t result = x * y * z * t; return result; } /// Comparison of two dim4 in lexicographical order - _CCCL_HOST_DEVICE constexpr bool operator<(const dim4& rhs) const + _CCCL_HOST_DEVICE_API constexpr bool operator<(const dim4& rhs) const { if (x != rhs.x) { @@ -178,13 +178,13 @@ public: } /// Equality test between two dim4 - _CCCL_HOST_DEVICE constexpr bool operator==(const dim4& rhs) const + _CCCL_HOST_DEVICE_API constexpr bool operator==(const dim4& rhs) const { return x == rhs.x && y == rhs.y && z == rhs.z && t == rhs.t; } /// Compute the dim4 class obtained by taking the minimum of two dim4 along each axis - _CCCL_HOST_DEVICE static constexpr dim4 min(const dim4& a, const dim4& b) + _CCCL_HOST_DEVICE_API static constexpr dim4 min(const dim4& a, const dim4& b) { return dim4( ::cuda::std::min(a.x, b.x), ::cuda::std::min(a.y, b.y), ::cuda::std::min(a.z, b.z), ::cuda::std::min(a.t, b.t)); @@ -192,7 +192,7 @@ public: /// Get the coordinate corresponding to a 1D index within a dim4 class /// (inverse of get_index: dimension 0 varies fastest) - _CCCL_HOST_DEVICE constexpr pos4 index_to_pos(size_t index) const + _CCCL_HOST_DEVICE_API constexpr pos4 index_to_pos(size_t index) const { _CCCL_ASSERT(index < size(), "invalid index"); const size_t px = index % x; @@ -205,7 +205,7 @@ public: } /// Get the 1D index of a coordinate defined by a pos4 class within a dim4 class - _CCCL_HOST_DEVICE constexpr size_t get_index(const pos4& p) const + _CCCL_HOST_DEVICE_API constexpr size_t get_index(const pos4& p) const { const size_t px = static_cast(p.get(0)); const size_t py = static_cast(p.get(1)); @@ -221,7 +221,7 @@ public: } /// Get the maximum dimension that is not 1 - _CCCL_HOST_DEVICE constexpr size_t get_rank() const + _CCCL_HOST_DEVICE_API constexpr size_t get_rank() const { if (t > 1) { @@ -240,7 +240,7 @@ public: } /// Convert the dim4 to a string - ::std::string to_string() const + _CCCL_HOST_API ::std::string to_string() const { return ::std::string("dim4(" + ::std::to_string(x) + "," + ::std::to_string(y) + "," + ::std::to_string(z) + "," + ::std::to_string(t) + ")"); @@ -264,7 +264,7 @@ public: ///@{ @name Constructors /// Construct an explicit shape from its lower and upper bounds (inclusive lower bounds, exclusive upper bounds) template - _CCCL_HOST_DEVICE box(const ::std::array<::std::pair, dimensions>& bounds) + _CCCL_HOST_DEVICE_API box(const ::std::array<::std::pair, dimensions>& bounds) { for (const size_t ind : each(0, dimensions)) { @@ -275,7 +275,7 @@ public: /// Construct an explicit shape from its upper bounds (exclusive upper bounds) template - _CCCL_HOST_DEVICE box(const ::std::array& sizes) + _CCCL_HOST_DEVICE_API box(const ::std::array& sizes) { for (size_t ind : each(0, dimensions)) { @@ -290,7 +290,7 @@ public: /// Construct an explicit shape from its upper bounds (exclusive upper bounds) template - _CCCL_HOST_DEVICE box(Int... args) + _CCCL_HOST_DEVICE_API box(Int... args) { static_assert(sizeof...(Int) == dimensions, "Number of dimensions must match"); each_in_pack( @@ -312,7 +312,7 @@ public: /// Construct an explicit shape from its lower and upper bounds (inclusive lower bounds, exclusive upper bounds) template - _CCCL_HOST_DEVICE box(::std::initializer_list... args) + _CCCL_HOST_DEVICE_API box(::std::initializer_list... args) { static_assert(sizeof...(E) == dimensions, "Number of dimensions must match"); each_in_pack( @@ -339,7 +339,7 @@ public: // } // } - _CCCL_HOST_DEVICE void print() + _CCCL_HOST_DEVICE_API void print() { printf("EXPLICIT SHAPE\n"); for (size_t ind = 0; ind < dimensions; ind++) @@ -350,25 +350,25 @@ public: } /// Get the number of elements along a dimension - _CCCL_HOST_DEVICE ::std::ptrdiff_t get_extent(size_t dim) const + _CCCL_HOST_DEVICE_API ::std::ptrdiff_t get_extent(size_t dim) const { return s[dim].second - s[dim].first; } /// Get the first coordinate (included) in a specific dimension - _CCCL_HOST_DEVICE ::std::ptrdiff_t get_begin(size_t dim) const + _CCCL_HOST_DEVICE_API ::std::ptrdiff_t get_begin(size_t dim) const { return s[dim].first; } /// Get the last coordinate (excluded) in a specific dimension - _CCCL_HOST_DEVICE ::std::ptrdiff_t get_end(size_t dim) const + _CCCL_HOST_DEVICE_API ::std::ptrdiff_t get_end(size_t dim) const { return s[dim].second; } /// Get the total number of elements in this explicit shape - _CCCL_HOST_DEVICE ::std::ptrdiff_t size() const + _CCCL_HOST_DEVICE_API ::std::ptrdiff_t size() const { if constexpr (dimensions == 1) { @@ -386,7 +386,7 @@ public: } /// get the dimensionnality of the explicit shape - _CCCL_HOST_DEVICE constexpr size_t get_rank() const + _CCCL_HOST_DEVICE_API constexpr size_t get_rank() const { return dimensions; } @@ -398,7 +398,7 @@ public: ::std::array<::std::ptrdiff_t, dimensions> current; // Array to store the current position in each dimension public: - _CCCL_HOST_DEVICE iterator(const box& b, bool at_end = false) + _CCCL_HOST_DEVICE_API iterator(const box& b, bool at_end = false) : iterated(b) { if (at_end) @@ -418,7 +418,7 @@ public: } // Overload the dereference operator to get the current position - _CCCL_HOST_DEVICE auto& operator*() + _CCCL_HOST_DEVICE_API auto& operator*() { if constexpr (dimensions == 1UL) { @@ -431,7 +431,7 @@ public: } // Overload the pre-increment operator to move to the next position - _CCCL_HOST_DEVICE iterator& operator++() + _CCCL_HOST_DEVICE_API iterator& operator++() { if constexpr (dimensions == 1UL) { @@ -458,7 +458,7 @@ public: } // Overload the equality operator to check if two iterators are equal - _CCCL_HOST_DEVICE bool operator==(const iterator& rhs) const + _CCCL_HOST_DEVICE_API bool operator==(const iterator& rhs) const { /*printf("EQUALITY TEST index %d %d shape equal ? %s\n", index, other.index, (&shape == &other.shape)?"yes":"no"); */ _CCCL_ASSERT(iterated == rhs.iterated, "Cannot compare iterators in different boxes."); @@ -474,14 +474,14 @@ public: } // Overload the inequality operator to check if two iterators are not equal - _CCCL_HOST_DEVICE bool operator!=(const iterator& other) const + _CCCL_HOST_DEVICE_API bool operator!=(const iterator& other) const { return !(*this == other); } }; // Functions to create the begin and end iterators - _CCCL_HOST_DEVICE iterator begin() const + _CCCL_HOST_DEVICE_API iterator begin() const { for (size_t i = 0; i < dimensions; ++i) { @@ -493,13 +493,13 @@ public: return iterator(*this); } - _CCCL_HOST_DEVICE iterator end() const + _CCCL_HOST_DEVICE_API iterator end() const { return iterator(*this, true); } // Overload the equality operator to check if two shapes are equal - _CCCL_HOST_DEVICE bool operator==(const box& rhs) const + _CCCL_HOST_DEVICE_API bool operator==(const box& rhs) const { // NOLINTNEXTLINE(readability-use-anyofallof) for (const size_t i : each(0, dimensions)) @@ -512,7 +512,7 @@ public: return true; } - _CCCL_HOST_DEVICE bool operator!=(const box& rhs) const + _CCCL_HOST_DEVICE_API bool operator!=(const box& rhs) const { return !(*this == rhs); } @@ -520,7 +520,7 @@ public: using coords_t = ::cuda::std::array; // This transforms a tuple of (shape, 1D index) into a coordinate - _CCCL_HOST_DEVICE coords_t index_to_coords(size_t index) const + _CCCL_HOST_DEVICE_API coords_t index_to_coords(size_t index) const { coords_t coords{}; for (size_t i = 0; i < dimensions; ++i) @@ -770,7 +770,7 @@ UNITTEST("pos4 comparison operators") template <> struct hash { - ::std::size_t operator()(pos4 const& s) const noexcept + _CCCL_HOST_API ::std::size_t operator()(pos4 const& s) const noexcept { return hash_all(s.x, s.y, s.z, s.t); } @@ -780,7 +780,7 @@ struct hash template <> struct hash { - ::std::size_t operator()(const dim4& s) const noexcept + _CCCL_HOST_API ::std::size_t operator()(const dim4& s) const noexcept { return hash_all(s.x, s.y, s.z, s.t); } diff --git a/cudax/include/cuda/experimental/__stf/utility/exception_policy.cuh b/cudax/include/cuda/experimental/__stf/utility/exception_policy.cuh index bcf2ebba8086..2b169b23a7e0 100644 --- a/cudax/include/cuda/experimental/__stf/utility/exception_policy.cuh +++ b/cudax/include/cuda/experimental/__stf/utility/exception_policy.cuh @@ -111,12 +111,12 @@ struct nullval final // prvalue is the result of a call that never returns, so control provably cannot arrive here // short of undefined behavior already committed elsewhere. template - [[noreturn]] _CCCL_HOST_DEVICE operator _Tp&&() const noexcept + [[noreturn]] _CCCL_HOST_DEVICE_API operator _Tp&&() const noexcept { _CCCL_UNREACHABLE(); } template - [[noreturn]] _CCCL_HOST_DEVICE operator _Tp&() const noexcept + [[noreturn]] _CCCL_HOST_DEVICE_API operator _Tp&() const noexcept { _CCCL_UNREACHABLE(); } @@ -175,7 +175,7 @@ using __status_failed_of _CCCL_NODEBUG = decltype(status_traits<_Status>::failed // Whether a status reports a failure: the trait's own `failed` when it has one, else a comparison // with its success value. template -[[nodiscard]] bool __status_failed(const _Status __status) noexcept +[[nodiscard]] _CCCL_HOST_API bool __status_failed(const _Status __status) noexcept { if constexpr (::cuda::std::_IsValidExpansion<__status_failed_of, _Status>::value) { @@ -190,7 +190,7 @@ template // Re-run a status-returning action up to `__n` times (forever when negative) while its result // fails: the first success is handled, exhaustion passes the latest status through. template -__status_answer<_Status> __retry_status(_Status __latest, _Fn& __fn, const int __n) +_CCCL_HOST_API __status_answer<_Status> __retry_status(_Status __latest, _Fn& __fn, const int __n) { static_assert(::cuda::std::is_same_v<::cuda::std::remove_cvref_t, _Status>, "re-running on the code channel needs an action that returns the same status type"); @@ -245,15 +245,15 @@ inline constexpr bool __selects_returned_v = false; // Composition of the properties above: a function rather than `|`/`||`, whose operands coincide // when both sides of a composite are the same policy type. -constexpr unsigned __either(const unsigned __a, const unsigned __b) noexcept +_CCCL_HOST_API constexpr unsigned __either(const unsigned __a, const unsigned __b) noexcept { return __a | __b; } -constexpr bool __either(const bool __a, const bool __b) noexcept +_CCCL_HOST_API constexpr bool __either(const bool __a, const bool __b) noexcept { return __a || __b; } -constexpr unsigned __both(const unsigned __a, const unsigned __b) noexcept +_CCCL_HOST_API constexpr unsigned __both(const unsigned __a, const unsigned __b) noexcept { return __a & __b; } @@ -282,7 +282,7 @@ template struct __sink_at { template , _Self>, int> = 0> - // NOLINTNEXTLINE(bugprone-forwarding-reference-overload) + _CCCL_HOST_API // NOLINTNEXTLINE(bugprone-forwarding-reference-overload) __sink_at(_U&& __u, const ::cuda::std::source_location __loc = ::cuda::std::source_location::current()) noexcept( ::cuda::std::is_nothrow_constructible_v<__sink_storage_t<_Self>, _U&&>) : __sink_(::cuda::std::forward<_U>(__u)) @@ -313,14 +313,14 @@ struct __sink_at_for # define _CUDAX_STF_SINK(_Self) \ using __exception_sink_tag _CCCL_NODEBUG = void; \ template \ - friend auto operator<<( \ + _CCCL_HOST_API friend auto operator<<( \ typename ::cuda::experimental::stf::sump::detail::__sink_at_for<_Self, _X>::type __s, \ _X&& __x) noexcept(noexcept(__sink_shift(::cuda::std::move(__s), ::cuda::std::forward<_X>(__x)))) \ { \ return __sink_shift(::cuda::std::move(__s), ::cuda::std::forward<_X>(__x)); \ } \ template \ - friend decltype(auto) operator->*( \ + _CCCL_HOST_API friend decltype(auto) operator->*( \ typename ::cuda::experimental::stf::sump::detail::__sink_at_for<_Self, _X>::type __s, \ _X&& __x) noexcept(noexcept(__sink_yield(::cuda::std::move(__s), ::cuda::std::forward<_X>(__x)))) \ { \ @@ -360,7 +360,7 @@ struct notify_t ::std::ostream* __os_ = nullptr; //! @brief Returns a copy configured to report on `__file` instead of `stderr`. - notify_t operator()(::FILE* __file) const + _CCCL_HOST_API notify_t operator()(::FILE* __file) const { notify_t __copy; __copy.__file_ = __file; @@ -368,7 +368,7 @@ struct notify_t } //! @brief Returns a copy configured to report on `__os` instead of `stderr`. - notify_t operator()(::std::ostream& __os) const + _CCCL_HOST_API notify_t operator()(::std::ostream& __os) const { notify_t __copy; __copy.__os_ = &__os; @@ -379,7 +379,8 @@ struct notify_t //! else `__file_` (default `stderr`). The ostream write is best-effort: a stream configured //! to throw does not get to end the program from inside a handler. template - void on_thrown(const ::std::exception* __exception, const ::cuda::std::source_location __loc, _Fn&) const noexcept + _CCCL_HOST_API void + on_thrown(const ::std::exception* __exception, const ::cuda::std::source_location __loc, _Fn&) const noexcept { __report(__exception ? __exception->what() : "nonstandard exception", __loc); } @@ -388,7 +389,7 @@ struct notify_t //! handled and left unchanged. //! A name that cannot be formatted (its allocation failed) is reported as such. template - void on_returned(const _Status __status, const ::cuda::std::source_location __loc, _Fn&) const noexcept + _CCCL_HOST_API void on_returned(const _Status __status, const ::cuda::std::source_location __loc, _Fn&) const noexcept { _CCCL_TRY { @@ -402,7 +403,7 @@ struct notify_t } private: - void __report(const char* __what, const ::cuda::std::source_location __loc) const noexcept + _CCCL_HOST_API void __report(const char* __what, const ::cuda::std::source_location __loc) const noexcept { if (__os_) { @@ -451,7 +452,7 @@ struct effect_t // maybe_unused: when f is nullary, only the discarded constexpr branch reads the failure; // gcc 9 reports it as set-but-unused. template - void + _CCCL_HOST_API void on_thrown([[maybe_unused]] const ::std::exception* __exception, const ::cuda::std::source_location, _Fn&) noexcept( ::cuda::std::is_nothrow_invocable_v<_F&, const ::std::exception*> || ::cuda::std::is_nothrow_invocable_v<_F&>) { @@ -467,8 +468,10 @@ struct effect_t } template - void on_returned([[maybe_unused]] const _Status __status, const ::cuda::std::source_location, _Fn&) noexcept( - ::cuda::std::is_nothrow_invocable_v<_F&, _Status> || ::cuda::std::is_nothrow_invocable_v<_F&>) + _CCCL_HOST_API void on_returned( + [[maybe_unused]] const _Status __status, + const ::cuda::std::source_location, + _Fn&) noexcept(::cuda::std::is_nothrow_invocable_v<_F&, _Status> || ::cuda::std::is_nothrow_invocable_v<_F&>) { if constexpr (::cuda::std::is_invocable_v<_F&, _Status>) { @@ -484,7 +487,7 @@ struct effect_t //! @brief Build an effect-only sink from a callable; see @ref effect_t. template -effect_t<::cuda::std::decay_t<_F>> effect(_F&& __f) +_CCCL_HOST_API effect_t<::cuda::std::decay_t<_F>> effect(_F&& __f) { return {::cuda::std::forward<_F>(__f)}; } @@ -512,7 +515,7 @@ struct abort_t //! @brief The exception hook: report, then die. template - [[noreturn]] nullval + [[noreturn]] _CCCL_HOST_API nullval on_thrown(const ::std::exception* __exception, const ::cuda::std::source_location __loc, _Fn& __fn) const noexcept { notify.on_thrown(__exception, __loc, __fn); @@ -521,7 +524,7 @@ struct abort_t //! @brief The code-channel hook: report the status's name, then die. template - [[noreturn]] nullval + [[noreturn]] _CCCL_HOST_API nullval on_returned(const _Status __status, const ::cuda::std::source_location __loc, _Fn& __fn) const noexcept { notify.on_returned(__status, __loc, __fn); @@ -538,7 +541,7 @@ struct terminate_t //! @endcond template - [[noreturn]] nullval + [[noreturn]] _CCCL_HOST_API nullval on_thrown(const ::std::exception* __exception, const ::cuda::std::source_location __loc, _Fn& __fn) const noexcept { notify.on_thrown(__exception, __loc, __fn); @@ -547,7 +550,7 @@ struct terminate_t //! @brief The code-channel hook: report the status's name, then terminate. template - [[noreturn]] nullval + [[noreturn]] _CCCL_HOST_API nullval on_returned(const _Status __status, const ::cuda::std::source_location __loc, _Fn& __fn) const noexcept { notify.on_returned(__status, __loc, __fn); @@ -573,13 +576,13 @@ struct noop_t //! @brief The exception hook: do nothing, decide nothing (Nothing). As the whole policy the //! expression takes its default value. template - void on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn&) const noexcept + _CCCL_HOST_API void on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn&) const noexcept {} //! @brief The code-channel hook: "keep". The status is handled and yielded unchanged, so a `|` //! stops here: `if_equal(cudaErrorNotReady)(noop) | unwind` yields the not-ready code. template - detail::__status_answer<_Status> + _CCCL_HOST_API detail::__status_answer<_Status> on_returned(const _Status __status, const ::cuda::std::source_location, _Fn&) const noexcept { return {__status, false}; @@ -610,7 +613,8 @@ struct defer_t //! @brief Captures the in-flight exception; the answer converts to the expression's type. template - ::std::exception_ptr on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn&) const noexcept + _CCCL_HOST_API ::std::exception_ptr + on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn&) const noexcept { return ::std::current_exception(); } @@ -634,14 +638,14 @@ struct fwd_t //! @endcond template - [[noreturn]] nullval on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn&) const + [[noreturn]] _CCCL_HOST_API nullval on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn&) const { _CCCL_RETHROW; } //! @brief The code-channel hook: answers `fwd_t`, so the status is yielded unchanged. template - fwd_t on_returned(const _Status, const ::cuda::std::source_location, _Fn&) const noexcept + _CCCL_HOST_API fwd_t on_returned(const _Status, const ::cuda::std::source_location, _Fn&) const noexcept { return {}; } @@ -665,14 +669,15 @@ struct unwind_t //! @endcond template - [[noreturn]] nullval on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn&) const + [[noreturn]] _CCCL_HOST_API nullval on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn&) const { _CCCL_RETHROW; } //! @brief The code-channel hook: throws the exception the status's traits build. template - [[noreturn]] nullval on_returned(const _Status __status, const ::cuda::std::source_location __loc, _Fn&) const + [[noreturn]] _CCCL_HOST_API nullval + on_returned(const _Status __status, const ::cuda::std::source_location __loc, _Fn&) const { using _Exception = decltype(status_traits<_Status>::to_exception(__status, __loc)); _CCCL_THROW(_Exception, status_traits<_Status>::to_exception(__status, __loc)); @@ -716,9 +721,10 @@ struct subst_t _V __v_; template - decltype(auto) on_thrown([[maybe_unused]] const ::std::exception* __exception, - [[maybe_unused]] const ::cuda::std::source_location __loc, - [[maybe_unused]] _Fn& __fn) noexcept + _CCCL_HOST_API decltype(auto) + on_thrown([[maybe_unused]] const ::std::exception* __exception, + [[maybe_unused]] const ::cuda::std::source_location __loc, + [[maybe_unused]] _Fn& __fn) noexcept { if constexpr (::cuda::std::is_invocable_v<_V&, const ::std::exception*, ::cuda::std::source_location, _Fn&>) { @@ -743,7 +749,7 @@ struct subst_t ::cuda::std::enable_if_t< !::cuda::std::is_invocable_v<_Self&, const ::std::exception*, ::cuda::std::source_location, _Fn&>, int> = 0> - decltype(auto) on_returned(const _Status, const ::cuda::std::source_location, _Fn&) noexcept + _CCCL_HOST_API decltype(auto) on_returned(const _Status, const ::cuda::std::source_location, _Fn&) noexcept { if constexpr (::cuda::std::is_invocable_v<_V&>) { @@ -758,7 +764,7 @@ struct subst_t //! @brief Creates a value-substitution policy; see @ref subst_t. template -auto subst(_V&& __v) +_CCCL_HOST_API auto subst(_V&& __v) { return subst_t<_V>{::cuda::std::forward<_V>(__v)}; } @@ -784,7 +790,7 @@ struct __retry_n_t // The exception channel: re-run while the action throws; exhaustion passes the last // exception through. decltype(auto) for the same reason as retry_t's hook. template - decltype(auto) on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn& __fn) const + _CCCL_HOST_API decltype(auto) on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn& __fn) const { if (__n_ == 0) { @@ -825,7 +831,8 @@ struct __retry_n_t // The code channel: re-run while the action's result fails; exhaustion passes the latest // status through. An exception from a re-run escapes. template - __status_answer<_Status> on_returned(const _Status __status, const ::cuda::std::source_location, _Fn& __fn) const + _CCCL_HOST_API __status_answer<_Status> + on_returned(const _Status __status, const ::cuda::std::source_location, _Fn& __fn) const { return __retry_status(__status, __fn, __n_); } @@ -859,7 +866,7 @@ struct retry_t // decltype(auto), not auto: a reference-returning callable must re-run to the same object, // not to a copy (the ignore branch deduces the object type and returns a copy of the tag). template - decltype(auto) on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn& __fn) + _CCCL_HOST_API decltype(auto) on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn& __fn) { if constexpr (::cuda::std::is_void_v) { @@ -874,7 +881,7 @@ struct retry_t //! @brief The code-channel hook: one re-run of a status-returning action. template - detail::__status_answer<_Status> + _CCCL_HOST_API detail::__status_answer<_Status> on_returned(const _Status __status, const ::cuda::std::source_location, _Fn& __fn) const { return detail::__retry_status(__status, __fn, 1); @@ -883,7 +890,7 @@ struct retry_t //! @brief Up to `__n` re-attempts in one policy; exhaustion passes the last failure through. //! @param[in] __n The number of re-attempts, non-negative. //! @return The counted re-attempt policy. - detail::__retry_n_t operator()(const int __n) const + _CCCL_HOST_API detail::__retry_n_t operator()(const int __n) const { _CCCL_ASSERT(__n >= 0, "retry(n) requires a non-negative count; use retry(forever) for no bound"); return detail::__retry_n_t{__n}; @@ -891,7 +898,7 @@ struct retry_t //! @brief Re-attempts until the action succeeds. //! @return The unbounded re-attempt policy. - detail::__retry_n_t operator()(forever_t) const noexcept + _CCCL_HOST_API detail::__retry_n_t operator()(forever_t) const noexcept { return detail::__retry_n_t{-1}; } @@ -938,7 +945,8 @@ struct as_expected_t _CUDAX_STF_SINK(as_expected_t) template ()())> - auto on_thrown(const ::std::exception* __exception, const ::cuda::std::source_location, _Fn&) const -> _Raw + _CCCL_HOST_API auto on_thrown(const ::std::exception* __exception, const ::cuda::std::source_location, _Fn&) const + -> _Raw { // nvcc instantiates this body when forming the callable-independent presence probe // (`void (&)()`). Keep that archetype admissible; the assert still fires at a real @@ -985,7 +993,7 @@ inline constexpr as_expected_t as_expected{}; namespace detail { template -auto __normalize(_R&& __r); +_CCCL_HOST_API auto __normalize(_R&& __r); } // namespace detail /** @@ -1010,19 +1018,19 @@ struct when _CUDAX_STF_SINK(when) _Pred __pred_; - explicit when(_Pred __pred) + _CCCL_HOST_API explicit when(_Pred __pred) : __pred_(::cuda::std::move(__pred)) {} //! @brief Applies the guard to a sink: `when(pred)(s)` is `when(pred) & s`. template - auto operator()(_P&& __p) const& + _CCCL_HOST_API auto operator()(_P&& __p) const& { return *this & detail::__normalize(::cuda::std::forward<_P>(__p)); } template - auto operator()(_P&& __p) && + _CCCL_HOST_API auto operator()(_P&& __p) && { return ::cuda::std::move(*this) & detail::__normalize(::cuda::std::forward<_P>(__p)); } @@ -1038,7 +1046,7 @@ struct when // asks the claim outside its try treats the throw as the hook's forward (see `__policy_or`). // maybe_unused: when the predicate is nullary, only the discarded constexpr // branch reads __exception; gcc 9 reports it as set-but-unused. - __claim __claims([[maybe_unused]] const ::std::exception* __exception) noexcept(__nothrow_pred) + _CCCL_HOST_API __claim __claims([[maybe_unused]] const ::std::exception* __exception) noexcept(__nothrow_pred) { if constexpr (::cuda::std::is_invocable_v<_Pred&, const ::std::exception*>) { @@ -1051,11 +1059,11 @@ struct when } template - void __claimed(const ::std::exception*, const ::cuda::std::source_location, _Fn&) noexcept + _CCCL_HOST_API void __claimed(const ::std::exception*, const ::cuda::std::source_location, _Fn&) noexcept {} template - void on_thrown(const ::std::exception* __exception, const ::cuda::std::source_location, _Fn&) + _CCCL_HOST_API void on_thrown(const ::std::exception* __exception, const ::cuda::std::source_location, _Fn&) { if (__claims(__exception) == __claim::__yes) { @@ -1071,7 +1079,8 @@ struct when class _Fn, class _Self = _Pred, ::cuda::std::enable_if_t<::cuda::std::is_invocable_v<_Self&>, int> = 0> - detail::__status_answer<_Status> on_returned(const _Status __status, const ::cuda::std::source_location, _Fn&) + _CCCL_HOST_API detail::__status_answer<_Status> + on_returned(const _Status __status, const ::cuda::std::source_location, _Fn&) { if (__pred_()) { @@ -1111,13 +1120,14 @@ struct translate_t<_To> using __pure_claim_tag _CCCL_NODEBUG = void; //! Everything is claimed; the funnel pointer, null or not, says how to build the `_To`. - __claim __claims(const ::std::exception*) const noexcept + _CCCL_HOST_API __claim __claims(const ::std::exception*) const noexcept { return __claim::__yes; } template - [[noreturn]] nullval __claimed(const ::std::exception* __e, const ::cuda::std::source_location, _Fn&) const + [[noreturn]] _CCCL_HOST_API nullval + __claimed(const ::std::exception* __e, const ::cuda::std::source_location, _Fn&) const { if (__e) { @@ -1128,7 +1138,8 @@ struct translate_t<_To> } template - [[noreturn]] nullval on_thrown(const ::std::exception* __e, const ::cuda::std::source_location __loc, _Fn& __fn) const + [[noreturn]] _CCCL_HOST_API nullval + on_thrown(const ::std::exception* __e, const ::cuda::std::source_location __loc, _Fn& __fn) const { __claimed(__e, __loc, __fn); _CCCL_UNREACHABLE(); // nvcc's front end does not propagate [[noreturn]] through the call above @@ -1137,7 +1148,7 @@ struct translate_t<_To> // Builds and throws the `_To`: from the caught `_From` when such a constructor exists, from // its `what()` otherwise, default-constructed failing both. Shared with `translate_t<_From, _To>`. template - [[noreturn]] static void __throw_translated(const _From& __from) + [[noreturn]] _CCCL_HOST_API static void __throw_translated(const _From& __from) { if constexpr (::cuda::std::is_constructible_v<_To, const _From&>) { @@ -1160,7 +1171,7 @@ struct translate_t<_To> // The `_To` for a non-standard exception, which offers nothing to build from: say so when // `_To` takes a message, default-construct otherwise. - [[noreturn]] static void __throw_translated() + [[noreturn]] _CCCL_HOST_API static void __throw_translated() { if constexpr (::cuda::std::is_constructible_v<_To, const char*>) { @@ -1190,7 +1201,7 @@ struct translate_t<_From, _To> // The funnel pointer decides the claim for std-derived exceptions; a non-std exception is // re-observed by the hook. - __claim __claims(const ::std::exception* __e) const noexcept + _CCCL_HOST_API __claim __claims(const ::std::exception* __e) const noexcept { if (__e) { @@ -1200,7 +1211,8 @@ struct translate_t<_From, _To> } template - [[noreturn]] nullval __claimed(const ::std::exception* __e, const ::cuda::std::source_location, _Fn&) const + [[noreturn]] _CCCL_HOST_API nullval + __claimed(const ::std::exception* __e, const ::cuda::std::source_location, _Fn&) const { // The claim was answered yes, so the cast holds. translate_t<_To>::__throw_translated(*dynamic_cast(__e)); @@ -1208,7 +1220,8 @@ struct translate_t<_From, _To> } template - [[noreturn]] nullval on_thrown(const ::std::exception* __e, const ::cuda::std::source_location, _Fn&) const + [[noreturn]] _CCCL_HOST_API nullval + on_thrown(const ::std::exception* __e, const ::cuda::std::source_location, _Fn&) const { if (__e) { @@ -1250,7 +1263,7 @@ struct nest_t static_assert(::cuda::std::is_copy_constructible_v<_E>, "nest(e) requires a copyable exception object"); template - [[noreturn]] nullval on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn&) + [[noreturn]] _CCCL_HOST_API nullval on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn&) { _CCCL_TRY { @@ -1266,7 +1279,7 @@ struct nest_t //! @brief Stores an exception by value and nests the active exception beneath it. template -auto nest(_E&& __exception) +_CCCL_HOST_API auto nest(_E&& __exception) { using _Stored = ::cuda::std::decay_t<_E>; return nest_t<_Stored>{::cuda::std::forward<_E>(__exception)}; @@ -1284,14 +1297,14 @@ struct delay_t _Duration __duration_; template - void on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn&) + _CCCL_HOST_API void on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn&) { ::std::this_thread::sleep_for(__duration_); } //! @brief The code-channel hook: the same sleep, an effect. template - void on_returned(const _Status, const ::cuda::std::source_location, _Fn&) + _CCCL_HOST_API void on_returned(const _Status, const ::cuda::std::source_location, _Fn&) { ::std::this_thread::sleep_for(__duration_); } @@ -1299,7 +1312,7 @@ struct delay_t //! @brief Creates a sleeping effect policy; see @ref delay_t. template -auto delay(_Duration&& __duration) +_CCCL_HOST_API auto delay(_Duration&& __duration) { return delay_t<_Duration>{::cuda::std::forward<_Duration>(__duration)}; } @@ -1318,7 +1331,7 @@ struct backoff_t ::std::chrono::milliseconds __initial_; template - decltype(auto) on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn& __fn) + _CCCL_HOST_API decltype(auto) on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn& __fn) { if (__n_ == 0) { @@ -1369,7 +1382,8 @@ struct backoff_t //! @brief The code-channel hook: re-runs a status-returning action after each delay while its //! result fails; exhaustion passes the latest status through. An exception from a re-run escapes. template - detail::__status_answer<_Status> on_returned(_Status __status, const ::cuda::std::source_location, _Fn& __fn) const + _CCCL_HOST_API detail::__status_answer<_Status> + on_returned(_Status __status, const ::cuda::std::source_location, _Fn& __fn) const { static_assert(::cuda::std::is_same_v<::cuda::std::remove_cvref_t, _Status>, "re-running on the code channel needs an action that returns the same status type"); @@ -1398,7 +1412,8 @@ private: // One step of the decorrelated jitter: the next sleep lies between the base and three times the // current sleep, capped. template - static void __jitter(unsigned long long& __state, _Rep& __sleep, const _Rep __base, const _Rep __cap) noexcept + _CCCL_HOST_API static void + __jitter(unsigned long long& __state, _Rep& __sleep, const _Rep __base, const _Rep __cap) noexcept { __state ^= __state << 13; __state ^= __state >> 7; @@ -1419,7 +1434,7 @@ inline constexpr bool __needs_future_v = true; #endif // _CCCL_DOXYGEN_INVOKED //! @brief Creates a decorrelated-jitter retry policy. -inline backoff_t backoff(int __n, ::std::chrono::milliseconds __initial) +_CCCL_HOST_API inline backoff_t backoff(int __n, ::std::chrono::milliseconds __initial) { _CCCL_ASSERT(__n >= 0, "backoff requires a non-negative retry count"); return backoff_t{__n, __initial}; @@ -1440,15 +1455,17 @@ struct remember_t _Ptr __cell_; // a raw pointer or a shared_ptr; *__cell_ is the last-known-good value template - ::cuda::std::conditional_t<::cuda::std::is_lvalue_reference_v<_R&&>, _R&&, ::cuda::std::remove_cvref_t<_R>> - on_success(_R&& __result) + _CCCL_HOST_API ::cuda::std:: + conditional_t<::cuda::std::is_lvalue_reference_v<_R&&>, _R&&, ::cuda::std::remove_cvref_t<_R>> + on_success(_R&& __result) { *__cell_ = __result; return ::cuda::std::forward<_R>(__result); } template - decltype(auto) on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn&) const noexcept + _CCCL_HOST_API decltype(auto) + on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn&) const noexcept { return *__cell_; } @@ -1456,7 +1473,7 @@ struct remember_t //! @brief Creates a last-known-good policy over a caller-owned cell. template -auto remember(_T* __cell) +_CCCL_HOST_API auto remember(_T* __cell) { _CCCL_ASSERT(__cell, "remember requires a non-null cell"); return remember_t<_T*>{__cell}; @@ -1464,7 +1481,7 @@ auto remember(_T* __cell) //! @brief Creates a last-known-good policy that shares ownership of its cell. template -auto remember(::std::shared_ptr<_T> __cell) +_CCCL_HOST_API auto remember(::std::shared_ptr<_T> __cell) { _CCCL_ASSERT(__cell, "remember requires a non-null cell"); return remember_t<::std::shared_ptr<_T>>{::cuda::std::move(__cell)}; @@ -1498,7 +1515,7 @@ inline constexpr bool __produces_value_v> = true; //! handles it. struct circuit_open : ::std::runtime_error { - circuit_open() + _CCCL_HOST_API circuit_open() : ::std::runtime_error("circuit breaker open: the failure budget is spent") {} }; @@ -1528,7 +1545,7 @@ struct circuit_breaker_t int __initial_; //! @brief The entry gate: refuses the attempt once the budget is spent. - void on_enter() const + _CCCL_HOST_API void on_enter() const { if (*__budget_ <= 0) { @@ -1538,21 +1555,22 @@ struct circuit_breaker_t //! @brief The exception hook: record the failure, answer as an effect. template - void on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn&) const noexcept + _CCCL_HOST_API void on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn&) const noexcept { --*__budget_; } //! @brief Success restores the budget to its creation-time value. template - ::cuda::std::conditional_t<::cuda::std::is_lvalue_reference_v<_R&&>, _R&&, ::cuda::std::remove_cvref_t<_R>> - on_success(_R&& __result) const + _CCCL_HOST_API ::cuda::std:: + conditional_t<::cuda::std::is_lvalue_reference_v<_R&&>, _R&&, ::cuda::std::remove_cvref_t<_R>> + on_success(_R&& __result) const { *__budget_ = __initial_; return ::cuda::std::forward<_R>(__result); } - void on_success() const + _CCCL_HOST_API void on_success() const { *__budget_ = __initial_; } @@ -1560,7 +1578,7 @@ struct circuit_breaker_t //! @brief Creates a counter-based circuit breaker; see @ref circuit_breaker_t. The initial //! `*__budget` is the failure allowance restored on success. -inline circuit_breaker_t circuit_breaker(::std::shared_ptr __budget) +_CCCL_HOST_API inline circuit_breaker_t circuit_breaker(::std::shared_ptr __budget) { _CCCL_ASSERT(__budget, "circuit_breaker requires a non-null budget"); const int __initial = *__budget; @@ -1627,9 +1645,9 @@ struct store_t _Target __target_; template - void on_thrown([[maybe_unused]] const ::std::exception* __e, - const ::cuda::std::source_location, - _Fn&) noexcept(!detail::__store_may_decline<_Target>) + _CCCL_HOST_API void on_thrown([[maybe_unused]] const ::std::exception* __e, + const ::cuda::std::source_location, + _Fn&) noexcept(!detail::__store_may_decline<_Target>) { if constexpr (::cuda::std::is_same_v<_Target, ::cuda::std::span>) { @@ -1696,7 +1714,8 @@ struct store_t class _Self = _Target, ::cuda::std::enable_if_t, int> = 0> // maybe_unused: only the exception_ptr target reads the location; gcc 9 flags the others. - void on_returned(const _Status __status, [[maybe_unused]] const ::cuda::std::source_location __loc, _Fn&) noexcept + _CCCL_HOST_API void + on_returned(const _Status __status, [[maybe_unused]] const ::cuda::std::source_location __loc, _Fn&) noexcept { if constexpr (::cuda::std::is_same_v<_Target, ::cuda::std::span>) { @@ -1743,34 +1762,34 @@ struct store_t }; //! @brief Store the active exception, full dynamic type preserved, into `*__target`. -inline auto store(::std::exception_ptr* __target) +_CCCL_HOST_API inline auto store(::std::exception_ptr* __target) { _CCCL_ASSERT(__target != nullptr, "store requires a non-null target"); return store_t<::std::exception_ptr*>{__target}; } //! @brief See @ref store. Shared-ownership form. -inline auto store(::std::shared_ptr<::std::exception_ptr> __target) +_CCCL_HOST_API inline auto store(::std::shared_ptr<::std::exception_ptr> __target) { _CCCL_ASSERT(__target, "store requires a non-null target"); return store_t<::std::shared_ptr<::std::exception_ptr>>{::cuda::std::move(__target)}; } //! @brief Store the active exception's message into a caller-owned buffer, allocation-free. -inline auto store(::cuda::std::span __target) +_CCCL_HOST_API inline auto store(::cuda::std::span __target) { return store_t<::cuda::std::span>{__target}; } //! @brief Store the active exception's message into `*__target`, allocating; best effort. -inline auto store(::std::string* __target) +_CCCL_HOST_API inline auto store(::std::string* __target) { _CCCL_ASSERT(__target != nullptr, "store requires a non-null target"); return store_t<::std::string*>{__target}; } //! @brief See @ref store. Shared-ownership form. -inline auto store(::std::shared_ptr<::std::string> __target) +_CCCL_HOST_API inline auto store(::std::shared_ptr<::std::string> __target) { _CCCL_ASSERT(__target, "store requires a non-null target"); return store_t<::std::shared_ptr<::std::string>>{::cuda::std::move(__target)}; @@ -1780,7 +1799,7 @@ inline auto store(::std::shared_ptr<::std::string> __target) template < class _E, ::cuda::std::enable_if_t<::cuda::std::is_base_of_v<::std::exception, _E> && !::cuda::std::is_const_v<_E>, int> = 0> -auto store(_E* __target) +_CCCL_HOST_API auto store(_E* __target) { _CCCL_ASSERT(__target != nullptr, "store requires a non-null target"); return store_t<_E*>{__target}; @@ -1790,7 +1809,7 @@ auto store(_E* __target) template < class _E, ::cuda::std::enable_if_t<::cuda::std::is_base_of_v<::std::exception, _E> && !::cuda::std::is_const_v<_E>, int> = 0> -auto store(::std::shared_ptr<_E> __target) +_CCCL_HOST_API auto store(::std::shared_ptr<_E> __target) { _CCCL_ASSERT(__target, "store requires a non-null target"); return store_t<::std::shared_ptr<_E>>{::cuda::std::move(__target)}; @@ -1876,7 +1895,7 @@ inline constexpr bool __has_claim = // policy without one, so code that names neither `__claims` nor the trait's operand directly // compiles inside generic lambdas, where nvcc 12.0 instantiates discarded constexpr branches. template -constexpr bool __nothrow_claim_impl() +_CCCL_HOST_API constexpr bool __nothrow_claim_impl() { if constexpr (__has_claim<_P>) { @@ -1892,7 +1911,7 @@ template inline constexpr bool __nothrow_claim = __nothrow_claim_impl<_P>(); template -_CCCL_FORCEINLINE __claim +_CCCL_HOST_API _CCCL_FORCEINLINE __claim __ask_claim(_P& __p, [[maybe_unused]] const ::std::exception* __exception) noexcept(__nothrow_claim<_P>) { if constexpr (__has_claim<_P>) @@ -1920,7 +1939,7 @@ struct __claimed_view _P& __p_; template - _CCCL_FORCEINLINE decltype(auto) + _CCCL_HOST_API _CCCL_FORCEINLINE decltype(auto) on_thrown(const ::std::exception* __exception, const ::cuda::std::source_location __loc, _Fn& __fn) { return __p_.__claimed(__exception, __loc, __fn); @@ -2016,7 +2035,7 @@ inline constexpr bool __never_passes_through_v = __has_exception_hook<_P, _Fn> & // Invoke the exception hook of a policy (`on_thrown`) or of a raw callable (its call operator). template -_CCCL_FORCEINLINE decltype(auto) +_CCCL_HOST_API _CCCL_FORCEINLINE decltype(auto) __call_hook(_P& __p, const ::std::exception* __exception, const ::cuda::std::source_location __loc, _Fn& __fn) noexcept( __hook_nothrow_v<_P, _Fn>) { @@ -2056,7 +2075,7 @@ struct __past_result { _Status __status_; - _Status operator()() const noexcept + _CCCL_HOST_API _Status operator()() const noexcept { return __status_; } @@ -2104,7 +2123,7 @@ inline constexpr bool __status_path_nothrow_v<_P, _Status, _Fn, false, true> = // Offer a failing status to a policy (defined with the answer interpretation below). template -__status_answer<_Status> +_CCCL_HOST_API __status_answer<_Status> __offer_status(_P& __policy, _Status __status, ::cuda::std::source_location __loc, _Fn& __fn) noexcept( __status_path_nothrow_v<_P, _Status, _Fn>); @@ -2119,19 +2138,19 @@ struct __forwards_success _P __p_; template , int> = 0> - decltype(auto) on_success(_R&& __r) + _CCCL_HOST_API decltype(auto) on_success(_R&& __r) { return __p_.on_success(::cuda::std::forward<_R>(__r)); } template , int> = 0> - decltype(auto) on_success() + _CCCL_HOST_API decltype(auto) on_success() { return __p_.on_success(); } template , int> = 0> - void on_enter() noexcept(__on_enter_nothrow_v<_P>) + _CCCL_HOST_API void on_enter() noexcept(__on_enter_nothrow_v<_P>) { __p_.on_enter(); } @@ -2171,7 +2190,7 @@ struct __if_isa_t : __forwards_success<_P> // active exception is std-derived; otherwise unknown, and the hook re-observes. A guard over a // hookless policy has no hook to claim for, so it has no claim either. template , int> = 0> - __claim __claims([[maybe_unused]] const ::std::exception* __exception) const noexcept + _CCCL_HOST_API __claim __claims([[maybe_unused]] const ::std::exception* __exception) const noexcept { if constexpr ((::cuda::std::is_class_v<_Es> && ...)) { @@ -2184,7 +2203,7 @@ struct __if_isa_t : __forwards_success<_P> } template , int> = 0> - _CCCL_FORCEINLINE decltype(auto) + _CCCL_HOST_API _CCCL_FORCEINLINE decltype(auto) __claimed(const ::std::exception* __exception, const ::cuda::std::source_location __loc, _Fn& __fn) { return detail::__call_hook(this->__p_, __exception, __loc, __fn); @@ -2193,7 +2212,7 @@ struct __if_isa_t : __forwards_success<_P> // Does the active exception match any listed type? A recursive ladder of re-observations; // the binding must be named for the no-exceptions expansion of _CCCL_CATCH. template - static bool __matches_active(const ::std::exception* __e) + _CCCL_HOST_API static bool __matches_active(const ::std::exception* __e) { // Fast path: for a class target and a std-derived active exception, the funnel pointer // decides via dynamic_cast -- no rethrow. (dynamic_cast agrees with catch matching: @@ -2239,7 +2258,7 @@ struct __if_isa_t : __forwards_success<_P> } template , int> = 0> - _CCCL_FORCEINLINE decltype(auto) + _CCCL_HOST_API _CCCL_FORCEINLINE decltype(auto) on_thrown(const ::std::exception* __exception, const ::cuda::std::source_location __loc, _Fn& __fn) { if (__matches_active<_Es...>(__exception)) @@ -2252,7 +2271,7 @@ struct __if_isa_t : __forwards_success<_P> // A type selector has nothing to match on the code channel: the status passes through. template - fwd_t on_returned(const _Status, const ::cuda::std::source_location, _Fn&) const noexcept + _CCCL_HOST_API fwd_t on_returned(const _Status, const ::cuda::std::source_location, _Fn&) const noexcept { return {}; } @@ -2295,20 +2314,20 @@ struct __if_monomorphic_t : __forwards_success<_P> // Exact matching reads typeid through the funnel, so the claim is always decided: a null // funnel is a non-std exception, which this guard declines. Hookless guards have no claim. template , int> = 0> - __claim __claims(const ::std::exception* __exception) const noexcept + _CCCL_HOST_API __claim __claims(const ::std::exception* __exception) const noexcept { return (__exception != nullptr && ((typeid(*__exception) == typeid(_Es)) || ...)) ? __claim::__yes : __claim::__no; } template , int> = 0> - _CCCL_FORCEINLINE decltype(auto) + _CCCL_HOST_API _CCCL_FORCEINLINE decltype(auto) __claimed(const ::std::exception* __exception, const ::cuda::std::source_location __loc, _Fn& __fn) { return detail::__call_hook(this->__p_, __exception, __loc, __fn); } template , int> = 0> - _CCCL_FORCEINLINE decltype(auto) + _CCCL_HOST_API _CCCL_FORCEINLINE decltype(auto) on_thrown(const ::std::exception* __exception, const ::cuda::std::source_location __loc, _Fn& __fn) { if (__claims(__exception) == __claim::__yes) @@ -2320,7 +2339,7 @@ struct __if_monomorphic_t : __forwards_success<_P> // A type selector has nothing to match on the code channel: the status passes through. template - fwd_t on_returned(const _Status, const ::cuda::std::source_location, _Fn&) const noexcept + _CCCL_HOST_API fwd_t on_returned(const _Status, const ::cuda::std::source_location, _Fn&) const noexcept { return {}; } @@ -2344,7 +2363,7 @@ struct __as_policy : __forwards_success<_P> _CUDAX_STF_SINK(__as_policy) template , int> = 0> - _CCCL_FORCEINLINE decltype(auto) + _CCCL_HOST_API _CCCL_FORCEINLINE decltype(auto) on_thrown(const ::std::exception* __exception, const ::cuda::std::source_location __loc, _Fn& __fn) { return detail::__call_hook(this->__p_, __exception, __loc, __fn); @@ -2355,7 +2374,7 @@ struct __as_policy : __forwards_success<_P> // value stored inside carries its own ref-ness (e.g. subst_t for a reference result). // Every normal form is sink-tagged: raw capability-bearing callables wrap in `__as_policy`. template -auto __normalize(_R&& __r) +_CCCL_HOST_API auto __normalize(_R&& __r) { using _P = ::cuda::std::remove_cvref_t<_R>; if constexpr (__is_exception_sink_v<_P>) @@ -2393,7 +2412,7 @@ struct __composite_hooks template || __has_on_success_with<_L, _Rr>, int> = 0> - decltype(auto) on_success(_Rr&& __r) + _CCCL_HOST_API decltype(auto) on_success(_Rr&& __r) { if constexpr (__has_on_success_with<_R, _Rr>) { @@ -2408,7 +2427,7 @@ struct __composite_hooks template || __has_on_success_void<_LL>, int> = 0> - decltype(auto) on_success() + _CCCL_HOST_API decltype(auto) on_success() { if constexpr (__has_on_success_void<_R>) { @@ -2422,7 +2441,7 @@ struct __composite_hooks // Entry gates run left to right: each side may refuse the attempt before it starts. template || __has_on_enter<_RR>, int> = 0> - void on_enter() noexcept(__on_enter_nothrow_v<_L> && __on_enter_nothrow_v<_R>) + _CCCL_HOST_API void on_enter() noexcept(__on_enter_nothrow_v<_L> && __on_enter_nothrow_v<_R>) { if constexpr (__has_on_enter<_L>) { @@ -2459,7 +2478,7 @@ struct __policy_and : __composite_hooks<_L, _R> class _LL = _L, class _RR = _R, ::cuda::std::enable_if_t<__has_exception_hook<_LL> || __has_exception_hook<_RR>, int> = 0> - _CCCL_FORCEINLINE decltype(auto) + _CCCL_HOST_API _CCCL_FORCEINLINE decltype(auto) on_thrown(const ::std::exception* __exception, const ::cuda::std::source_location __loc, _Fn& __fn) { if constexpr (__has_exception_hook<_L>) @@ -2476,13 +2495,13 @@ struct __policy_and : __composite_hooks<_L, _R> // whole sequence before anything has run, and it is the only element that runs nothing before // deciding. A later selector is asked in its own position, after the effects before it. template , int> = 0> - __claim __claims(const ::std::exception* __exception) noexcept(__nothrow_claim<_LL>) + _CCCL_HOST_API __claim __claims(const ::std::exception* __exception) noexcept(__nothrow_claim<_LL>) { return this->__l_.__claims(__exception); } template , int> = 0> - _CCCL_FORCEINLINE decltype(auto) + _CCCL_HOST_API _CCCL_FORCEINLINE decltype(auto) __claimed(const ::std::exception* __exception, const ::cuda::std::source_location __loc, _Fn& __fn) { static_cast(this->__l_.__claimed(__exception, __loc, __fn)); @@ -2501,7 +2520,7 @@ struct __policy_and : __composite_hooks<_L, _R> class _LL = _L, class _RR = _R, ::cuda::std::enable_if_t<__has_status_path<_LL, _Status, _Fn> || __has_status_path<_RR, _Status, _Fn>, int> = 0> - __status_answer<_Status> on_returned( + _CCCL_HOST_API __status_answer<_Status> on_returned( const _Status __status, const ::cuda::std::source_location __loc, _Fn& __fn) noexcept(__status_path_nothrow_v<_L, _Status, _Fn> && __status_path_nothrow_v<_R, _Status, _Fn>) @@ -2615,7 +2634,7 @@ struct __policy_or : __composite_hooks<_L, _R> class _LL = _L, class _RR = _R, ::cuda::std::enable_if_t<__has_exception_hook<_LL> && __has_exception_hook<_RR>, int> = 0> - _CCCL_FORCEINLINE __composite_answer_t<_Fn> + _CCCL_HOST_API _CCCL_FORCEINLINE __composite_answer_t<_Fn> on_thrown(const ::std::exception* __exception, const ::cuda::std::source_location __loc, _Fn& __fn) { // A selecting left arm is asked first. `__no` is the decline, by return: the right arm runs @@ -2686,7 +2705,7 @@ struct __policy_or : __composite_hooks<_L, _R> // The code channel: `_L` gets first claim; a status it passes through goes to `_R`. Passing // through is a returned answer here, so there is nothing to re-observe. template - __status_answer<_Status> on_returned( + _CCCL_HOST_API __status_answer<_Status> on_returned( const _Status __status, const ::cuda::std::source_location __loc, _Fn& __fn) noexcept(__status_path_nothrow_v<_L, _Status, _Fn> && __status_path_nothrow_v<_R, _Status, _Fn>) @@ -2707,7 +2726,8 @@ struct __policy_or : __composite_hooks<_L, _R> template && __has_pure_claim<_RR>, int> = 0> - __claim __claims(const ::std::exception* __exception) noexcept(__nothrow_claim<_LL> && __nothrow_claim<_RR>) + _CCCL_HOST_API __claim + __claims(const ::std::exception* __exception) noexcept(__nothrow_claim<_LL> && __nothrow_claim<_RR>) { const __claim __l = this->__l_.__claims(__exception); if (__l == __claim::__yes) @@ -2726,7 +2746,7 @@ struct __policy_or : __composite_hooks<_L, _R> class _LL = _L, class _RR = _R, ::cuda::std::enable_if_t<__has_pure_claim<_LL> && __has_pure_claim<_RR>, int> = 0> - _CCCL_FORCEINLINE __composite_answer_t<_Fn> + _CCCL_HOST_API _CCCL_FORCEINLINE __composite_answer_t<_Fn> __claimed(const ::std::exception* __exception, const ::cuda::std::source_location __loc, _Fn& __fn) { return this->on_thrown(__exception, __loc, __fn); @@ -2735,7 +2755,7 @@ struct __policy_or : __composite_hooks<_L, _R> private: // One arm's answer at the composite's answer type (the resume tag for a void callable). template - static _CCCL_FORCEINLINE __composite_answer_t<_Fn> + _CCCL_HOST_API static _CCCL_FORCEINLINE __composite_answer_t<_Fn> __answer(_Arm& __arm, const ::std::exception* __exception, const ::cuda::std::source_location __loc, _Fn& __fn) { using _Raw = decltype(__fn()); @@ -2751,7 +2771,7 @@ private: } template - _CCCL_FORCEINLINE __composite_answer_t<_Fn> + _CCCL_HOST_API _CCCL_FORCEINLINE __composite_answer_t<_Fn> __right(const ::std::exception* __exception, const ::cuda::std::source_location __loc, _Fn& __fn) { return __answer<_Fn>(this->__r_, __exception, __loc, __fn); @@ -2798,7 +2818,7 @@ struct __policy_pow : __forwards_success<_P> "unreachable"); template - _CCCL_FORCEINLINE __composite_answer_t<_Fn> + _CCCL_HOST_API _CCCL_FORCEINLINE __composite_answer_t<_Fn> on_thrown(const ::std::exception* __exception, const ::cuda::std::source_location __loc, _Fn& __fn) { using _Expr = decltype(__fn()); @@ -2914,13 +2934,13 @@ struct __policy_pow : __forwards_success<_P> // The repetition's claim is the repeated policy's, when pure: a no is a no for every // repetition. template , int> = 0> - __claim __claims(const ::std::exception* __exception) noexcept(__nothrow_claim<_PP>) + _CCCL_HOST_API __claim __claims(const ::std::exception* __exception) noexcept(__nothrow_claim<_PP>) { return this->__p_.__claims(__exception); } template , int> = 0> - _CCCL_FORCEINLINE __composite_answer_t<_Fn> + _CCCL_HOST_API _CCCL_FORCEINLINE __composite_answer_t<_Fn> __claimed(const ::std::exception* __exception, const ::cuda::std::source_location __loc, _Fn& __fn) { return this->on_thrown(__exception, __loc, __fn); @@ -2929,9 +2949,10 @@ struct __policy_pow : __forwards_success<_P> // The code channel: offer the status up to `__n_` times while it passes through; `n == 0` // passes it through at once. template - __status_answer<_Status> on_returned(const _Status __status, - const ::cuda::std::source_location __loc, - _Fn& __fn) noexcept(__status_path_nothrow_v<_P, _Status, _Fn>) + _CCCL_HOST_API __status_answer<_Status> + on_returned(const _Status __status, + const ::cuda::std::source_location __loc, + _Fn& __fn) noexcept(__status_path_nothrow_v<_P, _Status, _Fn>) { _Status __latest = __status; for (int __i = 0; __i < __n_; ++__i) @@ -2982,7 +3003,7 @@ struct __integral_base<_T, true> }; template -constexpr bool __value_preserving_impl() +_CCCL_HOST_API constexpr bool __value_preserving_impl() { using _F = typename __integral_base<::cuda::std::remove_cvref_t<_From>>::type; using _T = ::cuda::std::remove_cvref_t<_To>; @@ -3027,7 +3048,7 @@ inline constexpr bool __value_preserving_v = // `_Final` is false where `&` discards the answer: only "passed through" matters there. // maybe_unused: only the fwd answer reads the status; gcc 9 flags the other instantiations. template -__status_answer<_Status> __interpret_status([[maybe_unused]] const _Status __status, _Call&& __call) +_CCCL_HOST_API __status_answer<_Status> __interpret_status([[maybe_unused]] const _Status __status, _Call&& __call) { using _Answer = decltype(__call()); using _A = ::cuda::std::remove_cvref_t<_Answer>; @@ -3074,7 +3095,7 @@ __status_answer<_Status> __interpret_status([[maybe_unused]] const _Status __sta // object caught ([except.throw]), so identity tells passing through from a new exception, which // escapes. Exceptions that do not derive from std::exception cannot be compared and escape too. template -__status_answer<_Status> +_CCCL_HOST_API __status_answer<_Status> __handle_via_exception(_P& __policy, const _Status __status, const ::cuda::std::source_location __loc, _Fn& __fn) { using _Exception = decltype(status_traits<_Status>::to_exception(__status, __loc)); @@ -3120,7 +3141,7 @@ __handle_via_exception(_P& __policy, const _Status __status, const ::cuda::std:: // hook, else nothing answers and the status passes through (reading only the status, which gcc 9 // flags without the attributes). template -__status_answer<_Status> __offer_status( +_CCCL_HOST_API __status_answer<_Status> __offer_status( [[maybe_unused]] _P& __policy, const _Status __status, [[maybe_unused]] const ::cuda::std::source_location __loc, @@ -3150,7 +3171,7 @@ __status_answer<_Status> __offer_status( // Interpret the final element's answer as the expression's value, converting to `_Expr`. template -_CCCL_FORCEINLINE _Expr __interpret_answer( +_CCCL_HOST_API _CCCL_FORCEINLINE _Expr __interpret_answer( _P& __policy, const ::std::exception* __exception, const ::cuda::std::source_location __loc, _Fn& __fn) { using _Answer = __hook_answer_t<_P, _Fn>; @@ -3209,7 +3230,7 @@ _CCCL_FORCEINLINE _Expr __interpret_answer( // interpret. The parameters go unread in the propagate instantiation, which gcc 9 flags // without the attribute. template -_CCCL_FORCEINLINE _Expr __on_exception( +_CCCL_HOST_API _CCCL_FORCEINLINE _Expr __on_exception( _P& __policy, [[maybe_unused]] const ::std::exception* __exception, [[maybe_unused]] const ::cuda::std::source_location __loc, @@ -3239,7 +3260,7 @@ class __chain_state public: template - void __fix_type() noexcept + _CCCL_HOST_API void __fix_type() noexcept { static_assert( ::cuda::is_trivially_copyable_v<_Status> && sizeof(_Status) <= sizeof(__bytes_) @@ -3255,7 +3276,7 @@ public: } template - void __record(const __status_answer<_Status> __answer) noexcept + _CCCL_HOST_API void __record(const __status_answer<_Status> __answer) noexcept { __forwarded_ |= __answer.__passed_; if (!__recorded_ && __answer.__value_ != status_traits<_Status>::success()) @@ -3266,7 +3287,7 @@ public: } template - [[nodiscard]] _Status __first() const noexcept + [[nodiscard]] _CCCL_HOST_API _Status __first() const noexcept { _CCCL_VERIFY(__type_ == nullptr || *__type_ == _CCCL_TYPEID(_Status), "status(): S is not the chain's status type"); @@ -3278,7 +3299,7 @@ public: return __status; } - [[nodiscard]] bool __passed() const noexcept + [[nodiscard]] _CCCL_HOST_API bool __passed() const noexcept { return __forwarded_; } @@ -3293,7 +3314,7 @@ struct __on_throw_policy const ::cuda::std::source_location __loc_; __chain_state __chain_; - __on_throw_policy(_Reaction __reaction, const ::cuda::std::source_location __loc) + _CCCL_HOST_API __on_throw_policy(_Reaction __reaction, const ::cuda::std::source_location __loc) : __reaction_(::cuda::std::move(__reaction)) , __loc_(__loc) {} @@ -3306,14 +3327,14 @@ struct __on_throw_policy //! @tparam _Status The chain's status type, verified with `_CCCL_VERIFY`. //! @return The first yielded failing status, or `status_traits<_Status>::success()`. template - [[nodiscard]] _Status status() const noexcept + [[nodiscard]] _CCCL_HOST_API _Status status() const noexcept { return __chain_.template __first<_Status>(); } //! @brief Whether any status fed to the chain forwarded. //! @return `true` if one did. - [[nodiscard]] bool forwarded() const noexcept + [[nodiscard]] _CCCL_HOST_API bool forwarded() const noexcept { return __chain_.__passed(); } @@ -3333,7 +3354,7 @@ struct __guarded_action __on_throw_policy<_Reaction>& __policy_; _Fn& __fn_; - _Expr operator()() const + _CCCL_HOST_API _Expr operator()() const { _CCCL_TRY { @@ -3375,7 +3396,7 @@ inline constexpr bool __callable_runner_nothrow_v<_Reaction, _Fn, _Feed, _Expr, // goes to the policy's code channel with the action itself, which re-running policies call again. // The answer says whether the policy forwarded, which `<<` records. template -__status_answer<_Status> +_CCCL_HOST_API __status_answer<_Status> __on_returned_status(_P& __policy, const _Status __status, const ::cuda::std::source_location __loc, _Fn& __fn) { if (__status_failed(__status)) @@ -3409,8 +3430,8 @@ template & __policy, - _Fn&& __fn) noexcept(__callable_runner_nothrow_v<_Reaction, _Fn, _Feed>) +_CCCL_HOST_API decltype(auto) __run_under([[maybe_unused]] __on_throw_policy<_Reaction>& __policy, + _Fn&& __fn) noexcept(__callable_runner_nothrow_v<_Reaction, _Fn, _Feed>) { // Bind as a non-const lvalue: a hook may invoke it again later, and a mutable callable needs it. // NOLINTNEXTLINE(misc-const-correctness) @@ -3554,8 +3575,9 @@ template // In clang-tidy's device pass _CCCL_TRY/_CCCL_CATCH expand to no handler, so bugprone-exception-escape // sees a throw from the exception bridge escape this runner in instantiations whose policy makes it noexcept. // NOLINTNEXTLINE(bugprone-exception-escape) -__status_answer<_Status> __offer_past(__on_throw_policy<_Reaction>& __policy, const _Status __status) noexcept( - __status_path_nothrow_v<_Reaction, _Status, __past_result<_Status>>) +_CCCL_HOST_API __status_answer<_Status> +__offer_past(__on_throw_policy<_Reaction>& __policy, + const _Status __status) noexcept(__status_path_nothrow_v<_Reaction, _Status, __past_result<_Status>>) { static_assert(!__selects_thrown_v<_Reaction>, "a type selector (if_isa, if_monomorphic, thrown) never matches a status operand, which cannot " @@ -3575,7 +3597,7 @@ __status_answer<_Status> __offer_past(__on_throw_policy<_Reaction>& __policy, co // itself for a forwarded one, for the caller to propagate with `return`. template , int> = 0> // NOLINTNEXTLINE(bugprone-exception-escape) -_Status __run_under(__on_throw_policy<_Reaction>& __policy, const _Status __status) noexcept( +_CCCL_HOST_API _Status __run_under(__on_throw_policy<_Reaction>& __policy, const _Status __status) noexcept( __status_path_nothrow_v<_Reaction, _Status, __past_result<_Status>>) { return __offer_past(__policy, __status).__value_; @@ -3586,7 +3608,7 @@ template > && !::cuda::std::is_invocable_v<_X&>, int> = 0> -void __run_under(__on_throw_policy<_Reaction>&, _X&&) +_CCCL_HOST_API void __run_under(__on_throw_policy<_Reaction>&, _X&&) { static_assert(!::cuda::std::is_same_v<_X, _X>, "the right operand of policy ->* ... is a callable; the right operand of << ... is a callable " @@ -3597,8 +3619,8 @@ void __run_under(__on_throw_policy<_Reaction>&, _X&&) // the policy made of it. template >, int> = 0> // NOLINTNEXTLINE(bugprone-exception-escape) -decltype(auto) operator->*(__on_throw_policy<_Reaction> __policy, - _Fn&& __fn) noexcept(noexcept(__run_under(__policy, ::cuda::std::forward<_Fn>(__fn)))) +_CCCL_HOST_API decltype(auto) operator->*(__on_throw_policy<_Reaction> __policy, _Fn&& __fn) noexcept( + noexcept(__run_under(__policy, ::cuda::std::forward<_Fn>(__fn)))) { return __run_under(__policy, ::cuda::std::forward<_Fn>(__fn)); } @@ -3606,7 +3628,7 @@ decltype(auto) operator->*(__on_throw_policy<_Reaction> __policy, // `p ->* status` is ill-formed: `->*` yields the value of a callable; a status (whose // value is of no interest once handled) goes on the right of `<<`. template , int> = 0> -void operator->*(__on_throw_policy<_Reaction>, const _Status) +_CCCL_HOST_API void operator->*(__on_throw_policy<_Reaction>, const _Status) { static_assert(!::cuda::std::is_same_v<_Status, _Status>, "policy ->* x: x must be a callable; feed a status with << instead"); @@ -3631,7 +3653,7 @@ struct __sink_unknown_return // The noexcept surface of `<<`: a status operand keeps the code channel's; an action keeps the // runner's for what it returns; anything else reads as potentially throwing (and does not compile). template -constexpr bool __feed_nothrow() +_CCCL_HOST_API constexpr bool __feed_nothrow() { using _T = ::cuda::std::remove_cvref_t<_X>; if constexpr (__is_status_v<_T>) @@ -3662,7 +3684,7 @@ constexpr bool __feed_nothrow() template // NOLINTNEXTLINE(bugprone-exception-escape) -void __feed(__on_throw_policy<_Reaction>& __sink, _X&& __x) noexcept(__feed_nothrow<_Reaction, _X>()) +_CCCL_HOST_API void __feed(__on_throw_policy<_Reaction>& __sink, _X&& __x) noexcept(__feed_nothrow<_Reaction, _X>()) { if constexpr (__is_status_v<::cuda::std::remove_cvref_t<_X>>) { @@ -3692,7 +3714,7 @@ void __feed(__on_throw_policy<_Reaction>& __sink, _X&& __x) noexcept(__feed_noth } template -__on_throw_policy<_Reaction>& +_CCCL_HOST_API __on_throw_policy<_Reaction>& operator<<(__on_throw_policy<_Reaction>& __sink, _X&& __x) noexcept(__feed_nothrow<_Reaction, _X>()) { __feed(__sink, ::cuda::std::forward<_X>(__x)); @@ -3701,7 +3723,7 @@ operator<<(__on_throw_policy<_Reaction>& __sink, _X&& __x) noexcept(__feed_nothr // A temporary carrier is handed back by value, so that `auto&& c = p << a;` does not dangle. template -__on_throw_policy<_Reaction> +_CCCL_HOST_API __on_throw_policy<_Reaction> operator<<(__on_throw_policy<_Reaction>&& __sink, _X&& __x) noexcept(__feed_nothrow<_Reaction, _X>()) { __feed(__sink, ::cuda::std::forward<_X>(__x)); @@ -3714,7 +3736,7 @@ struct __status_set { _Status __values_[_Count]; - [[nodiscard]] bool __contains(const _Status __status) const noexcept + [[nodiscard]] _CCCL_HOST_API bool __contains(const _Status __status) const noexcept { return ::cuda::std::any_of(__values_, __values_ + _Count, [&__status](const _Status& __value) { return __value == __status; @@ -3732,15 +3754,16 @@ struct __if_equal_t : __forwards_success<_P> __status_set<_Status, _Count> __set_; template - [[noreturn]] nullval on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn&) const + [[noreturn]] _CCCL_HOST_API nullval on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn&) const { _CCCL_RETHROW; } template - __status_answer<_Operand> on_returned(const _Operand __status, - const ::cuda::std::source_location __loc, - _Fn& __fn) noexcept(__status_path_nothrow_v<_P, _Operand, _Fn>) + _CCCL_HOST_API __status_answer<_Operand> + on_returned(const _Operand __status, + const ::cuda::std::source_location __loc, + _Fn& __fn) noexcept(__status_path_nothrow_v<_P, _Operand, _Fn>) { static_assert(::cuda::std::is_same_v<_Operand, _Status>, "if_equal(values...) matches a status of the values' own type; the operand is a status of another " @@ -3759,7 +3782,7 @@ struct __if_equal_selector __status_set<_Status, _Count> __set_; template - auto operator()(_P&& __p) const + _CCCL_HOST_API auto operator()(_P&& __p) const { auto __np = __normalize(::cuda::std::forward<_P>(__p)); return __if_equal_t{{::cuda::std::move(__np)}, __set_}; @@ -3784,13 +3807,14 @@ struct __thrown_t : __forwards_success<_P> _CUDAX_STF_SINK(__thrown_t) template , int> = 0> - decltype(auto) on_thrown(const ::std::exception* __exception, const ::cuda::std::source_location __loc, _Fn& __fn) + _CCCL_HOST_API decltype(auto) + on_thrown(const ::std::exception* __exception, const ::cuda::std::source_location __loc, _Fn& __fn) { return detail::__call_hook(this->__p_, __exception, __loc, __fn); } template - fwd_t on_returned(const _Status, const ::cuda::std::source_location, _Fn&) const noexcept + _CCCL_HOST_API fwd_t on_returned(const _Status, const ::cuda::std::source_location, _Fn&) const noexcept { return {}; } @@ -3804,15 +3828,16 @@ struct __returned_t : __forwards_success<_P> _CUDAX_STF_SINK(__returned_t) template - [[noreturn]] nullval on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn&) const + [[noreturn]] _CCCL_HOST_API nullval on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn&) const { _CCCL_RETHROW; } template - __status_answer<_Status> on_returned(const _Status __status, - const ::cuda::std::source_location __loc, - _Fn& __fn) noexcept(__status_path_nothrow_v<_P, _Status, _Fn>) + _CCCL_HOST_API __status_answer<_Status> + on_returned(const _Status __status, + const ::cuda::std::source_location __loc, + _Fn& __fn) noexcept(__status_path_nothrow_v<_P, _Status, _Fn>) { return __offer_status<_Status>(this->__p_, __status, __loc, __fn); } @@ -3827,7 +3852,8 @@ struct __thrown_fn_t _F __f_; template - auto on_thrown([[maybe_unused]] const ::std::exception* __exception, const ::cuda::std::source_location, _Fn&) + _CCCL_HOST_API auto + on_thrown([[maybe_unused]] const ::std::exception* __exception, const ::cuda::std::source_location, _Fn&) -> decltype(::cuda::std::declval<_F&>()(::cuda::std::declval())) { if constexpr (::cuda::std::is_base_of_v<::std::exception, _E>) @@ -3859,7 +3885,7 @@ struct __thrown_fn_t } template - fwd_t on_returned(const _Status, const ::cuda::std::source_location, _Fn&) const noexcept + _CCCL_HOST_API fwd_t on_returned(const _Status, const ::cuda::std::source_location, _Fn&) const noexcept { return {}; } @@ -3874,14 +3900,16 @@ struct __returned_fn_t _F __f_; template - [[noreturn]] nullval on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn&) const + [[noreturn]] _CCCL_HOST_API nullval on_thrown(const ::std::exception*, const ::cuda::std::source_location, _Fn&) const { _CCCL_RETHROW; } template - __status_answer<_Status> on_returned(const _Status __status, const ::cuda::std::source_location, _Fn&) noexcept( - !::cuda::std::is_same_v<_Status, _S> || ::cuda::std::is_nothrow_invocable_v<_F&, _S>) + _CCCL_HOST_API __status_answer<_Status> + on_returned(const _Status __status, + const ::cuda::std::source_location, + _Fn&) noexcept(!::cuda::std::is_same_v<_Status, _S> || ::cuda::std::is_nothrow_invocable_v<_F&, _S>) { if constexpr (::cuda::std::is_same_v<_Status, _S>) { @@ -3943,9 +3971,9 @@ inline constexpr bool __is_policy_argument_v = // on a dead arm. Declared here, defined after `dynamic`. template inline constexpr bool __is_dynamic_v = ::cuda::std::is_same_v<::cuda::std::remove_cvref_t<_P>, dynamic>; -inline void __check_dynamic_left_of_or(const dynamic& __p); -inline void __check_dynamic_left_of_and(const dynamic& __p); -inline void __check_dynamic_repeated(const dynamic& __p); +_CCCL_HOST_API inline void __check_dynamic_left_of_or(const dynamic& __p); +_CCCL_HOST_API inline void __check_dynamic_left_of_and(const dynamic& __p); +_CCCL_HOST_API inline void __check_dynamic_repeated(const dynamic& __p); struct __dynamic_view; // the left operand of a named `dynamic`'s `<<` and `->*`; defined after it template <> struct __sink_storage @@ -3966,7 +3994,7 @@ struct __sink_storage * claims another (identical or base-of) is rejected -- the claimed entry would be dead. */ template -auto if_isa(_P&& __p) +_CCCL_HOST_API auto if_isa(_P&& __p) { static_assert(sizeof...(_Es) > 0, "if_isa requires at least one exception type"); static_assert(detail::__if_isa_pack_ok<_Es...>, @@ -3990,7 +4018,7 @@ auto if_isa(_P&& __p) * rest of its cone; the reverse order starves the exact arm and is a compile error. */ template -auto if_monomorphic(_P&& __p) +_CCCL_HOST_API auto if_monomorphic(_P&& __p) { static_assert(sizeof...(_Es) > 0, "if_monomorphic requires at least one exception type"); static_assert((::cuda::std::is_base_of_v<::std::exception, _Es> && ...), @@ -4018,7 +4046,7 @@ auto if_monomorphic(_P&& __p) * @return A selector; calling it with a policy `p` yields the restricted policy. */ template -auto if_equal(const _Status __first, const _More... __more) +_CCCL_HOST_API auto if_equal(const _Status __first, const _More... __more) { static_assert((::cuda::std::is_same_v<_Status, _More> && ...), "if_equal(values...) requires all values to be of one status type"); @@ -4047,7 +4075,7 @@ auto if_equal(const _Status __first, const _More... __more) * @return The selector or converter policy. */ template -auto thrown(_X&& __x) +_CCCL_HOST_API auto thrown(_X&& __x) { if constexpr (detail::__is_policy_argument_v<_X>) { @@ -4082,7 +4110,7 @@ auto thrown(_X&& __x) * @return The selector or converter policy. */ template -auto returned(_X&& __x) +_CCCL_HOST_API auto returned(_X&& __x) { if constexpr (::cuda::std::is_void_v<_S>) { @@ -4110,7 +4138,7 @@ auto returned(_X&& __x) template || detail::__is_exception_sink_v<_R>, int> = 0> -[[nodiscard]] auto operator&(_L&& __l, _R&& __r) +[[nodiscard]] _CCCL_HOST_API auto operator&(_L&& __l, _R&& __r) { if constexpr (detail::__is_dynamic_v<_L>) { @@ -4122,7 +4150,7 @@ template , int> = 0> -[[nodiscard]] auto operator&(noop_t, _R&& __r) +[[nodiscard]] _CCCL_HOST_API auto operator&(noop_t, _R&& __r) { return detail::__normalize(::cuda::std::forward<_R>(__r)); } @@ -4132,7 +4160,7 @@ template , noop_t> && detail::__normalizes_to_exception_sink_v<_L>, int> = 0> -[[nodiscard]] auto operator&(_L&& __l, noop_t) +[[nodiscard]] _CCCL_HOST_API auto operator&(_L&& __l, noop_t) { using _NL = detail::__normalized_t<_L>; static_assert(!detail::__produces_value_v<_NL>, @@ -4155,7 +4183,7 @@ template || detail::__is_exception_sink_v<_R>, int> = 0> -[[nodiscard]] auto operator|(_L&& __l, _R&& __r) +[[nodiscard]] _CCCL_HOST_API auto operator|(_L&& __l, _R&& __r) { if constexpr (detail::__is_dynamic_v<_L>) { @@ -4167,7 +4195,7 @@ template , int> = 0> -[[nodiscard]] auto operator|(fwd_t, _R&& __r) +[[nodiscard]] _CCCL_HOST_API auto operator|(fwd_t, _R&& __r) { return detail::__normalize(::cuda::std::forward<_R>(__r)); } @@ -4177,7 +4205,7 @@ template , fwd_t> && detail::__normalizes_to_exception_sink_v<_L>, int> = 0> -[[nodiscard]] auto operator|(_L&& __l, fwd_t) +[[nodiscard]] _CCCL_HOST_API auto operator|(_L&& __l, fwd_t) { using _NL = detail::__normalized_t<_L>; static_assert(!detail::__never_passes_through_v<_NL>, @@ -4202,7 +4230,7 @@ template > || detail::__has_any_capability<::cuda::std::remove_cvref_t<_P>>, int> = 0> -[[nodiscard]] auto operator*(_P&& __p, int __n) +[[nodiscard]] _CCCL_HOST_API auto operator*(_P&& __p, int __n) { _CCCL_ASSERT(__n >= 0, "repetition requires a non-negative count"); using _Np = detail::__normalized_t<_P>; @@ -4218,7 +4246,7 @@ template > || detail::__has_any_capability<::cuda::std::remove_cvref_t<_P>>, int> = 0> -[[nodiscard]] auto operator*(int __n, _P&& __p) +[[nodiscard]] _CCCL_HOST_API auto operator*(int __n, _P&& __p) { return ::cuda::std::forward<_P>(__p) * __n; } @@ -4241,7 +4269,8 @@ struct always_t _B __fin_; template - decltype(auto) on_thrown(const ::std::exception* __e, const ::cuda::std::source_location __loc, _Fn& __fn) + _CCCL_HOST_API decltype(auto) + on_thrown(const ::std::exception* __e, const ::cuda::std::source_location __loc, _Fn& __fn) { _CCCL_TRY { @@ -4289,7 +4318,7 @@ struct always_t //! answers the head's answer. If the head throws, the finalizers still run and the head's //! exception continues onward. template - detail::__status_answer<_Status> + _CCCL_HOST_API detail::__status_answer<_Status> on_returned(const _Status __status, const ::cuda::std::source_location __loc, _Fn& __fn) noexcept( detail::__status_path_nothrow_v<_A, _Status, _Fn> && detail::__status_path_nothrow_v<_B, _Status, _Fn>) { @@ -4309,13 +4338,13 @@ struct always_t // The head's success hook is a type-preserving side-effect; forward it. template , int> = 0> - decltype(auto) on_success(_R&& __r) + _CCCL_HOST_API decltype(auto) on_success(_R&& __r) { return __head_.on_success(::cuda::std::forward<_R>(__r)); } template , int> = 0> - decltype(auto) on_success() + _CCCL_HOST_API decltype(auto) on_success() { return __head_.on_success(); } @@ -4324,7 +4353,7 @@ struct always_t template || detail::__has_on_enter<_BB>, int> = 0> - void on_enter() noexcept(detail::__on_enter_nothrow_v<_A> && detail::__on_enter_nothrow_v<_B>) + _CCCL_HOST_API void on_enter() noexcept(detail::__on_enter_nothrow_v<_A> && detail::__on_enter_nothrow_v<_B>) { if constexpr (detail::__has_on_enter<_A>) { @@ -4358,7 +4387,7 @@ inline constexpr bool __selects_returned_v> = //! @brief See @ref always_t. Variadic: `always(a, b, c)` folds left, so both `b` and `c` run //! regardless of `a`, and `c` runs regardless of `b`. template -auto always(_A&& __a, _B&& __b, _Cs&&... __cs) +_CCCL_HOST_API auto always(_A&& __a, _B&& __b, _Cs&&... __cs) { if constexpr (sizeof...(_Cs) == 0) { @@ -4450,7 +4479,7 @@ public: //! for hand-written models. const bool has_success_hook; - sink_base( + _CCCL_HOST_API sink_base( answer_kind __kind = answer_kind::own_result, bool __may_passthrough = true, long long __min_value = 0, @@ -4477,25 +4506,25 @@ public: , answer_name(__answer_name) , has_success_hook(__has_success_hook) {} - virtual ~sink_base() = default; - virtual sink_base* clone() const = 0; + _CCCL_HIDE_FROM_ABI virtual ~sink_base() = default; + virtual sink_base* clone() const = 0; virtual ::std::any hook(const ::std::exception*, ::cuda::std::source_location, ::std::function<::std::any()>&) = 0; - virtual ::std::any on_success(::std::any __boxed) + _CCCL_HOST_API virtual ::std::any on_success(::std::any __boxed) { return __boxed; } - virtual ::std::any on_success() + _CCCL_HOST_API virtual ::std::any on_success() { return {}; } //! @brief Entry gate, run before each attempt. Default: admit. - virtual void on_enter() {} + _CCCL_HOST_API virtual void on_enter() {} }; private: template - static constexpr answer_kind __kind_of() + _CCCL_HOST_API static constexpr answer_kind __kind_of() { using _T = ::cuda::std::remove_cvref_t<_Ans>; if constexpr (::cuda::std::is_void_v<_T>) @@ -4529,7 +4558,7 @@ private: } template - static constexpr long long __limits_min() + _CCCL_HOST_API static constexpr long long __limits_min() { if constexpr (::cuda::std::is_unsigned_v<_Int>) { @@ -4542,13 +4571,13 @@ private: } template - static constexpr unsigned long long __limits_max() + _CCCL_HOST_API static constexpr unsigned long long __limits_max() { return static_cast(::std::numeric_limits<_Int>::max()); } template - static constexpr long long __stored_min() + _CCCL_HOST_API static constexpr long long __stored_min() { using _T = ::cuda::std::remove_cvref_t<_Ans>; if constexpr (__kind_of<_T>() == answer_kind::integral) @@ -4569,7 +4598,7 @@ private: } template - static constexpr unsigned long long __stored_max() + _CCCL_HOST_API static constexpr unsigned long long __stored_max() { using _T = ::cuda::std::remove_cvref_t<_Ans>; if constexpr (__kind_of<_T>() == answer_kind::integral) @@ -4599,7 +4628,7 @@ private: using __ans_t _CCCL_NODEBUG = detail::__hook_answer_t<_P, ::std::function<::std::any()>>; static constexpr answer_kind __skind = __kind_of<__ans_t>(); - explicit __model(_P __p) + _CCCL_HOST_API explicit __model(_P __p) : sink_base( __skind, !detail::__never_passes_through_v<_P, ::std::function<::std::any()>>, @@ -4619,12 +4648,12 @@ private: , __p_(::cuda::std::move(__p)) {} - sink_base* clone() const override + _CCCL_HOST_API sink_base* clone() const override { return new __model(*this); // a fresh policy copy: re-armed state } - ::std::any + _CCCL_HOST_API ::std::any hook(const ::std::exception* __e, ::cuda::std::source_location __loc, ::std::function<::std::any()>& __fn) override { if constexpr (__skind == answer_kind::effects || __skind == answer_kind::resumes) @@ -4647,7 +4676,7 @@ private: } } - ::std::any on_success(::std::any __boxed) override + _CCCL_HOST_API ::std::any on_success(::std::any __boxed) override { if constexpr (detail::__has_on_success_with<_P, ::std::any>) { @@ -4661,7 +4690,7 @@ private: return __boxed; } } - ::std::any on_success() override + _CCCL_HOST_API ::std::any on_success() override { if constexpr (detail::__has_on_success_void<_P>) { @@ -4680,7 +4709,7 @@ private: return {}; } } - void on_enter() override + _CCCL_HOST_API void on_enter() override { if constexpr (detail::__has_on_enter<_P>) { @@ -4696,9 +4725,9 @@ private: friend struct detail::__dynamic_view; struct __hollow_t {}; - explicit dynamic(__hollow_t) noexcept {} + _CCCL_HOST_API explicit dynamic(__hollow_t) noexcept {} - [[noreturn]] void __throw_mismatch(::std::string_view __stored, ::std::string_view __wanted) const + [[noreturn]] _CCCL_HOST_API void __throw_mismatch(::std::string_view __stored, ::std::string_view __wanted) const { ::std::string __msg{"dynamic cannot convert "}; __msg.append(__stored.data(), __stored.size()); @@ -4708,7 +4737,7 @@ private: } template - [[nodiscard]] bool __range_contains_int() const + [[nodiscard]] _CCCL_HOST_API bool __range_contains_int() const { const auto __raw_max = ::std::numeric_limits<_Int>::max(); if constexpr (::cuda::std::is_unsigned_v<_Int>) @@ -4731,7 +4760,7 @@ private: } template - [[nodiscard]] bool __range_contains() const + [[nodiscard]] _CCCL_HOST_API bool __range_contains() const { using _T = ::cuda::std::remove_cvref_t<_Raw>; if constexpr (::cuda::std::is_enum_v<_T>) @@ -4745,7 +4774,7 @@ private: } template - void __check_compatible() const + _CCCL_HOST_API void __check_compatible() const { using _T = ::cuda::std::remove_cvref_t<_Raw>; const answer_kind __kind = __p_->kind; @@ -4795,7 +4824,7 @@ private: } template - [[nodiscard]] _Raw __unbox(const ::std::any& __box) const + [[nodiscard]] _CCCL_HOST_API _Raw __unbox(const ::std::any& __box) const { using _T = ::cuda::std::remove_cvref_t<_Raw>; if constexpr (::cuda::std::is_same_v<_T, ::std::any>) @@ -4916,23 +4945,23 @@ public: ::cuda::std::enable_if_t, int> = 0> // Constrained above; the check only recognizes std::enable_if, not cuda::std::enable_if_t. // NOLINTNEXTLINE(bugprone-forwarding-reference-overload) - dynamic(_P&& __p) + _CCCL_HOST_API dynamic(_P&& __p) : __p_(new __model>(detail::__normalize(::cuda::std::forward<_P>(__p)))) {} //! @brief Adopt a custom model derived from @ref sink_base. - explicit dynamic(::std::unique_ptr __custom) + _CCCL_HOST_API explicit dynamic(::std::unique_ptr __custom) : __p_(::std::move(__custom)) { _CCCL_ASSERT(__p_, "dynamic requires a non-null model"); } - dynamic(const dynamic& __other) + _CCCL_HOST_API dynamic(const dynamic& __other) : __p_(__other.__p_->clone()) {} - dynamic(dynamic&&) noexcept = default; - dynamic& operator=(dynamic&&) noexcept = default; - dynamic& operator=(const dynamic& __other) + _CCCL_HIDE_FROM_ABI dynamic(dynamic&&) noexcept = default; + _CCCL_HIDE_FROM_ABI dynamic& operator=(dynamic&&) noexcept = default; + _CCCL_HOST_API dynamic& operator=(const dynamic& __other) { if (this != &__other) { @@ -4941,48 +4970,48 @@ public: return *this; } - [[nodiscard]] answer_kind kind() const noexcept + [[nodiscard]] _CCCL_HOST_API answer_kind kind() const noexcept { return __p_->kind; } //! @brief Whether the erased policy may pass an error through (`false`: alternatives after //! this sink are unreachable). - [[nodiscard]] bool may_passthrough() const noexcept + [[nodiscard]] _CCCL_HOST_API bool may_passthrough() const noexcept { return __p_->may_passthrough; } //! @brief Whether the erased policy may throw. - [[nodiscard]] bool may_throw() const noexcept + [[nodiscard]] _CCCL_HOST_API bool may_throw() const noexcept { return __p_->may_throw; } //! @brief Whether the erased policy always diverges (unwind, abort, terminate). - [[nodiscard]] bool diverges() const noexcept + [[nodiscard]] _CCCL_HOST_API bool diverges() const noexcept { return __p_->diverges; } //! @brief Whether the erased policy's answer is a value standing in for the expression's. - [[nodiscard]] bool produces_value() const noexcept + [[nodiscard]] _CCCL_HOST_API bool produces_value() const noexcept { return __p_->produces_value; } //! @brief Whether the erased policy re-runs the action. - [[nodiscard]] bool needs_future() const noexcept + [[nodiscard]] _CCCL_HOST_API bool needs_future() const noexcept { return __p_->needs_future; } //! @brief Whether the erased policy selects exceptions by type. - [[nodiscard]] bool selects_thrown() const noexcept + [[nodiscard]] _CCCL_HOST_API bool selects_thrown() const noexcept { return __p_->selects_thrown; } //! @brief Whether the erased policy selects failing statuses. - [[nodiscard]] bool selects_returned() const noexcept + [[nodiscard]] _CCCL_HOST_API bool selects_returned() const noexcept { return __p_->selects_returned; } //! @brief Former name of @ref may_passthrough (deprecated). - CCCL_DEPRECATED_BECAUSE("use may_passthrough()") bool may_rethrow() const noexcept + CCCL_DEPRECATED_BECAUSE("use may_passthrough()") _CCCL_HOST_API bool may_rethrow() const noexcept { return may_passthrough(); } @@ -4990,7 +5019,7 @@ public: //! @brief The uniform hook: answers `decltype(fn())`. A void callable answers //! `decltype(::std::ignore)` so the presence-probe archetype stays admissible. template ()())> - auto on_thrown(const ::std::exception* __e, const ::cuda::std::source_location __loc, _Fn& __fn) + _CCCL_HOST_API auto on_thrown(const ::std::exception* __e, const ::cuda::std::source_location __loc, _Fn& __fn) -> ::cuda::std::conditional_t<::cuda::std::is_void_v<_Raw>, decltype(::std::ignore), _Raw> { static_assert(!::cuda::std::is_reference_v<_Raw>, @@ -5033,14 +5062,14 @@ public: } //! @brief Entry gate: forwards to the erased policy (no-op when it has none). - void on_enter() + _CCCL_HOST_API void on_enter() { __p_->on_enter(); } //! @brief Type-preserving success pass-through: box, delegate, unbox to the same type. template - _R on_success(_R&& __r) + _CCCL_HOST_API _R on_success(_R&& __r) { static_assert(!::cuda::std::is_reference_v<_R>, "dynamic cannot serve a reference-returning callable: std::any cannot " @@ -5059,7 +5088,7 @@ public: return __unbox<_R>(__p_->on_success(::std::any(::cuda::std::forward<_R>(__r)))); } } - void on_success() + _CCCL_HOST_API void on_success() { if (__p_->has_success_hook) { @@ -5079,19 +5108,19 @@ struct __dynamic_view { using __exception_sink_tag _CCCL_NODEBUG = void; - __dynamic_view(dynamic& __d) noexcept + _CCCL_HOST_API __dynamic_view(dynamic& __d) noexcept : __owned_(dynamic::__hollow_t{}) , __ref_(&__d) {} - __dynamic_view(dynamic&& __d) noexcept + _CCCL_HOST_API __dynamic_view(dynamic&& __d) noexcept : __owned_(::cuda::std::move(__d)) , __ref_(&__owned_) {} - __dynamic_view(const dynamic& __d) + _CCCL_HOST_API __dynamic_view(const dynamic& __d) : __owned_(__d) , __ref_(&__owned_) {} - __dynamic_view(__dynamic_view&& __other) noexcept + _CCCL_HOST_API __dynamic_view(__dynamic_view&& __other) noexcept : __owned_(::cuda::std::move(__other.__owned_)) , __ref_(__other.__ref_ == &__other.__owned_ ? &__owned_ : __other.__ref_) {} @@ -5100,21 +5129,22 @@ struct __dynamic_view __dynamic_view& operator=(__dynamic_view&&) = delete; template - _CCCL_FORCEINLINE auto on_thrown(const ::std::exception* __e, const ::cuda::std::source_location __loc, _Fn& __fn) + _CCCL_HOST_API _CCCL_FORCEINLINE auto + on_thrown(const ::std::exception* __e, const ::cuda::std::source_location __loc, _Fn& __fn) -> decltype(::cuda::std::declval().on_thrown(__e, __loc, __fn)) { return __ref_->on_thrown(__e, __loc, __fn); } - _CCCL_FORCEINLINE void on_enter() + _CCCL_HOST_API _CCCL_FORCEINLINE void on_enter() { __ref_->on_enter(); } template - _CCCL_FORCEINLINE _R on_success(_R&& __r) + _CCCL_HOST_API _CCCL_FORCEINLINE _R on_success(_R&& __r) { return __ref_->on_success(::cuda::std::forward<_R>(__r)); } - _CCCL_FORCEINLINE void on_success() + _CCCL_HOST_API _CCCL_FORCEINLINE void on_success() { __ref_->on_success(); } @@ -5124,7 +5154,7 @@ private: dynamic* __ref_; }; -inline void __check_dynamic_left_of_or(const dynamic& __p) +_CCCL_HOST_API inline void __check_dynamic_left_of_or(const dynamic& __p) { if (__p.diverges()) { @@ -5137,7 +5167,7 @@ inline void __check_dynamic_left_of_or(const dynamic& __p) } } -inline void __check_dynamic_left_of_and(const dynamic& __p) +_CCCL_HOST_API inline void __check_dynamic_left_of_and(const dynamic& __p) { // The static checks of `&`, against the metadata recorded from the same traits at erasure. if (__p.kind() == dynamic::answer_kind::dies && !__p.selects_returned()) @@ -5152,7 +5182,7 @@ inline void __check_dynamic_left_of_and(const dynamic& __p) } } -inline void __check_dynamic_repeated(const dynamic& __p) +_CCCL_HOST_API inline void __check_dynamic_repeated(const dynamic& __p) { if (__p.diverges()) { @@ -5184,7 +5214,7 @@ namespace sump::detail // from the sink and the call site, then apply the carrier's operator. Found by ADL through // `__sink_at`. template -auto __sink_shift(__sink_at<_Sink>&& __s, _X&& __x) noexcept( +_CCCL_HOST_API auto __sink_shift(__sink_at<_Sink>&& __s, _X&& __x) noexcept( ::cuda::std::is_nothrow_move_constructible_v<_Sink> && noexcept(::cuda::std::declval<__on_throw_policy<__sink_storage_t<_Sink>>>() << ::cuda::std::declval<_X>())) { @@ -5193,7 +5223,7 @@ auto __sink_shift(__sink_at<_Sink>&& __s, _X&& __x) noexcept( } template -decltype(auto) __sink_yield(__sink_at<_Sink>&& __s, _X&& __x) noexcept( +_CCCL_HOST_API decltype(auto) __sink_yield(__sink_at<_Sink>&& __s, _X&& __x) noexcept( ::cuda::std::is_nothrow_move_constructible_v<_Sink> && noexcept(::cuda::std::declval<__on_throw_policy<__sink_storage_t<_Sink>>>()->*::cuda::std::declval<_X>())) { @@ -5312,8 +5342,8 @@ namespace sump * `status()` and `forwarded()`. */ template -auto sink(_Reaction&& __reaction, - const ::cuda::std::source_location __loc = ::cuda::std::source_location::current()) noexcept +_CCCL_HOST_API auto sink(_Reaction&& __reaction, + const ::cuda::std::source_location __loc = ::cuda::std::source_location::current()) noexcept { return detail::__on_throw_policy{detail::__normalize(::cuda::std::forward<_Reaction>(__reaction)), __loc}; } @@ -5368,7 +5398,7 @@ auto sink(_Reaction&& __reaction, * `using cuda::experimental::stf::operator|=;`. */ template , int> = 0> -::std::exception_ptr& operator|=(::std::exception_ptr& __first, _Fn&& __step) noexcept +_CCCL_HOST_API ::std::exception_ptr& operator|=(::std::exception_ptr& __first, _Fn&& __step) noexcept { ::std::exception_ptr __e = sump::defer->*[&]() -> ::std::exception_ptr { __step(); @@ -5820,7 +5850,7 @@ UNITTEST("error sinks") struct movable { int v; - explicit movable(int value_) + _CCCL_HOST explicit movable(int value_) : v(value_) {} movable(const movable&) = delete; @@ -6105,13 +6135,13 @@ UNITTEST("alternation and repetition nested under & and |") # if _CCCL_HAS_EXCEPTIONS() struct key : ::std::logic_error { - key() + _CCCL_HOST key() : logic_error("key") {} }; struct boom : ::std::runtime_error { - boom() + _CCCL_HOST boom() : runtime_error("boom") {} }; @@ -6204,13 +6234,13 @@ UNITTEST("selecting arms decline by return") using sump::detail::__has_pure_claim; struct key : ::std::logic_error { - key() + _CCCL_HOST key() : logic_error("key") {} }; struct boom : ::std::runtime_error { - boom() + _CCCL_HOST boom() : runtime_error("boom") {} }; @@ -6647,7 +6677,7 @@ struct __ut_error_from_exception { int marker; - explicit __ut_error_from_exception(const ::std::exception& __exception) + _CCCL_HOST_API explicit __ut_error_from_exception(const ::std::exception& __exception) : marker(__exception.what()[0]) {} }; @@ -7349,7 +7379,7 @@ UNITTEST("store") { struct virt_error : virtual ::std::exception { - explicit virt_error(int c) + _CCCL_HOST explicit virt_error(int c) : code(c) {} int code; @@ -7385,7 +7415,7 @@ UNITTEST("store") // within the UNITTEST initializer ("a variable declared with an auto type specifier cannot appear in // its own initializer"). template -bool __erasure_rejects(_Build&& __build) +_CCCL_HOST_API bool __erasure_rejects(_Build&& __build) { // Set only in the catch clause, which the device pass discards. bool __thrown = false; // NOLINT(misc-const-correctness) @@ -7657,14 +7687,15 @@ UNITTEST("type erasure") { struct halving_sink final : dynamic::sink_base { - halving_sink() + _CCCL_HOST halving_sink() : sink_base(dynamic::answer_kind::own_result, false) {} - sink_base* clone() const override + _CCCL_HOST sink_base* clone() const override { return new halving_sink(); } - ::std::any hook(const ::std::exception*, ::cuda::std::source_location, ::std::function<::std::any()>&) override + _CCCL_HOST ::std::any + hook(const ::std::exception*, ::cuda::std::source_location, ::std::function<::std::any()>&) override { return ::std::any(21); } @@ -7759,15 +7790,16 @@ UNITTEST("a named dynamic is viewed by ->* and <<, not cloned") static int clones = 0; struct counting_sink final : dynamic::sink_base { - counting_sink() + _CCCL_HOST counting_sink() : sink_base(dynamic::answer_kind::own_result, false) {} - sink_base* clone() const override + _CCCL_HOST sink_base* clone() const override { ++clones; return new counting_sink(); } - ::std::any hook(const ::std::exception*, ::cuda::std::source_location, ::std::function<::std::any()>&) override + _CCCL_HOST ::std::any + hook(const ::std::exception*, ::cuda::std::source_location, ::std::function<::std::any()>&) override { return ::std::any(7); } @@ -7804,11 +7836,11 @@ struct __ut_counting_success { static inline int seen = 0; template - ::std::any on_thrown(const ::std::exception*, ::cuda::std::source_location, _Fn&) + _CCCL_HOST_API ::std::any on_thrown(const ::std::exception*, ::cuda::std::source_location, _Fn&) { return {}; } - ::std::any on_success(::std::any __r) + _CCCL_HOST_API ::std::any on_success(::std::any __r) { ++seen; return __r; @@ -7824,15 +7856,15 @@ UNITTEST("a dynamic over a policy without a success hook passes the value unboxe struct tracked { int v; - tracked(int __v) noexcept + _CCCL_HOST tracked(int __v) noexcept : v(__v) {} - tracked(const tracked& __o) noexcept + _CCCL_HOST tracked(const tracked& __o) noexcept : v(__o.v) { ++copies; } - tracked(tracked&& __o) noexcept + _CCCL_HOST tracked(tracked&& __o) noexcept : v(__o.v) {} }; @@ -7933,7 +7965,7 @@ enum class success // Calls the guard body, passing the outcome when `SCOPE(exit, name)` asked for it and nothing // when it did not. Whether a throw from here escapes is the caller's decision, not this one's. template -void invoke_body(F& f, bool failing) +_CCCL_HOST_API void invoke_body(F& f, bool failing) { if constexpr (::cuda::std::is_invocable_v) { @@ -7951,7 +7983,7 @@ void invoke_body(F& f, bool failing) // Runs the body under `abort`: a throw is reported (message and location) and the // program aborts, rather than escaping a noexcept destructor as a bare std::terminate. template -void invoke_nothrow(F& f, ::cuda::std::source_location loc, bool failing = false) noexcept +_CCCL_HOST_API void invoke_nothrow(F& f, ::cuda::std::source_location loc, bool failing = false) noexcept { // The body may throw; that is what the abort policy is for. In clang-tidy's device pass the policy's // catch is erased, so the check sees the throw escape this noexcept function. @@ -7962,8 +7994,9 @@ void invoke_nothrow(F& f, ::cuda::std::source_location loc, bool failing = false } template -auto operator->*(with_location where, F&& f) +_CCCL_HOST_API auto operator->*(with_location where, F&& f) { + // Local class members have no linkage, so visibility attributes do not apply. struct result { F f; @@ -7973,12 +8006,13 @@ auto operator->*(with_location where, F&& f) int exceptions = ::std::uncaught_exceptions(); // F can be a reference, so preserve its value category. - result(F&& f, ::cuda::std::source_location loc) // NOLINT(cppcoreguidelines-rvalue-reference-param-not-moved) + _CCCL_HOST result(F&& f, + ::cuda::std::source_location loc) // NOLINT(cppcoreguidelines-rvalue-reference-param-not-moved) : f(::cuda::std::forward(f)) , loc(loc) {} result(result&) = delete; - result(result&& rhs) noexcept(::cuda::std::is_nothrow_move_constructible_v) + _CCCL_HOST result(result&& rhs) noexcept(::cuda::std::is_nothrow_move_constructible_v) : f(mv(rhs.f)) , loc(rhs.loc) , exceptions(::cuda::std::exchange(rhs.exceptions, -1)) @@ -7986,7 +8020,7 @@ auto operator->*(with_location where, F&& f) // May throw: see below. Not `noexcept`, so a body that fails on the normal path reports // the failure to the caller instead of ending the program. - ~result() noexcept(false) + _CCCL_HOST ~result() noexcept(false) { if (exceptions == -1) { @@ -8013,12 +8047,13 @@ auto operator->*(with_location where, F&& f) } template -auto operator->*(with_location where, F&& f) +_CCCL_HOST_API auto operator->*(with_location where, F&& f) { static_assert(::cuda::std::is_invocable_v, "SCOPE(fail) runs only when the scope is left by an exception, so its outcome is " "already known and it takes no parameter. Write SCOPE(fail) { ... }, or use " "SCOPE(exit, name) to branch on the outcome."); + // Local class members have no linkage, so visibility attributes do not apply. struct result { F f; @@ -8027,21 +8062,21 @@ auto operator->*(with_location where, F&& f) int exceptions; // F can be a reference, so preserve its value category. - result(F&& f, // NOLINT(cppcoreguidelines-rvalue-reference-param-not-moved) - ::cuda::std::source_location loc, - int exceptions) + _CCCL_HOST result(F&& f, // NOLINT(cppcoreguidelines-rvalue-reference-param-not-moved) + ::cuda::std::source_location loc, + int exceptions) : f(::cuda::std::forward(f)) , loc(loc) , exceptions(exceptions) {} result(result&) = delete; - result(result&& rhs) noexcept(::cuda::std::is_nothrow_move_constructible_v) + _CCCL_HOST result(result&& rhs) noexcept(::cuda::std::is_nothrow_move_constructible_v) : f(mv(rhs.f)) , loc(rhs.loc) , exceptions(::cuda::std::exchange(rhs.exceptions, -1)) {} - ~result() noexcept + _CCCL_HOST ~result() noexcept { if (::std::uncaught_exceptions() == exceptions) { @@ -8055,7 +8090,7 @@ auto operator->*(with_location where, F&& f) } template -auto operator->*(success, F&& f) +_CCCL_HOST_API auto operator->*(success, F&& f) { static_assert(::cuda::std::is_invocable_v, "SCOPE(success) runs only when the scope is left normally, so its outcome is " @@ -8065,6 +8100,7 @@ auto operator->*(success, F&& f) static_assert(::cuda::std::is_void_v(f)())>, "SCOPE requires a void-returning callable"); + // Local class members have no linkage, so visibility attributes do not apply. struct result { F f; @@ -8072,18 +8108,18 @@ auto operator->*(success, F&& f) int exceptions; // F can be a reference, so preserve its value category. - result(F&& f, int exceptions) // NOLINT(cppcoreguidelines-rvalue-reference-param-not-moved) + _CCCL_HOST result(F&& f, int exceptions) // NOLINT(cppcoreguidelines-rvalue-reference-param-not-moved) : f(::cuda::std::forward(f)) , exceptions(exceptions) {} result(result&) = delete; - result(result&& rhs) noexcept(::cuda::std::is_nothrow_move_constructible_v) + _CCCL_HOST result(result&& rhs) noexcept(::cuda::std::is_nothrow_move_constructible_v) : f(mv(rhs.f)) , exceptions(::cuda::std::exchange(rhs.exceptions, -1)) {} // May throw — unlike exit/fail. - ~result() noexcept(false) + _CCCL_HOST ~result() noexcept(false) { if (::std::uncaught_exceptions() == exceptions) { @@ -8115,7 +8151,7 @@ struct spy int& hits; _CUDAX_STF_SINK(spy) template - void on_thrown(const ::std::exception*, ::cuda::std::source_location, Fn&) noexcept + _CCCL_HOST_API void on_thrown(const ::std::exception*, ::cuda::std::source_location, Fn&) noexcept { ++hits; } @@ -8235,7 +8271,7 @@ struct picky { _CUDAX_STF_SINK(picky) template - [[noreturn]] cuda::experimental::stf::nullval + [[noreturn]] _CCCL_HOST_API cuda::experimental::stf::nullval on_thrown(const ::std::exception*, ::cuda::std::source_location, Fn&) const { _CCCL_RETHROW; @@ -8252,7 +8288,7 @@ struct succeed_on { _CUDAX_STF_SINK(succeed_on) template - cudaError_t on_thrown(const ::std::exception* e, ::cuda::std::source_location, Fn&) const + _CCCL_HOST_API cudaError_t on_thrown(const ::std::exception* e, ::cuda::std::source_location, Fn&) const { if (dynamic_cast(e) != nullptr) { @@ -8266,7 +8302,7 @@ struct converter { _CUDAX_STF_SINK(converter) template - [[noreturn]] cuda::experimental::stf::nullval + [[noreturn]] _CCCL_HOST_API cuda::experimental::stf::nullval on_thrown(const ::std::exception* e, ::cuda::std::source_location, Fn&) const { throw my_error(e ? e->what() : "?"); diff --git a/cudax/include/cuda/experimental/__stf/utility/handle.cuh b/cudax/include/cuda/experimental/__stf/utility/handle.cuh index 539c98e7ebec..dfaa1b8b162f 100644 --- a/cudax/include/cuda/experimental/__stf/utility/handle.cuh +++ b/cudax/include/cuda/experimental/__stf/utility/handle.cuh @@ -46,14 +46,14 @@ inline enum class use_dynamic_cast {} use_dynamic_cast = {}; /// @brief Performs bitwise OR operation on `handle_flags`. /// @return The result of bitwise OR operation. -constexpr handle_flags operator|(handle_flags a, handle_flags b) +_CCCL_HOST_API constexpr handle_flags operator|(handle_flags a, handle_flags b) { return handle_flags(static_cast(a) | static_cast(b)); } /// @brief Performs bitwise AND operation on `handle_flags`. /// @return The result of bitwise AND operation. -constexpr handle_flags operator&(handle_flags a, handle_flags b) +_CCCL_HOST_API constexpr handle_flags operator&(handle_flags a, handle_flags b) { return handle_flags(static_cast(a) & static_cast(b)); } @@ -79,15 +79,15 @@ class handle public: /// @name Defaulted constructors and assignment operators /// @{ - handle(handle&) = default; - handle(const handle&) = default; - handle(handle&&) = default; - handle& operator=(const handle&) = default; - handle& operator=(handle&&) = default; + _CCCL_HIDE_FROM_ABI handle(handle&) = default; + _CCCL_HIDE_FROM_ABI handle(const handle&) = default; + _CCCL_HIDE_FROM_ABI handle(handle&&) = default; + _CCCL_HIDE_FROM_ABI handle& operator=(const handle&) = default; + _CCCL_HIDE_FROM_ABI handle& operator=(handle&&) = default; /// @} /// @brief Default constructor. - handle() + _CCCL_HOST_API handle() { static_assert(!::cuda::std::is_constructible_v, "T's default constructor must be protected."); if constexpr ((f & handle_flags::non_null) != handle_flags::defaults) @@ -100,7 +100,7 @@ public: /// @brief Constructs a handle from another handle. template - handle(handle rhs) + _CCCL_HOST_API handle(handle rhs) : impl(mv(rhs.impl)) { if constexpr ((f & handle_flags::non_null) != handle_flags::defaults) @@ -112,7 +112,7 @@ public: /// @brief Variadic template constructor for creating a handle. template - handle(Args&&... args) + _CCCL_HOST_API handle(Args&&... args) : impl(make(::cuda::std::forward(args)...)) { static_assert(!::cuda::std::is_constructible_v, "T's constructors must be protected."); @@ -120,13 +120,13 @@ public: /// @brief Constructs a handle from another handle with static_cast. template - handle(handle& src, decltype(use_static_cast)) + _CCCL_HOST_API handle(handle& src, decltype(use_static_cast)) : handle(const_cast&>(src), use_static_cast) {} /// @brief Similar constructor as above but for const handle. template - handle(const handle& src, decltype(use_static_cast)) + _CCCL_HOST_API handle(const handle& src, decltype(use_static_cast)) : handle(::std::static_pointer_cast(src.impl)) { if constexpr ((f & handle_flags::non_null) != handle_flags::defaults) @@ -138,13 +138,13 @@ public: /// @brief Constructs a handle from another handle with dynamic_cast. template - handle(handle& src, decltype(use_dynamic_cast)) + _CCCL_HOST_API handle(handle& src, decltype(use_dynamic_cast)) : handle(const_cast&>(src), use_dynamic_cast) {} /// @brief Similar constructor as above but for const handle. template - handle(const handle& src, decltype(use_dynamic_cast)) + _CCCL_HOST_API handle(const handle& src, decltype(use_dynamic_cast)) : handle(::std::dynamic_pointer_cast(src.impl)) { if constexpr ((f & handle_flags::non_null) != handle_flags::defaults) @@ -156,13 +156,13 @@ public: /// @brief Constructs a handle from a shared_ptr with dynamic_cast. template - handle(const ::std::shared_ptr& src, decltype(use_dynamic_cast)) + _CCCL_HOST_API handle(const ::std::shared_ptr& src, decltype(use_dynamic_cast)) : handle(::std::dynamic_pointer_cast(src)) {} /// @brief Assignment operator for assigning one handle to another. template - handle& operator=(handle rhs) + _CCCL_HOST_API handle& operator=(handle rhs) { if constexpr ((f & handle_flags::non_null) != handle_flags::defaults) { @@ -173,43 +173,43 @@ public: return *this; } - ~handle() = default; + _CCCL_HIDE_FROM_ABI ~handle() = default; /// @brief Dereference operator. - T* operator->() const + _CCCL_HOST_API T* operator->() const { assert(*this); return impl.get(); } /// @brief Dereference the handle to get the actual object. - T& operator*() + _CCCL_HOST_API T& operator*() { return *operator->(); } /// @brief Const version of the dereference operator. - const T& operator*() const + _CCCL_HOST_API const T& operator*() const { return *operator->(); } /// @brief Conversion operator to bool. - explicit operator bool() const + _CCCL_HOST_API explicit operator bool() const { return impl.get() != nullptr; } /// @brief Conversion operator to shared_ptr. template - operator ::std::shared_ptr() const + _CCCL_HOST_API operator ::std::shared_ptr() const { return impl; } /// @brief Equality operator. template - bool operator==(const handle& rhs) + _CCCL_HOST_API bool operator==(const handle& rhs) { return impl == rhs.impl; } @@ -218,14 +218,14 @@ public: using weak_t = ::std::weak_ptr; /// @brief Returns a weak pointer to the underlying object. - weak_t weak() const + _CCCL_HOST_API weak_t weak() const { return impl; } /// @brief If weak_ptr is null, does nothing and returns false. Otherwise, calls the lambda and returns true. template - static bool if_valid(const weak_t& wp, Fun&& fun) + _CCCL_HOST_API static bool if_valid(const weak_t& wp, Fun&& fun) { if (auto p = wp.lock()) { @@ -246,13 +246,13 @@ private: struct Derived : public U { template - Derived(Args&&... args) + _CCCL_HOST_API Derived(Args&&... args) : U(::cuda::std::forward(args)...) {} }; template - static auto make(Arg&& arg, Args&&... args) + _CCCL_HOST_API static auto make(Arg&& arg, Args&&... args) { if constexpr (sizeof...(args) == 0 && ::cuda::std::is_convertible_v>) { diff --git a/cudax/include/cuda/experimental/__stf/utility/hash.cuh b/cudax/include/cuda/experimental/__stf/utility/hash.cuh index 071cfaa7bb99..d690b84c64da 100644 --- a/cudax/include/cuda/experimental/__stf/utility/hash.cuh +++ b/cudax/include/cuda/experimental/__stf/utility/hash.cuh @@ -82,7 +82,7 @@ inline constexpr bool has_std_hash_v = has_std_hash::value; * Taken from WG21 P0814R0 */ template -void hash_combine(size_t& seed, const T& val) +_CCCL_HOST_API void hash_combine(size_t& seed, const T& val) { if constexpr (reserved::has_std_hash_v) { @@ -117,7 +117,7 @@ void hash_combine(size_t& seed, const T& val) //! \param[in] vals The values to hash and combine. //! \return The combined hash value as a `size_t`. template -size_t hash_all(const Ts&... vals) +_CCCL_HOST_API size_t hash_all(const Ts&... vals) { if constexpr (sizeof...(Ts) == 1) { @@ -165,7 +165,7 @@ struct hash<::std::pair> * @param p The pair to hash. * @return size_t The hash value of the tuple. */ - size_t operator()(const ::std::pair& p) const + _CCCL_HOST_API size_t operator()(const ::std::pair& p) const { return cuda::experimental::stf::hash_all(p.first, p.second); } @@ -191,7 +191,7 @@ struct hash<::std::tuple> * @param p The tuple to hash. * @return size_t The hash value of the tuple. */ - size_t operator()(const ::std::tuple& p) const + _CCCL_HOST_API size_t operator()(const ::std::tuple& p) const { return ::std::apply(cuda::experimental::stf::hash_all, p); } diff --git a/cudax/include/cuda/experimental/__stf/utility/memory.cuh b/cudax/include/cuda/experimental/__stf/utility/memory.cuh index afccfdb01536..c36294a921db 100644 --- a/cudax/include/cuda/experimental/__stf/utility/memory.cuh +++ b/cudax/include/cuda/experimental/__stf/utility/memory.cuh @@ -41,14 +41,14 @@ namespace cuda::experimental::stf namespace reserved { // host-allocated pool, modeled as a hashtable keyed by size -inline auto& host_pool() +_CCCL_HOST_API inline auto& host_pool() { static ::std::unordered_multimap result; return result; } // managed memory pool, modeled as a hashtable keyed by size -inline auto& managed_pool() +_CCCL_HOST_API inline auto& managed_pool() { static ::std::unordered_multimap result; return result; @@ -64,7 +64,7 @@ inline constexpr size_t maxPoolEntries = 16 * 1024; * @param sz Number of bytes to allocate * @return void* pointer to allocated data */ -inline void* allocateHostMemory(size_t sz) +_CCCL_HOST_API inline void* allocateHostMemory(size_t sz) { auto& pool = reserved::host_pool(); if (auto i = pool.find(sz); i != pool.end()) @@ -104,7 +104,7 @@ inline void* allocateHostMemory(size_t sz) * @param sz Number of bytes to allocate * @return void* pointer to allocated data */ -inline void* allocateManagedMemory(size_t sz) +_CCCL_HOST_API inline void* allocateManagedMemory(size_t sz) { auto& pool = reserved::managed_pool(); @@ -143,7 +143,7 @@ inline void* allocateManagedMemory(size_t sz) * @param sz size in bytes * @param loc location of the call, defaulted */ -inline void deallocateHostMemory( +_CCCL_HOST_API inline void deallocateHostMemory( void* p, size_t sz, const ::cuda::std::source_location loc = ::cuda::std::source_location::current()) noexcept { ::std::ignore = loc; @@ -186,7 +186,7 @@ inline void deallocateHostMemory( * @param sz size in bytes * @param loc location of the call, defaulted */ -inline void deallocateManagedMemory( +_CCCL_HOST_API inline void deallocateManagedMemory( void* p, size_t sz, const ::cuda::std::source_location loc = ::cuda::std::source_location::current()) noexcept { ::std::ignore = loc; @@ -230,7 +230,7 @@ inline void deallocateManagedMemory( * @param sz size in bytes * @param stream the stream used for ordering */ -inline void deallocateHostMemory(void* p, size_t sz, cudaStream_t stream) +_CCCL_HOST_API inline void deallocateHostMemory(void* p, size_t sz, cudaStream_t stream) { SCOPE(fail) { @@ -268,7 +268,7 @@ inline void deallocateHostMemory(void* p, size_t sz, cudaStream_t stream) * @param sz size in bytes * @param stream the stream used for ordering */ -inline void deallocateManagedMemory(void* p, size_t sz, cudaStream_t stream) +_CCCL_HOST_API inline void deallocateManagedMemory(void* p, size_t sz, cudaStream_t stream) { SCOPE(fail) { @@ -305,7 +305,7 @@ inline void deallocateManagedMemory(void* p, size_t sz, cudaStream_t stream) * @param numDependencies number of elements in `pDependencies` * @return cudaGraphNode_t the newly inserted node */ -inline cudaGraphNode_t deallocateHostMemory( +_CCCL_HOST_API inline cudaGraphNode_t deallocateHostMemory( void* p, size_t sz, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies) { // Own the heap pair until the graph-node add succeeds; release to the @@ -339,7 +339,7 @@ inline cudaGraphNode_t deallocateHostMemory( * @return bool value indicating if p corresponds to a piece of memory that is already pinned or not. */ template -bool address_is_pinned(T* p) +_CCCL_HOST_API bool address_is_pinned(T* p) { const auto attr = cuda_try(p); EXPECT(attr.type != cudaMemoryTypeDevice); @@ -354,7 +354,7 @@ bool address_is_pinned(T* p) * @param n number of elements in the block */ template -cudaError_t pin_memory(T* p, size_t n) +_CCCL_HOST_API cudaError_t pin_memory(T* p, size_t n) { assert(p); if (address_is_pinned(p)) @@ -384,7 +384,7 @@ cudaError_t pin_memory(T* p, size_t n) * @param p pointer to beginning of memory block */ template -void unpin_memory(T* p) +_CCCL_HOST_API void unpin_memory(T* p) { assert(p); @@ -405,7 +405,7 @@ void unpin_memory(T* p) * @brief Pins arrays in host memory */ template -cudaError_t pin_memory(T (&array)[N]) +_CCCL_HOST_API cudaError_t pin_memory(T (&array)[N]) { return pin_memory(array, N); } @@ -414,7 +414,7 @@ cudaError_t pin_memory(T (&array)[N]) * @brief Pins vectors in host memory */ template -cudaError_t pin_memory(::std::vector& v) +_CCCL_HOST_API cudaError_t pin_memory(::std::vector& v) { return pin_memory(v.data(), v.size()); } @@ -423,7 +423,7 @@ cudaError_t pin_memory(::std::vector& v) * @brief Unpin vectors in host memory */ template -void unpin_memory(::std::vector& v) +_CCCL_HOST_API void unpin_memory(::std::vector& v) { unpin_memory(v.data()); } @@ -502,9 +502,9 @@ public: using const_reverse_iterator = ::std::reverse_iterator; // Constructors - small_vector() = default; + _CCCL_HIDE_FROM_ABI small_vector() = default; - small_vector(const small_vector& rhs) + _CCCL_HOST_API small_vector(const small_vector& rhs) { if (rhs.size() <= small_cap) { @@ -525,7 +525,7 @@ public: } } - small_vector(small_vector&& rhs) noexcept + _CCCL_HOST_API small_vector(small_vector&& rhs) noexcept { if (rhs.size() <= small_cap) { @@ -548,7 +548,7 @@ public: } } - small_vector(::std::initializer_list init) + _CCCL_HOST_API small_vector(::std::initializer_list init) { if (init.size() <= small_cap) { @@ -570,7 +570,7 @@ public: } // Assignment operators - small_vector& operator=(const small_vector& rhs) + _CCCL_HOST_API small_vector& operator=(const small_vector& rhs) { if (this == &rhs) { @@ -599,7 +599,7 @@ public: return *this; } - small_vector& operator=(small_vector&& rhs) noexcept + _CCCL_HOST_API small_vector& operator=(small_vector&& rhs) noexcept { if (this == &rhs) { @@ -644,7 +644,7 @@ public: small_vector& operator=(::std::initializer_list ilist); - ~small_vector() + _CCCL_HOST_API ~small_vector() { if (is_small()) { @@ -665,7 +665,7 @@ public: } // Element access - T& operator[](size_t pos) + _CCCL_HOST_API T& operator[](size_t pos) { if (is_small()) { @@ -676,7 +676,7 @@ public: return big()[pos]; } - const T& operator[](size_t pos) const + _CCCL_HOST_API const T& operator[](size_t pos) const { if (is_small()) { @@ -689,11 +689,11 @@ public: T& at(size_t pos); const T& at(size_t pos) const; - T& front() + _CCCL_HOST_API T& front() { return *begin(); } - const T& front() const + _CCCL_HOST_API const T& front() const { return *begin(); } @@ -704,30 +704,30 @@ public: const T* data() const noexcept; // Iterators - iterator begin() noexcept + _CCCL_HOST_API iterator begin() noexcept { return is_small() ? small_begin() : big().data(); } - const_iterator begin() const noexcept + _CCCL_HOST_API const_iterator begin() const noexcept { return is_small() ? small_begin() : big().data(); } - const_iterator cbegin() const noexcept + _CCCL_HOST_API const_iterator cbegin() const noexcept { return is_small() ? small_begin() : big().data(); } - iterator end() noexcept + _CCCL_HOST_API iterator end() noexcept { return begin() + size(); } - const_iterator end() const noexcept + _CCCL_HOST_API const_iterator end() const noexcept { return begin() + size(); } - const_iterator cend() const noexcept + _CCCL_HOST_API const_iterator cend() const noexcept { return begin() + size(); } @@ -740,22 +740,22 @@ public: const_reverse_iterator crend() const noexcept; // Capacity - bool empty() const noexcept + _CCCL_HOST_API bool empty() const noexcept { return is_small() ? small_length == 0 : big().empty(); } - size_t size() const noexcept + _CCCL_HOST_API size_t size() const noexcept { return is_small() ? small_length : big().size(); } - size_t max_size() const noexcept + _CCCL_HOST_API size_t max_size() const noexcept { return big().max_size(); } - void reserve(size_t new_cap) + _CCCL_HOST_API void reserve(size_t new_cap) { if (is_small()) { @@ -786,14 +786,14 @@ public: big().reserve(new_cap); } - size_t capacity() const noexcept + _CCCL_HOST_API size_t capacity() const noexcept { return is_small() ? small_cap : big().capacity(); } void shrink_to_fit(); // Modifiers - void clear() noexcept + _CCCL_HOST_API void clear() noexcept { if (is_small()) { @@ -818,7 +818,7 @@ public: iterator insert(const_iterator pos, size_t count, const T& value); template - iterator insert(iterator pos, InputIt first, InputIt last) + _CCCL_HOST_API iterator insert(iterator pos, InputIt first, InputIt last) { if (is_small()) { @@ -873,7 +873,7 @@ public: iterator emplace(const_iterator pos, Args&&... args); iterator erase(const_iterator pos); - iterator erase(iterator first, iterator last) + _CCCL_HOST_API iterator erase(iterator first, iterator last) { if (is_small()) { @@ -891,12 +891,12 @@ public: return big().data() + (result - big().begin()); } - void push_back(const T& value) + _CCCL_HOST_API void push_back(const T& value) { push_back(T(value)); // force an rvalue } - void push_back(T&& value) + _CCCL_HOST_API void push_back(T&& value) { if (is_small()) { @@ -916,7 +916,7 @@ public: template void emplace_back(Args&&... args); - void pop_back() + _CCCL_HOST_API void pop_back() { if (is_small()) { @@ -930,7 +930,7 @@ public: } } - void resize(size_t new_size) + _CCCL_HOST_API void resize(size_t new_size) { if (is_small()) { @@ -949,7 +949,7 @@ public: } } - void resize(size_t new_size, const value_type& value) + _CCCL_HOST_API void resize(size_t new_size, const value_type& value) { if (is_small()) { @@ -995,7 +995,7 @@ public: } } - void swap(small_vector& other) noexcept + _CCCL_HOST_API void swap(small_vector& other) noexcept { if (is_small()) { @@ -1042,7 +1042,7 @@ public: } // Non-member functions - friend bool operator==(const small_vector& lhs, const small_vector& rhs) + _CCCL_HOST_API friend bool operator==(const small_vector& lhs, const small_vector& rhs) { return lhs.size() == rhs.size() && ::std::equal(lhs.begin(), lhs.end(), rhs.begin()); } @@ -1054,32 +1054,32 @@ public: // friend void swap(small_vector& lhs, small_vector& rhs) noexcept; private: - auto small_begin() + _CCCL_HOST_API auto small_begin() { return reinterpret_cast(&small_); } - auto small_begin() const + _CCCL_HOST_API auto small_begin() const { return reinterpret_cast(&small_); } - auto& big() + _CCCL_HOST_API auto& big() { return *reinterpret_cast<::std::vector*>(&big_); } - auto& big() const + _CCCL_HOST_API auto& big() const { return *reinterpret_cast*>(&big_); } - bool is_small() const + _CCCL_HOST_API bool is_small() const { return small_length <= small_cap; } - void shrink_small(size_t delta) + _CCCL_HOST_API void shrink_small(size_t delta) { assert(delta <= small_length); assert(is_small()); @@ -1098,7 +1098,7 @@ private: // Return false from @p construct_one to stop. On success sets small_length; on // throw destroys any elements constructed so far and rethrows. template - void construct_small_elements(F&& construct_one) + _CCCL_HOST_API void construct_small_elements(F&& construct_one) { small_size_t n = 0; SCOPE(fail) @@ -1115,14 +1115,14 @@ private: small_length = n; } - void adopt_big_vector(::std::vector&& vec) + _CCCL_HOST_API void adopt_big_vector(::std::vector&& vec) { new (&big())::std::vector(::cuda::std::move(vec)); small_length = small_size_t(-1); } template - void loop(F&& f) + _CCCL_HOST_API void loop(F&& f) { if constexpr (small_cap < 16) { diff --git a/cudax/include/cuda/experimental/__stf/utility/source_location.cuh b/cudax/include/cuda/experimental/__stf/utility/source_location.cuh index d7ef72479234..1fc314e301bd 100644 --- a/cudax/include/cuda/experimental/__stf/utility/source_location.cuh +++ b/cudax/include/cuda/experimental/__stf/utility/source_location.cuh @@ -55,7 +55,7 @@ struct with_location with_location(const with_location&) = delete; // Required so a converting temporary can initialize a by-value parameter. - with_location(with_location&&) = default; + _CCCL_HIDE_FROM_ABI with_location(with_location&&) = default; // No assignment: when `T` is a reference, assigning would write THROUGH the referent // rather than rebind it, silently modifying the caller's object. A location-carrying @@ -72,7 +72,8 @@ struct with_location int> = 0> // Constrained above; the check only recognizes std::enable_if, not cuda::std::enable_if_t. // NOLINTNEXTLINE(bugprone-forwarding-reference-overload) - constexpr with_location(U&& payload, ::cuda::std::source_location loc = ::cuda::std::source_location::current()) + _CCCL_HOST_API constexpr with_location(U&& payload, + ::cuda::std::source_location loc = ::cuda::std::source_location::current()) : payload(::cuda::std::forward(payload)) , loc(loc) {} @@ -106,7 +107,7 @@ struct source_location_hash * function name into account because the same callsite could be used in * different instantiation of the same templated class, the name will reflect * the template parameters. */ - ::std::size_t operator()(const ::cuda::std::source_location& loc) const noexcept + _CCCL_HOST_API ::std::size_t operator()(const ::cuda::std::source_location& loc) const noexcept { return ::std::hash{}(loc.file_name()) ^ (::std::hash{}(loc.line()) << 1) ^ (::std::hash{}(loc.column()) << 2) ^ (::std::hash{}(loc.function_name()) << 3); @@ -115,7 +116,8 @@ struct source_location_hash struct source_location_equal { - bool operator()(const ::cuda::std::source_location& lhs, const ::cuda::std::source_location& rhs) const noexcept + _CCCL_HOST_API bool operator()(const ::cuda::std::source_location& lhs, + const ::cuda::std::source_location& rhs) const noexcept { // Comparing const char * is legit here because these are string literal constants return lhs.file_name() == rhs.file_name() && lhs.line() == rhs.line() && lhs.column() == rhs.column() diff --git a/docs/cub/Doxyfile b/docs/cub/Doxyfile index 42fb8b96ad75..37e457628774 100644 --- a/docs/cub/Doxyfile +++ b/docs/cub/Doxyfile @@ -138,10 +138,10 @@ PREDEFINED = __device__= \ "_CCCL_TRAILING_REQUIRES(x)=-> x requires " \ _CCCL_TYPE_VISIBILITY_DEFAULT= \ _CCCL_TYPE_VISIBILITY_HIDDEN= \ - _CCCL_API=inline \ - _CCCL_HOST_DEVICE_API=inline \ - _CCCL_DEVICE_API=inline \ - _CCCL_HOST_API=inline \ + _CCCL_API= \ + _CCCL_HOST_DEVICE_API= \ + _CCCL_DEVICE_API= \ + _CCCL_HOST_API= \ _CCCL_NODEBUG= \ _CCCL_NODEBUG_API=inline \ _CCCL_NODEBUG_DEVICE_API=inline \ @@ -156,9 +156,9 @@ PREDEFINED = __device__= \ _CCCL_CATCH=catch \ "_CCCL_CATCH_ALL=catch (...)" \ "_CCCL_CATCH_FALLTHROUGH=" \ - _CCCL_PUBLIC_API=inline \ - _CCCL_PUBLIC_DEVICE_API=inline \ - _CCCL_PUBLIC_HOST_API=inline \ + _CCCL_PUBLIC_API= \ + _CCCL_PUBLIC_DEVICE_API= \ + _CCCL_PUBLIC_HOST_API= \ "_CUDAX_CONSTEXPR_FRIEND=friend" \ "_LIBCUDACXX_HAS_SPACESHIP_OPERATOR()=1" \ "CCCL_DEPRECATED=" \ diff --git a/docs/cudax/Doxyfile b/docs/cudax/Doxyfile index 81805986c9bd..8cd4898b164d 100644 --- a/docs/cudax/Doxyfile +++ b/docs/cudax/Doxyfile @@ -98,10 +98,10 @@ PREDEFINED = \ "_CCCL_FORCEINLINE=" \ "_CCCL_CONSTEXPR_CXX20=constexpr" \ "_CCCL_CONSTEXPR_CXX23=constexpr" \ - "_CCCL_API=inline" \ - "_CCCL_HOST_DEVICE_API=inline" \ - "_CCCL_HOST_API=inline" \ - "_CCCL_DEVICE_API=inline" \ + "_CCCL_API=" \ + "_CCCL_HOST_DEVICE_API=" \ + "_CCCL_HOST_API=" \ + "_CCCL_DEVICE_API=" \ "_CCCL_NODEBUG=" \ "_CCCL_NODEBUG_API=inline" \ "_CCCL_NODEBUG_DEVICE_API=inline" \ @@ -109,9 +109,9 @@ PREDEFINED = \ "_CCCL_TRIVIAL_API=inline" \ "_CCCL_TRIVIAL_HOST_API=inline" \ "_CCCL_TRIVIAL_DEVICE_API=inline" \ - "_CCCL_PUBLIC_API=inline" \ - "_CCCL_PUBLIC_DEVICE_API=inline" \ - "_CCCL_PUBLIC_HOST_API=inline" \ + "_CCCL_PUBLIC_API=" \ + "_CCCL_PUBLIC_DEVICE_API=" \ + "_CCCL_PUBLIC_HOST_API=" \ "_CCCL_BEGIN_NAMESPACE_CUDA_STD=namespace cuda::std {" \ "_CCCL_END_NAMESPACE_CUDA_STD=}" \ "_CCCL_CATCH=catch" \ diff --git a/docs/libcudacxx/Doxyfile b/docs/libcudacxx/Doxyfile index 54e259ba1224..56333484fb2e 100644 --- a/docs/libcudacxx/Doxyfile +++ b/docs/libcudacxx/Doxyfile @@ -82,8 +82,8 @@ PREDEFINED = \ "_LIBCUDACXX_TEMPLATE_VIS=" \ "__cccl_lib_mdspan" \ "_CCCL_AND=&&" \ - "_CCCL_API=inline" \ - "_CCCL_HOST_DEVICE_API=inline" \ + "_CCCL_API=" \ + "_CCCL_HOST_DEVICE_API=" \ "_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT="\ "_CCCL_BEGIN_NAMESPACE_CUDA=namespace cuda {" \ "_CCCL_BEGIN_NAMESPACE_CUDA_DEVICE=namespace cuda::device {" \ @@ -106,7 +106,7 @@ PREDEFINED = \ "_CCCL_CTK_BELOW(x, y)=0" \ "_CCCL_CUDACC_AT_LEAST(x, y)=1" \ "_CCCL_CUDACC_BELOW(x, y)=0" \ - "_CCCL_DEVICE_API=inline" \ + "_CCCL_DEVICE_API=" \ "_CCCL_DIAG_POP=" \ "_CCCL_DIAG_PUSH=" \ "_CCCL_DIAG_SUPPRESS_CLANG(x)=" \ @@ -130,10 +130,10 @@ PREDEFINED = \ "_CCCL_GLOBAL_CONSTANT=inline constexpr" \ "_CCCL_HAS_CTK()=1" \ "_CCCL_HIDE_FROM_ABI=" \ - "_CCCL_HOST_API=inline" \ - "_CCCL_PUBLIC_API=inline" \ - "_CCCL_PUBLIC_DEVICE_API=inline" \ - "_CCCL_PUBLIC_HOST_API=inline" \ + "_CCCL_HOST_API=" \ + "_CCCL_PUBLIC_API=" \ + "_CCCL_PUBLIC_DEVICE_API=" \ + "_CCCL_PUBLIC_HOST_API=" \ "_CCCL_REQUIRES(x)= ::cuda::std::enable_if_t = 0>" \ "_CCCL_STD_VER=2020" \ "_CCCL_SUPPRESS_DEPRECATED_POP=" \ diff --git a/docs/thrust/Doxyfile b/docs/thrust/Doxyfile index c889a60d2c66..e84a17d09635 100644 --- a/docs/thrust/Doxyfile +++ b/docs/thrust/Doxyfile @@ -109,10 +109,10 @@ PREDEFINED = __device__= \ "_CCCL_TRAILING_REQUIRES(x)=-> x requires " \ _CCCL_TYPE_VISIBILITY_DEFAULT= \ _CCCL_TYPE_VISIBILITY_HIDDEN= \ - _CCCL_API=inline \ - _CCCL_HOST_DEVICE_API=inline \ - _CCCL_DEVICE_API=inline \ - _CCCL_HOST_API=inline \ + _CCCL_API= \ + _CCCL_HOST_DEVICE_API= \ + _CCCL_DEVICE_API= \ + _CCCL_HOST_API= \ _CCCL_NODEBUG= \ _CCCL_NODEBUG_API=inline \ _CCCL_NODEBUG_DEVICE_API=inline \ @@ -120,9 +120,9 @@ PREDEFINED = __device__= \ _CCCL_TRIVIAL_API=inline \ _CCCL_TRIVIAL_DEVICE_API=inline \ _CCCL_TRIVIAL_HOST_API=inline \ - _CCCL_PUBLIC_API=inline \ - _CCCL_PUBLIC_DEVICE_API=inline \ - _CCCL_PUBLIC_HOST_API=inline \ + _CCCL_PUBLIC_API= \ + _CCCL_PUBLIC_DEVICE_API= \ + _CCCL_PUBLIC_HOST_API= \ _CCCL_VISIBILITY_DEFAULT= \ _CCCL_VISIBILITY_HIDDEN= \ _CCCL_LIFETIMEBOUND= \ @@ -135,9 +135,9 @@ PREDEFINED = __device__= \ "THRUST_NAMESPACE_END=}" \ THRUST_PREVENT_MACRO_SUBSTITUTION \ "THRUST_FWD(x)=x" \ - "_CCCL_PUBLIC_API=inline" \ - "_CCCL_PUBLIC_DEVICE_API=inline" \ - "_CCCL_PUBLIC_HOST_API=inline" \ + "_CCCL_PUBLIC_API=" \ + "_CCCL_PUBLIC_DEVICE_API=" \ + "_CCCL_PUBLIC_HOST_API=" \ "_CCCL_BEGIN_NAMESPACE_CUDA_STD=namespace cuda::std {" \ "_CCCL_END_NAMESPACE_CUDA_STD=}" \ "_CUDAX_CONSTEXPR_FRIEND=friend" \ diff --git a/libcudacxx/include/cuda/__argument/argument_bounds.h b/libcudacxx/include/cuda/__argument/argument_bounds.h index 292d1348e752..a50f1d54572a 100644 --- a/libcudacxx/include/cuda/__argument/argument_bounds.h +++ b/libcudacxx/include/cuda/__argument/argument_bounds.h @@ -125,7 +125,7 @@ class runtime_bounds _Tp __upper_ = __type_highest<_Tp>(); public: - constexpr runtime_bounds() noexcept = default; + _CCCL_HIDE_FROM_ABI constexpr runtime_bounds() noexcept = default; _CCCL_API constexpr runtime_bounds(_Tp __lower, _Tp __upper) noexcept : __lower_(__lower) diff --git a/libcudacxx/include/cuda/__container/buffer.h b/libcudacxx/include/cuda/__container/buffer.h index 08d57b89e4a7..7e5290228045 100644 --- a/libcudacxx/include/cuda/__container/buffer.h +++ b/libcudacxx/include/cuda/__container/buffer.h @@ -132,7 +132,7 @@ class buffer //! @brief Helper to check container is compatible with this buffer template - [[nodiscard]] static constexpr bool __compatible_range() noexcept + [[nodiscard]] _CCCL_HOST_API static constexpr bool __compatible_range() noexcept { return ::cuda::std::ranges::__container_compatible_range<_Range, _Tp>; } @@ -140,20 +140,20 @@ class buffer //! @brief Helper to check whether a different buffer still satisfies all //! properties of this one template - [[nodiscard]] static constexpr bool __properties_match() noexcept + [[nodiscard]] _CCCL_HOST_API static constexpr bool __properties_match() noexcept { return ::cuda::std::__type_set_contains_v<::cuda::std::__make_type_set<_OtherProperties...>, _Properties...>; } //! @brief Helper to return an resource_ref to the currently used resource. //! Used to grow the buffer - __resource_ref_t __borrow_resource() const noexcept + _CCCL_HOST_API __resource_ref_t __borrow_resource() const noexcept { return const_cast<__resource_t&>(__buf_.memory_resource()); } template - static size_t __alignment_from_env(const _Env& __env) + _CCCL_HOST_API static size_t __alignment_from_env(const _Env& __env) { const auto __align = ::cuda::__call_or(::cuda::allocation_alignment, alignof(_Tp), __env); ::cuda::__validate_allocation_alignment(__align, alignof(_Tp)); @@ -1003,7 +1003,7 @@ _CCCL_TEMPLATE( _CCCL_REQUIRES( ::cuda::mr::synchronous_resource_with<::cuda::std::decay_t<_Resource>, _FirstProperty, _RestProperties...> _CCCL_AND __buffer_compatible_env<_Env>) -buffer<_Tp, _FirstProperty, _RestProperties...> +_CCCL_HOST_API buffer<_Tp, _FirstProperty, _RestProperties...> make_buffer(stream_ref __stream, _Resource&& __mr, const _Env& __env = {}) { return buffer<_Tp, _FirstProperty, _RestProperties...>{__stream, ::cuda::std::forward<_Resource>(__mr), __env}; @@ -1084,7 +1084,7 @@ _CCCL_TEMPLATE( _CCCL_REQUIRES( ::cuda::mr::synchronous_resource_with<::cuda::std::decay_t<_Resource>, _FirstProperty, _RestProperties...> _CCCL_AND __buffer_compatible_env<_Env>) -buffer<_Tp, _FirstProperty, _RestProperties...> +_CCCL_HOST_API buffer<_Tp, _FirstProperty, _RestProperties...> make_buffer(stream_ref __stream, _Resource&& __mr, size_t __size, ::cuda::no_init_t, const _Env& __env = {}) { return buffer<_Tp, _FirstProperty, _RestProperties...>{ @@ -1124,7 +1124,7 @@ _CCCL_TEMPLATE(class _Tp, _CCCL_REQUIRES( ::cuda::mr::synchronous_resource_with<::cuda::std::decay_t<_Resource>, _FirstProperty, _RestProperties...> _CCCL_AND __buffer_compatible_env<_Env> _CCCL_AND ::cuda::std::__has_forward_traversal<_Iter>) -buffer<_Tp, _FirstProperty, _RestProperties...> +_CCCL_HOST_API buffer<_Tp, _FirstProperty, _RestProperties...> make_buffer(stream_ref __stream, _Resource&& __mr, _Iter __first, _Iter __last, const _Env& __env = {}) { return buffer<_Tp, _FirstProperty, _RestProperties...>{ @@ -1162,7 +1162,7 @@ _CCCL_TEMPLATE( _CCCL_REQUIRES( ::cuda::mr::synchronous_resource_with<::cuda::std::decay_t<_Resource>, _FirstProperty, _RestProperties...> _CCCL_AND __buffer_compatible_env<_Env>) -buffer<_Tp, _FirstProperty, _RestProperties...> +_CCCL_HOST_API buffer<_Tp, _FirstProperty, _RestProperties...> make_buffer(stream_ref __stream, _Resource&& __mr, ::cuda::std::initializer_list<_Tp> __ilist, const _Env& __env = {}) { return buffer<_Tp, _FirstProperty, _RestProperties...>{ @@ -1202,7 +1202,7 @@ _CCCL_TEMPLATE(class _Tp, _CCCL_REQUIRES( ::cuda::mr::synchronous_resource_with<::cuda::std::decay_t<_Resource>, _FirstProperty, _RestProperties...> _CCCL_AND __buffer_compatible_env<_Env> _CCCL_AND ::cuda::std::ranges::forward_range<_Range>) -buffer<_Tp, _FirstProperty, _RestProperties...> +_CCCL_HOST_API buffer<_Tp, _FirstProperty, _RestProperties...> make_buffer(stream_ref __stream, _Resource&& __mr, _Range&& __range, const _Env& __env = {}) { return buffer<_Tp, _FirstProperty, _RestProperties...>{ diff --git a/libcudacxx/include/cuda/__container/uninitialized_async_buffer.h b/libcudacxx/include/cuda/__container/uninitialized_async_buffer.h index 284b02e71c6e..4342c08316ab 100644 --- a/libcudacxx/include/cuda/__container/uninitialized_async_buffer.h +++ b/libcudacxx/include/cuda/__container/uninitialized_async_buffer.h @@ -147,31 +147,32 @@ class __uninitialized_async_buffer { __async_resource* __resource_; - void* allocate_sync(std::size_t __size, std::size_t __alignment) + _CCCL_HOST_API void* allocate_sync(std::size_t __size, std::size_t __alignment) { return __resource_->allocate_sync(__size, __alignment); } - void deallocate_sync(void* __ptr, std::size_t __size, std::size_t __alignment) noexcept + _CCCL_HOST_API void deallocate_sync(void* __ptr, std::size_t __size, std::size_t __alignment) noexcept { __resource_->deallocate_sync(__ptr, __size, __alignment); } - void* allocate(::cuda::stream_ref __stream, std::size_t __size, std::size_t __alignment) + _CCCL_HOST_API void* allocate(::cuda::stream_ref __stream, std::size_t __size, std::size_t __alignment) { return __resource_->allocate(__stream, __size, __alignment); } - void deallocate(::cuda::stream_ref __stream, void* __ptr, std::size_t __size, std::size_t __alignment) noexcept + _CCCL_HOST_API void + deallocate(::cuda::stream_ref __stream, void* __ptr, std::size_t __size, std::size_t __alignment) noexcept { __resource_->deallocate(__stream, __ptr, __size, __alignment); } - friend bool operator==(const __fake_resource_ref& __lhs, const __fake_resource_ref& __rhs) noexcept + _CCCL_HOST_API friend bool operator==(const __fake_resource_ref& __lhs, const __fake_resource_ref& __rhs) noexcept { return *__lhs.__resource_ == *__rhs.__resource_; } - friend bool operator!=(const __fake_resource_ref& __lhs, const __fake_resource_ref& __rhs) noexcept + _CCCL_HOST_API friend bool operator!=(const __fake_resource_ref& __lhs, const __fake_resource_ref& __rhs) noexcept { return *__lhs.__resource_ != *__rhs.__resource_; } diff --git a/libcudacxx/include/cuda/__device/all_devices.h b/libcudacxx/include/cuda/__device/all_devices.h index f6353513b3db..40e3b2fa99d8 100644 --- a/libcudacxx/include/cuda/__device/all_devices.h +++ b/libcudacxx/include/cuda/__device/all_devices.h @@ -49,7 +49,7 @@ _CCCL_BEGIN_NAMESPACE_ABI_VER4_BUMP return __ret; } -[[nodiscard]] inline ::cuda::std::span __devices() +[[nodiscard]] _CCCL_HOST_API inline ::cuda::std::span __devices() { static const auto __devices = ::cuda::__make_devices(); return ::cuda::std::span{__devices.data(), __devices.size()}; diff --git a/libcudacxx/include/cuda/__event/timed_event.h b/libcudacxx/include/cuda/__event/timed_event.h index d1412520c623..1e74d359a6c4 100644 --- a/libcudacxx/include/cuda/__event/timed_event.h +++ b/libcudacxx/include/cuda/__event/timed_event.h @@ -60,10 +60,10 @@ class timed_event : public event : event(no_init) {} - timed_event(timed_event&&) noexcept = default; - timed_event(const timed_event&) = delete; - timed_event& operator=(timed_event&&) noexcept = default; - timed_event& operator=(const timed_event&) = delete; + _CCCL_HIDE_FROM_ABI timed_event(timed_event&&) noexcept = default; + timed_event(const timed_event&) = delete; + _CCCL_HIDE_FROM_ABI timed_event& operator=(timed_event&&) noexcept = default; + timed_event& operator=(const timed_event&) = delete; //! @brief Construct a `timed_event` object from a native `cudaEvent_t` handle. //! diff --git a/libcudacxx/include/cuda/__hierarchy/hierarchy_dimensions.h b/libcudacxx/include/cuda/__hierarchy/hierarchy_dimensions.h index cf6bc204e683..898afdf729fc 100644 --- a/libcudacxx/include/cuda/__hierarchy/hierarchy_dimensions.h +++ b/libcudacxx/include/cuda/__hierarchy/hierarchy_dimensions.h @@ -432,7 +432,8 @@ class hierarchy # if !_CCCL_COMPILER(NVRTC) template - friend constexpr auto hierarchy_add_level(const hierarchy<_Unit, _LevelDescs2...>& hierarchy, _NewLevel __lnew); + friend + _CCCL_API constexpr auto hierarchy_add_level(const hierarchy<_Unit, _LevelDescs2...>& hierarchy, _NewLevel __lnew); # endif // !_CCCL_COMPILER(NVRTC) }; @@ -488,7 +489,7 @@ _CCCL_DEDUCTION_GUIDE_ATTRIBUTES hierarchy(const _BottomUnit&, const ::cuda::std * @par */ template -constexpr auto make_hierarchy(_L1 __l1, _LevelDescs... __ls) noexcept +_CCCL_API constexpr auto make_hierarchy(_L1 __l1, _LevelDescs... __ls) noexcept { return __detail::__make_hierarchy<_LUnit>()(__detail::__as_level(__l1), __detail::__as_level(__ls)...); } @@ -518,7 +519,7 @@ constexpr auto make_hierarchy(_L1 __l1, _LevelDescs... __ls) noexcept * @par */ template -constexpr auto hierarchy_add_level(const hierarchy<_Unit, _LevelDescs...>& __hierarchy, _NewLevel __lnew) +_CCCL_API constexpr auto hierarchy_add_level(const hierarchy<_Unit, _LevelDescs...>& __hierarchy, _NewLevel __lnew) { auto __new_level = __detail::__as_level(__lnew); using __added_level = decltype(__new_level); diff --git a/libcudacxx/include/cuda/__iterator/counting_iterator.h b/libcudacxx/include/cuda/__iterator/counting_iterator.h index 7b28923c8543..b2b7bf0e68b2 100644 --- a/libcudacxx/include/cuda/__iterator/counting_iterator.h +++ b/libcudacxx/include/cuda/__iterator/counting_iterator.h @@ -175,10 +175,10 @@ class counting_iterator : public __counting_iterator_category<_Start> : __value_(::cuda::std::move(__value)) {} - constexpr counting_iterator(const counting_iterator&) = default; - constexpr counting_iterator(counting_iterator&&) = default; - constexpr counting_iterator& operator=(const counting_iterator&) = default; - constexpr counting_iterator& operator=(counting_iterator&&) = default; + _CCCL_HIDE_FROM_ABI constexpr counting_iterator(const counting_iterator&) = default; + _CCCL_HIDE_FROM_ABI constexpr counting_iterator(counting_iterator&&) = default; + _CCCL_HIDE_FROM_ABI constexpr counting_iterator& operator=(const counting_iterator&) = default; + _CCCL_HIDE_FROM_ABI constexpr counting_iterator& operator=(counting_iterator&&) = default; //! @brief Creates a @c counting_iterator from another @c counting_iterator of a different //! difference type. diff --git a/libcudacxx/include/cuda/__launch/configuration.h b/libcudacxx/include/cuda/__launch/configuration.h index 58aaeb1d8534..6dc9a34a0f73 100644 --- a/libcudacxx/include/cuda/__launch/configuration.h +++ b/libcudacxx/include/cuda/__launch/configuration.h @@ -305,12 +305,13 @@ class _CCCL_DECLSPEC_EMPTY_BASES dynamic_shared_memory_option } // Helper function to access private constructors - static constexpr dynamic_shared_memory_option __create(bool __non_portable = false) noexcept + _CCCL_HOST_API static constexpr dynamic_shared_memory_option __create(bool __non_portable = false) noexcept { return dynamic_shared_memory_option{__non_portable}; } - static constexpr dynamic_shared_memory_option __create(::cuda::std::size_t __n, bool __non_portable = false) noexcept + _CCCL_HOST_API static constexpr dynamic_shared_memory_option + __create(::cuda::std::size_t __n, bool __non_portable = false) noexcept { return dynamic_shared_memory_option{__n, __non_portable}; } @@ -327,7 +328,7 @@ class _CCCL_DECLSPEC_EMPTY_BASES dynamic_shared_memory_option }; template -[[nodiscard]] ::cudaError_t __apply_launch_option( +[[nodiscard]] _CCCL_HOST_API ::cudaError_t __apply_launch_option( const dynamic_shared_memory_option<_Tp>& __opt, ::CUlaunchConfig& __config, ::CUfunction __kernel) noexcept { ::cudaError_t __status = ::cudaSuccess; @@ -477,7 +478,7 @@ struct launch_priority : public __detail::launch_option static constexpr __detail::launch_option_kind kind = __detail::launch_option_kind::launch_priority; int priority; - launch_priority(int p) noexcept + _CCCL_HOST_API launch_priority(int p) noexcept : priority(p) {} }; @@ -498,7 +499,7 @@ template struct __filter_options { template - [[nodiscard]] auto __option_or_empty(const _Option& __option) + [[nodiscard]] _CCCL_HOST_API auto __option_or_empty(const _Option& __option) { if constexpr (_Pred) { @@ -511,7 +512,7 @@ struct __filter_options } template - [[nodiscard]] auto operator()(const _Options&... __options) + [[nodiscard]] _CCCL_HOST_API auto operator()(const _Options&... __options) { return ::cuda::std::tuple_cat( __option_or_empty>(__options)...); @@ -519,7 +520,7 @@ struct __filter_options }; template -auto __make_config_from_tuple(const _Dimensions& __dims, const ::cuda::std::tuple<_Options...>& __opts); +_CCCL_HOST_API auto __make_config_from_tuple(const _Dimensions& __dims, const ::cuda::std::tuple<_Options...>& __opts); template inline constexpr bool __is_kernel_config = false; @@ -553,12 +554,12 @@ struct kernel_config static_assert(::cuda::std::_And<::cuda::std::is_base_of<__detail::launch_option, _Options>...>::value); static_assert(__detail::no_duplicate_options<_Options...>); - constexpr kernel_config(const _Hierarchy& hierarchy, const _Options&... opts) + _CCCL_HOST_API constexpr kernel_config(const _Hierarchy& hierarchy, const _Options&... opts) : __hierarchy(hierarchy) - , __options(opts...) {}; - constexpr kernel_config(const _Hierarchy& hierarchy, const ::cuda::std::tuple<_Options...>& opts) + , __options(opts...){}; + _CCCL_HOST_API constexpr kernel_config(const _Hierarchy& hierarchy, const ::cuda::std::tuple<_Options...>& opts) : __hierarchy(hierarchy) - , __options(opts) {}; + , __options(opts){}; [[nodiscard]] _CCCL_API constexpr const _Hierarchy& hierarchy() const noexcept { @@ -580,7 +581,7 @@ struct kernel_config * _Options to be added to the configuration */ template - [[nodiscard]] auto add(const _NewOptions&... new_options) const + [[nodiscard]] _CCCL_HOST_API auto add(const _NewOptions&... new_options) const { return kernel_config<_Hierarchy, _Options..., _NewOptions...>( __hierarchy, ::cuda::std::tuple_cat(__options, ::cuda::std::make_tuple(new_options...))); @@ -605,7 +606,8 @@ struct kernel_config * Other configuration to combine with this configuration */ template - [[nodiscard]] auto combine(const kernel_config<_OtherDimensions, _OtherOptions...>& __other_config) const + [[nodiscard]] _CCCL_HOST_API auto + combine(const kernel_config<_OtherDimensions, _OtherOptions...>& __other_config) const { // can't use fully qualified kernel_config name here because of nvcc bug, // TODO remove __make_config_from_tuple once fixed @@ -630,7 +632,7 @@ struct kernel_config * Kernel functor to search for the default configuration */ template - [[nodiscard]] auto combine_with_default(const _Kernel& __kernel) const + [[nodiscard]] _CCCL_HOST_API auto combine_with_default(const _Kernel& __kernel) const { if constexpr (__kernel_has_default_config<_Kernel>) { @@ -671,7 +673,7 @@ operator&(const hierarchy_level_desc<_L1, _Dims1>& l1, const hierarchy_level_des } template -auto __make_config_from_tuple(const _Dimensions& __dims, const ::cuda::std::tuple<_Options...>& __opts) +_CCCL_HOST_API auto __make_config_from_tuple(const _Dimensions& __dims, const ::cuda::std::tuple<_Options...>& __opts) { return kernel_config(__dims, __opts); } @@ -680,7 +682,7 @@ template >> -[[nodiscard]] constexpr auto +[[nodiscard]] _CCCL_HOST_API constexpr auto operator&(const kernel_config<_Dimensions, _Options...>& config, const _Option& option) noexcept { return config.add(option); @@ -689,7 +691,7 @@ operator&(const kernel_config<_Dimensions, _Options...>& config, const _Option& template >> -[[nodiscard]] constexpr auto operator&(const hierarchy<_Levels...>& dims, const _Option& option) noexcept +[[nodiscard]] _CCCL_HOST_API constexpr auto operator&(const hierarchy<_Levels...>& dims, const _Option& option) noexcept { return kernel_config(dims, option); } @@ -711,7 +713,8 @@ template -[[nodiscard]] constexpr auto make_config(const hierarchy<_BottomUnit, _Levels...>& dims, const _Opts&... opts) noexcept +[[nodiscard]] _CCCL_HOST_API constexpr auto +make_config(const hierarchy<_BottomUnit, _Levels...>& dims, const _Opts&... opts) noexcept { return kernel_config, _Opts...>(dims, opts...); } @@ -736,14 +739,14 @@ template * @endcode */ template -constexpr auto distribute(int numElements) noexcept +_CCCL_HOST_API constexpr auto distribute(int numElements) noexcept { const int blocksPerGrid = (numElements + _ThreadsPerBlock - 1) / _ThreadsPerBlock; return make_config(make_hierarchy(grid_dims(blocksPerGrid), block_dims<_ThreadsPerBlock>())); } template -[[nodiscard]] constexpr auto __process_config_args(const ::cuda::std::tuple<_Prev...>& previous) +[[nodiscard]] _CCCL_HOST_API constexpr auto __process_config_args(const ::cuda::std::tuple<_Prev...>& previous) { if constexpr (sizeof...(_Prev) == 0) { @@ -757,7 +760,7 @@ template } template -[[nodiscard]] constexpr auto +[[nodiscard]] _CCCL_HOST_API constexpr auto __process_config_args(const ::cuda::std::tuple<_Prev...>& previous, const _Arg& arg, const _Rest&... rest) { if constexpr (::cuda::std::is_base_of_v<__detail::launch_option, _Arg>) @@ -781,7 +784,7 @@ __process_config_args(const ::cuda::std::tuple<_Prev...>& previous, const _Arg& } template -[[nodiscard]] constexpr auto make_config(const _Args&... args) +[[nodiscard]] _CCCL_HOST_API constexpr auto make_config(const _Args&... args) { return __process_config_args(::cuda::std::make_tuple(), args...); } @@ -789,13 +792,14 @@ template namespace __detail { template -unsigned int constexpr kernel_config_count_attr_space(const kernel_config<_Dimensions, _Options...>&) noexcept +_CCCL_HOST_API unsigned int constexpr kernel_config_count_attr_space( + const kernel_config<_Dimensions, _Options...>&) noexcept { return (0 + ... + _Options::needs_attribute_space); } template -[[nodiscard]] cudaError_t apply_kernel_config( +[[nodiscard]] _CCCL_HOST_API cudaError_t apply_kernel_config( const kernel_config<_Dimensions, _Options...>& config, CUlaunchConfig& cuda_config, CUfunction kernel) noexcept { return ::cuda::std::apply( diff --git a/libcudacxx/include/cuda/__launch/launch.h b/libcudacxx/include/cuda/__launch/launch.h index 57ce149a57d6..bdc6c59c2c46 100644 --- a/libcudacxx/include/cuda/__launch/launch.h +++ b/libcudacxx/include/cuda/__launch/launch.h @@ -285,7 +285,7 @@ inline constexpr bool __invoke_kernel_functor_with_config_v = // 3. Fallback without any attributes. template -__global__ static void +_CCCL_KERNEL_ATTRIBUTES static void // todo(dabayer): Re-enable this once cuda::launch with kernels that were compiled with .blocksareclusters directive is // fixed. // @@ -310,8 +310,9 @@ __kernel_launcher_with_block_size(const _CCCL_GRID_CONSTANT _Config __conf, _Ker } template -__global__ static void _CCCL_LAUNCH_BOUNDS(::cuda::__max_nthreads_per_block()) -__kernel_launcher_with_launch_bounds(const _CCCL_GRID_CONSTANT _Config __conf, _Kernel __kernel_fn, _Args... __args) +_CCCL_KERNEL_ATTRIBUTES static void + _CCCL_LAUNCH_BOUNDS(::cuda::__max_nthreads_per_block()) + __kernel_launcher_with_launch_bounds(const _CCCL_GRID_CONSTANT _Config __conf, _Kernel __kernel_fn, _Args... __args) { ::cuda::__assume_known_info(); @@ -326,7 +327,8 @@ __kernel_launcher_with_launch_bounds(const _CCCL_GRID_CONSTANT _Config __conf, _ } template -__global__ static void __kernel_launcher(const _CCCL_GRID_CONSTANT _Config __conf, _Kernel __kernel_fn, _Args... __args) +_CCCL_KERNEL_ATTRIBUTES static void +__kernel_launcher(const _CCCL_GRID_CONSTANT _Config __conf, _Kernel __kernel_fn, _Args... __args) { ::cuda::__assume_known_info(); diff --git a/libcudacxx/include/cuda/__memory_pool/device_memory_pool.h b/libcudacxx/include/cuda/__memory_pool/device_memory_pool.h index 10887db9ee5b..49483a1b737c 100644 --- a/libcudacxx/include/cuda/__memory_pool/device_memory_pool.h +++ b/libcudacxx/include/cuda/__memory_pool/device_memory_pool.h @@ -194,7 +194,7 @@ struct device_memory_pool : device_memory_pool_ref ::CU_MEM_ALLOCATION_TYPE_PINNED)) {} - ~device_memory_pool() noexcept + _CCCL_HOST_API ~device_memory_pool() noexcept { if (__pool_ != nullptr) { @@ -228,7 +228,7 @@ struct device_memory_pool : device_memory_pool_ref device_memory_pool& operator=(const device_memory_pool&) = delete; private: - device_memory_pool(::cudaMemPool_t __pool) noexcept + _CCCL_HOST_API device_memory_pool(::cudaMemPool_t __pool) noexcept : device_memory_pool_ref(__pool) {} }; diff --git a/libcudacxx/include/cuda/__memory_pool/shared_device_memory_pool.h b/libcudacxx/include/cuda/__memory_pool/shared_device_memory_pool.h index 95daedaba15d..4ac494be20df 100644 --- a/libcudacxx/include/cuda/__memory_pool/shared_device_memory_pool.h +++ b/libcudacxx/include/cuda/__memory_pool/shared_device_memory_pool.h @@ -58,8 +58,8 @@ class shared_device_memory_pool : public __shared_memory_pool_base= __min_alignment && __alignment != 0 && (__alignment & (__alignment - 1)) == 0; diff --git a/libcudacxx/include/cuda/__memory_resource/any_resource.h b/libcudacxx/include/cuda/__memory_resource/any_resource.h index 12b0a7128351..80bb5f3ebd2d 100644 --- a/libcudacxx/include/cuda/__memory_resource/any_resource.h +++ b/libcudacxx/include/cuda/__memory_resource/any_resource.h @@ -289,7 +289,7 @@ struct _CCCL_DECLSPEC_EMPTY_BASES any_resource using default_queries = ::cuda::mr::properties_list<_Properties...>; //! @cond - explicit any_resource(__from_base_tag, __base&& __b) noexcept + _CCCL_HOST_API explicit any_resource(__from_base_tag, __base&& __b) noexcept : __base(::cuda::std::move(__b)) {} //! @endcond @@ -300,7 +300,7 @@ struct _CCCL_DECLSPEC_EMPTY_BASES any_resource using __base::interface; - __base& __get_base() noexcept + _CCCL_HOST_API __base& __get_base() noexcept { return *this; } @@ -316,32 +316,33 @@ struct _CCCL_DECLSPEC_EMPTY_BASES synchronous_resource_ref // Inherit constructors from __basic_any _CCCL_DELEGATE_CONSTRUCTORS(synchronous_resource_ref, ::cuda::__basic_any, __iresource<_Properties...>&); - synchronous_resource_ref(const synchronous_resource_ref& __other) noexcept = default; + _CCCL_HIDE_FROM_ABI synchronous_resource_ref(const synchronous_resource_ref& __other) noexcept = default; // Allow narrowing conversion from a synchronous_resource_ref with a superset // of properties by rebinding to the same underlying object. _CCCL_TEMPLATE(class... _OtherProperties) _CCCL_REQUIRES((::cuda::std::__type_set_contains_v<::cuda::std::__type_set<_OtherProperties...>, _Properties...>) ) - synchronous_resource_ref(const synchronous_resource_ref<_OtherProperties...>& __other) noexcept + _CCCL_HOST_API synchronous_resource_ref(const synchronous_resource_ref<_OtherProperties...>& __other) noexcept : __base(__other.__get_base()) {} // resource_ref is convertible to synchronous_resource_ref _CCCL_TEMPLATE(class... _OtherProperties) _CCCL_REQUIRES((::cuda::std::__type_set_contains_v<::cuda::std::__type_set<_OtherProperties...>, _Properties...>) ) - synchronous_resource_ref(resource_ref<_OtherProperties...> __other) noexcept + _CCCL_HOST_API synchronous_resource_ref(resource_ref<_OtherProperties...> __other) noexcept : __base(__other.__get_base()) {} _CCCL_TEMPLATE(class... _OtherProperties) _CCCL_REQUIRES((::cuda::std::__type_set_contains_v<::cuda::std::__type_set<_OtherProperties...>, _Properties...>) ) - synchronous_resource_ref& operator=(const synchronous_resource_ref<_OtherProperties...>& __other) noexcept + _CCCL_HOST_API synchronous_resource_ref& + operator=(const synchronous_resource_ref<_OtherProperties...>& __other) noexcept { __basic_any_access::__cast_to(__other.__get_base(), __get_base()); return *this; } - synchronous_resource_ref& operator=(const synchronous_resource_ref& __other) noexcept + _CCCL_HOST_API synchronous_resource_ref& operator=(const synchronous_resource_ref& __other) noexcept { __basic_any_access::__cast_to(__other.__get_base(), __get_base()); return *this; @@ -350,7 +351,7 @@ struct _CCCL_DECLSPEC_EMPTY_BASES synchronous_resource_ref using default_queries = ::cuda::mr::properties_list<_Properties...>; //! @cond - explicit synchronous_resource_ref(__from_base_tag, __base&& __b) noexcept + _CCCL_HOST_API explicit synchronous_resource_ref(__from_base_tag, __base&& __b) noexcept : __base(::cuda::std::move(__b)) {} //! @endcond @@ -361,12 +362,12 @@ struct _CCCL_DECLSPEC_EMPTY_BASES synchronous_resource_ref using __base::interface; - __base& __get_base() noexcept + _CCCL_HOST_API __base& __get_base() noexcept { return *this; } - const __base& __get_base() const noexcept + _CCCL_HOST_API const __base& __get_base() const noexcept { return *this; } @@ -382,25 +383,25 @@ struct _CCCL_DECLSPEC_EMPTY_BASES resource_ref // Inherit other constructors from __basic_any _CCCL_DELEGATE_CONSTRUCTORS(resource_ref, ::cuda::__basic_any, __iasync_resource<_Properties...>&); - resource_ref(const resource_ref& __other) noexcept = default; + _CCCL_HIDE_FROM_ABI resource_ref(const resource_ref& __other) noexcept = default; // Allow narrowing conversion from a resource_ref with a superset of // properties by rebinding to the same underlying object. _CCCL_TEMPLATE(class... _OtherProperties) _CCCL_REQUIRES((::cuda::std::__type_set_contains_v<::cuda::std::__type_set<_OtherProperties...>, _Properties...>) ) - resource_ref(const resource_ref<_OtherProperties...>& __other) noexcept + _CCCL_HOST_API resource_ref(const resource_ref<_OtherProperties...>& __other) noexcept : __base(__other.__get_base()) {} _CCCL_TEMPLATE(class... _OtherProperties) _CCCL_REQUIRES((::cuda::std::__type_set_contains_v<::cuda::std::__type_set<_OtherProperties...>, _Properties...>) ) - resource_ref& operator=(const resource_ref<_OtherProperties...>& __other) noexcept + _CCCL_HOST_API resource_ref& operator=(const resource_ref<_OtherProperties...>& __other) noexcept { __basic_any_access::__cast_to(__other.__get_base(), __get_base()); return *this; } - resource_ref& operator=(const resource_ref& __other) noexcept + _CCCL_HOST_API resource_ref& operator=(const resource_ref& __other) noexcept { __basic_any_access::__cast_to(__other.__get_base(), __get_base()); return *this; @@ -409,7 +410,7 @@ struct _CCCL_DECLSPEC_EMPTY_BASES resource_ref using default_queries = ::cuda::mr::properties_list<_Properties...>; //! @cond - explicit resource_ref(__from_base_tag, __base&& __b) noexcept + _CCCL_HOST_API explicit resource_ref(__from_base_tag, __base&& __b) noexcept : __base(::cuda::std::move(__b)) {} //! @endcond @@ -422,12 +423,12 @@ struct _CCCL_DECLSPEC_EMPTY_BASES resource_ref using __base::interface; - __base& __get_base() noexcept + _CCCL_HOST_API __base& __get_base() noexcept { return *this; } - const __base& __get_base() const noexcept + _CCCL_HOST_API const __base& __get_base() const noexcept { return *this; } @@ -446,19 +447,21 @@ inline constexpr bool __disable_default_dynamic_accessibility_property) -synchronous_resource_ref<_Properties...> __as_resource_ref(_Resource& __mr) noexcept +_CCCL_HOST_API synchronous_resource_ref<_Properties...> __as_resource_ref(_Resource& __mr) noexcept { return synchronous_resource_ref<_Properties...>(__mr); } template -synchronous_resource_ref<_Properties...> __as_resource_ref(synchronous_resource_ref<_Properties...> const __mr) noexcept +_CCCL_HOST_API synchronous_resource_ref<_Properties...> +__as_resource_ref(synchronous_resource_ref<_Properties...> const __mr) noexcept { return __mr; } template -synchronous_resource_ref<_Properties...> __as_resource_ref(resource_ref<_Properties...> const __mr) noexcept +_CCCL_HOST_API synchronous_resource_ref<_Properties...> +__as_resource_ref(resource_ref<_Properties...> const __mr) noexcept { return __mr; } @@ -981,7 +984,7 @@ inline constexpr bool __is_resource_ref> = true; //! //! @endrst template -auto make_any_synchronous_resource(_Args&&... __args) -> any_synchronous_resource<_Properties...> +_CCCL_HOST_API auto make_any_synchronous_resource(_Args&&... __args) -> any_synchronous_resource<_Properties...> { static_assert(::cuda::mr::synchronous_resource<_Resource>, "_Resource does not satisfy the cuda::mr::synchronous_resource concept"); @@ -1008,7 +1011,7 @@ auto make_any_synchronous_resource(_Args&&... __args) -> any_synchronous_resourc //! //! @endrst template -auto make_any_resource(_Args&&... __args) -> any_resource<_Properties...> +_CCCL_HOST_API auto make_any_resource(_Args&&... __args) -> any_resource<_Properties...> { static_assert(::cuda::mr::resource<_Resource>, "_Resource does not satisfy the cuda::mr::resource concept"); static_assert(::cuda::mr::resource_with<_Resource, _Properties...>, diff --git a/libcudacxx/include/cuda/__memory_resource/resource.h b/libcudacxx/include/cuda/__memory_resource/resource.h index 061bf62c74bf..d796fa9a0029 100644 --- a/libcudacxx/include/cuda/__memory_resource/resource.h +++ b/libcudacxx/include/cuda/__memory_resource/resource.h @@ -100,7 +100,7 @@ template struct __different_resource__ { template - static constexpr bool __value(_OtherResource*) noexcept + _CCCL_HOST_API static constexpr bool __value(_OtherResource*) noexcept { return synchronous_resource<_OtherResource>; } @@ -109,7 +109,7 @@ struct __different_resource__ template <> struct __different_resource__ { - static constexpr bool __value(void*) noexcept + _CCCL_HOST_API static constexpr bool __value(void*) noexcept { return false; } diff --git a/libcudacxx/include/cuda/__nvtx/nvtx.h b/libcudacxx/include/cuda/__nvtx/nvtx.h index 844fd03e5155..b210b09f74c8 100644 --- a/libcudacxx/include/cuda/__nvtx/nvtx.h +++ b/libcudacxx/include/cuda/__nvtx/nvtx.h @@ -91,7 +91,7 @@ struct __nvtx_cccl_optional_range_host_only bool __engaged = false; alignas(__nvtx_cccl_range) unsigned char __storage[sizeof(__nvtx_cccl_range)]{}; - __nvtx_cccl_optional_range_host_only() = default; + _CCCL_HIDE_FROM_ABI __nvtx_cccl_optional_range_host_only() = default; _CCCL_HOST_API void __start(const ::nvtx3::v1::event_attributes& __attributes) { diff --git a/libcudacxx/include/cuda/__stream/internal_streams.h b/libcudacxx/include/cuda/__stream/internal_streams.h index 9d1acf3bf397..099ae61f40b8 100644 --- a/libcudacxx/include/cuda/__stream/internal_streams.h +++ b/libcudacxx/include/cuda/__stream/internal_streams.h @@ -37,7 +37,7 @@ _CCCL_BEGIN_NAMESPACE_CUDA //! @brief internal stream used for memory allocations, no real blocking work //! should ever be pushed into it -inline ::cuda::stream_ref __cccl_allocation_stream() noexcept +_CCCL_HOST_API inline ::cuda::stream_ref __cccl_allocation_stream() noexcept { // Intentionally leak the stream here to avoid stream destruction when the program exits, which is not guaraneed to // work. diff --git a/libcudacxx/include/cuda/__stream/launch_transform.h b/libcudacxx/include/cuda/__stream/launch_transform.h index bdfeb5efac40..713c13a28e57 100644 --- a/libcudacxx/include/cuda/__stream/launch_transform.h +++ b/libcudacxx/include/cuda/__stream/launch_transform.h @@ -119,7 +119,7 @@ struct _CCCL_TYPE_VISIBILITY_DEFAULT __launch_transform_t using ::cuda::std::optional<_Tp>::optional; // Use of explicit destructor is intentional. Without it, the argument may have a trivial // destructor and hence would be performed by the callee. - ~__optional_with_a_destructor() {} // NOLINT(modernize-use-equals-default) + _CCCL_HOST_API ~__optional_with_a_destructor() {} // NOLINT(modernize-use-equals-default) template _CCCL_API inline _CCCL_CONSTEXPR_CXX20 _Tp& __emplace_from_fn(_Fn&& __fn) diff --git a/libcudacxx/include/cuda/__utility/__basic_any/basic_any_base.h b/libcudacxx/include/cuda/__utility/__basic_any/basic_any_base.h index 2c12721d9b30..00bf57f83287 100644 --- a/libcudacxx/include/cuda/__utility/__basic_any/basic_any_base.h +++ b/libcudacxx/include/cuda/__utility/__basic_any/basic_any_base.h @@ -67,7 +67,7 @@ template struct __basic_any_base<_Interface, 2> : __interface_of<_Interface> // copyable interfaces { // NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init) - __basic_any_base() = default; + _CCCL_HIDE_FROM_ABI __basic_any_base() = default; _CCCL_HOST_DEVICE_API __basic_any_base(__basic_any_base&& __other) noexcept { @@ -107,17 +107,17 @@ struct __basic_any_base<_Interface, 2> : __interface_of<_Interface> // copyable template struct __basic_any_base<_Interface, 1> : __basic_any_base<_Interface, 2> // move-only interfaces { - __basic_any_base() = default; - __basic_any_base(__basic_any_base&&) noexcept = default; - __basic_any_base(__basic_any_base const&) = delete; - auto operator=(__basic_any_base&&) noexcept -> __basic_any_base& = default; - auto operator=(__basic_any_base const&) -> __basic_any_base& = delete; + _CCCL_HIDE_FROM_ABI __basic_any_base() = default; + _CCCL_HIDE_FROM_ABI __basic_any_base(__basic_any_base&&) noexcept = default; + __basic_any_base(__basic_any_base const&) = delete; + _CCCL_HIDE_FROM_ABI auto operator=(__basic_any_base&&) noexcept -> __basic_any_base& = default; + auto operator=(__basic_any_base const&) -> __basic_any_base& = delete; }; template struct __basic_any_base<_Interface, 0> : __basic_any_base<_Interface, 2> // immovable interfaces { - __basic_any_base() = default; + _CCCL_HIDE_FROM_ABI __basic_any_base() = default; __basic_any_base(__basic_any_base&&) noexcept = delete; __basic_any_base(__basic_any_base const&) = delete; auto operator=(__basic_any_base&&) noexcept -> __basic_any_base& = delete; diff --git a/libcudacxx/include/cuda/__utility/__basic_any/basic_any_ptr.h b/libcudacxx/include/cuda/__utility/__basic_any/basic_any_ptr.h index da7213055e5d..a4a0b7057cf7 100644 --- a/libcudacxx/include/cuda/__utility/__basic_any/basic_any_ptr.h +++ b/libcudacxx/include/cuda/__utility/__basic_any/basic_any_ptr.h @@ -54,7 +54,7 @@ struct _CCCL_TYPE_VISIBILITY_DEFAULT __basic_any<_Interface*> //! //! Constructors //! - __basic_any() = default; + _CCCL_HIDE_FROM_ABI __basic_any() = default; _CCCL_NODEBUG_HOST_DEVICE_API __basic_any(::cuda::std::nullptr_t) {} diff --git a/libcudacxx/include/cuda/__utility/__basic_any/basic_any_ref.h b/libcudacxx/include/cuda/__utility/__basic_any/basic_any_ref.h index cca155cc2c4e..72e037242c06 100644 --- a/libcudacxx/include/cuda/__utility/__basic_any/basic_any_ref.h +++ b/libcudacxx/include/cuda/__utility/__basic_any/basic_any_ref.h @@ -149,7 +149,7 @@ struct _CCCL_DECLSPEC_EMPTY_BASES __basic_any<__ireference<_Interface>> friend struct __basic_any; friend struct __basic_any_access; - __basic_any() = default; + _CCCL_HIDE_FROM_ABI __basic_any() = default; //! \brief Constructs a \c __basic_any<__ireference<_Interface>> from an lvalue //! reference to a \c __basic_any<_Interface>. @@ -317,7 +317,7 @@ struct _CCCL_TYPE_VISIBILITY_DEFAULT __basic_any<_Interface&> : __basic_any<__ir friend struct __basic_any; friend struct __basic_any_access; - __basic_any() = default; + _CCCL_HIDE_FROM_ABI __basic_any() = default; }; _CCCL_END_NAMESPACE_CUDA diff --git a/libcudacxx/include/cuda/__utility/__basic_any/basic_any_value.h b/libcudacxx/include/cuda/__utility/__basic_any/basic_any_value.h index fb45669c31da..4745605666b7 100644 --- a/libcudacxx/include/cuda/__utility/__basic_any/basic_any_value.h +++ b/libcudacxx/include/cuda/__utility/__basic_any/basic_any_value.h @@ -73,7 +73,7 @@ struct _CCCL_TYPE_VISIBILITY_DEFAULT __basic_any : __basic_any_base<_Interface> //! @brief Constructs an empty `__basic_any` object. //! @post `has_value() == false` - __basic_any() = default; + _CCCL_HIDE_FROM_ABI __basic_any() = default; // NOLINTBEGIN(bugprone-forwarding-reference-overload) //! @brief Constructs a `__basic_any` object that contains a copy of `__value`. @@ -136,8 +136,8 @@ struct _CCCL_TYPE_VISIBILITY_DEFAULT __basic_any : __basic_any_base<_Interface> // We use base classes to implement movability and copyability. All we need here is to // accept the default implementations. - __basic_any(__basic_any&& __other) = default; - __basic_any(__basic_any const& __other) = default; + _CCCL_HIDE_FROM_ABI __basic_any(__basic_any&& __other) = default; + _CCCL_HIDE_FROM_ABI __basic_any(__basic_any const& __other) = default; //! @brief Converting constructor that move constructs from a compatible //! `__basic_any` object. @@ -213,8 +213,8 @@ struct _CCCL_TYPE_VISIBILITY_DEFAULT __basic_any : __basic_any_base<_Interface> // We use base classes to implement movability and copyability. All we need here is to // accept the default implementations. - auto operator=(__basic_any&& __other) -> __basic_any& = default; - auto operator=(__basic_any const& __other) -> __basic_any& = default; + _CCCL_HIDE_FROM_ABI auto operator=(__basic_any&& __other) -> __basic_any& = default; + _CCCL_HIDE_FROM_ABI auto operator=(__basic_any const& __other) -> __basic_any& = default; #if !_CCCL_CUDA_COMPILER(NVCC, <, 12, 9) //! @brief Converting move assignment operator from a compatible `__basic_any` diff --git a/libcudacxx/include/cuda/__utility/__basic_any/conversions.h b/libcudacxx/include/cuda/__utility/__basic_any/conversions.h index 7f954063e387..efafdfdbf929 100644 --- a/libcudacxx/include/cuda/__utility/__basic_any/conversions.h +++ b/libcudacxx/include/cuda/__utility/__basic_any/conversions.h @@ -47,9 +47,9 @@ struct __archetype; template <> struct __archetype // immovable archetype { - __archetype() = default; - __archetype(__archetype&&) = delete; - __archetype(const __archetype&) = delete; + _CCCL_HIDE_FROM_ABI __archetype() = default; + __archetype(__archetype&&) = delete; + __archetype(const __archetype&) = delete; template _CCCL_HOST_DEVICE_API __archetype(_Value) noexcept; @@ -61,7 +61,7 @@ struct __archetype // immovable archetype template <> struct __archetype : __archetype // movable archetype { - __archetype() = default; + _CCCL_HIDE_FROM_ABI __archetype() = default; _CCCL_HOST_DEVICE_API __archetype(__archetype&&) noexcept; __archetype(const __archetype&) = delete; }; @@ -70,7 +70,7 @@ struct __archetype : __archetype // movable archetype template <> struct __archetype : __archetype { - __archetype() = default; + _CCCL_HIDE_FROM_ABI __archetype() = default; _CCCL_HOST_DEVICE_API __archetype(__archetype const&); }; diff --git a/libcudacxx/include/cuda/__utility/__basic_any/iset.h b/libcudacxx/include/cuda/__utility/__basic_any/iset.h index c97f2a93f574..7705d1228ff1 100644 --- a/libcudacxx/include/cuda/__utility/__basic_any/iset.h +++ b/libcudacxx/include/cuda/__utility/__basic_any/iset.h @@ -113,7 +113,7 @@ struct __iset_vptr : __base_vptr { using __iset_vtable _CCCL_NODEBUG = __vtable_for<__iset_<_Interfaces...>>; - __iset_vptr() = default; + _CCCL_HIDE_FROM_ABI __iset_vptr() = default; _CCCL_HOST_DEVICE_API constexpr __iset_vptr(__iset_vtable const* __vptr) noexcept : __base_vptr(__vptr) diff --git a/libcudacxx/include/cuda/__utility/__basic_any/virtual_ptrs.h b/libcudacxx/include/cuda/__utility/__basic_any/virtual_ptrs.h index eb476df42540..40bb8237874a 100644 --- a/libcudacxx/include/cuda/__utility/__basic_any/virtual_ptrs.h +++ b/libcudacxx/include/cuda/__utility/__basic_any/virtual_ptrs.h @@ -30,7 +30,7 @@ _CCCL_BEGIN_NAMESPACE_CUDA struct __base_vptr { - __base_vptr() = default; + _CCCL_HIDE_FROM_ABI __base_vptr() = default; _CCCL_NODEBUG_HOST_DEVICE_API constexpr __base_vptr(__rtti_base const* __vptr) noexcept : __vptr_(__vptr) @@ -55,7 +55,7 @@ struct __base_vptr } #if !defined(_CCCL_NO_THREE_WAY_COMPARISON) - bool operator==(__base_vptr const& __other) const noexcept = default; + _CCCL_HIDE_FROM_ABI bool operator==(__base_vptr const& __other) const noexcept = default; #else // ^^^ !_CCCL_NO_THREE_WAY_COMPARISON ^^^ / vvv _CCCL_NO_THREE_WAY_COMPARISON vvv [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr auto operator==(__base_vptr __lhs, __base_vptr __rhs) noexcept -> bool diff --git a/libcudacxx/include/cuda/std/__complex/complex.h b/libcudacxx/include/cuda/std/__complex/complex.h index b8ffc2da0613..03fa2d6badf0 100644 --- a/libcudacxx/include/cuda/std/__complex/complex.h +++ b/libcudacxx/include/cuda/std/__complex/complex.h @@ -686,7 +686,8 @@ template #if _CCCL_HOSTED() template -::std::basic_istream<_CharT, _Traits>& operator>>(::std::basic_istream<_CharT, _Traits>& __is, complex<_Tp>& __x) +_CCCL_HOST_API ::std::basic_istream<_CharT, _Traits>& +operator>>(::std::basic_istream<_CharT, _Traits>& __is, complex<_Tp>& __x) { ::std::complex<_Tp> __temp; __is >> __temp; @@ -695,7 +696,8 @@ ::std::basic_istream<_CharT, _Traits>& operator>>(::std::basic_istream<_CharT, _ } template -::std::basic_ostream<_CharT, _Traits>& operator<<(::std::basic_ostream<_CharT, _Traits>& __os, const complex<_Tp>& __x) +_CCCL_HOST_API ::std::basic_ostream<_CharT, _Traits>& +operator<<(::std::basic_ostream<_CharT, _Traits>& __os, const complex<_Tp>& __x) { return __os << static_cast<::std::complex<_Tp>>(__x); } diff --git a/libcudacxx/include/cuda/std/__complex/nvbf16.h b/libcudacxx/include/cuda/std/__complex/nvbf16.h index ba4df1a1441d..d15bf94a2d20 100644 --- a/libcudacxx/include/cuda/std/__complex/nvbf16.h +++ b/libcudacxx/include/cuda/std/__complex/nvbf16.h @@ -340,7 +340,7 @@ struct __get_complex_impl<__nv_bfloat16> # if _CCCL_HOSTED() template -::std::basic_istream<_CharT, _Traits>& +_CCCL_HOST_API ::std::basic_istream<_CharT, _Traits>& operator>>(::std::basic_istream<_CharT, _Traits>& __is, complex<__nv_bfloat16>& __x) { ::std::complex __temp; @@ -350,7 +350,7 @@ operator>>(::std::basic_istream<_CharT, _Traits>& __is, complex<__nv_bfloat16>& } template -::std::basic_ostream<_CharT, _Traits>& +_CCCL_HOST_API ::std::basic_ostream<_CharT, _Traits>& operator<<(::std::basic_ostream<_CharT, _Traits>& __os, const complex<__nv_bfloat16>& __x) { return __os << complex{__x}; diff --git a/libcudacxx/include/cuda/std/__complex/nvfp16.h b/libcudacxx/include/cuda/std/__complex/nvfp16.h index c4be713dc8d1..b4a18e2807c6 100644 --- a/libcudacxx/include/cuda/std/__complex/nvfp16.h +++ b/libcudacxx/include/cuda/std/__complex/nvfp16.h @@ -340,7 +340,8 @@ struct __get_complex_impl<__half> # if _CCCL_HOSTED() template -::std::basic_istream<_CharT, _Traits>& operator>>(::std::basic_istream<_CharT, _Traits>& __is, complex<__half>& __x) +_CCCL_HOST_API ::std::basic_istream<_CharT, _Traits>& +operator>>(::std::basic_istream<_CharT, _Traits>& __is, complex<__half>& __x) { ::std::complex __temp; __is >> __temp; @@ -349,7 +350,7 @@ ::std::basic_istream<_CharT, _Traits>& operator>>(::std::basic_istream<_CharT, _ } template -::std::basic_ostream<_CharT, _Traits>& +_CCCL_HOST_API ::std::basic_ostream<_CharT, _Traits>& operator<<(::std::basic_ostream<_CharT, _Traits>& __os, const complex<__half>& __x) { return __os << complex{__x}; diff --git a/libcudacxx/include/cuda/std/__expected/bad_expected_access.h b/libcudacxx/include/cuda/std/__expected/bad_expected_access.h index 52b29677d684..0bf4568ed42d 100644 --- a/libcudacxx/include/cuda/std/__expected/bad_expected_access.h +++ b/libcudacxx/include/cuda/std/__expected/bad_expected_access.h @@ -56,7 +56,7 @@ class bad_expected_access : public ::std::exception // have a profusion of these vtables in TUs, and the dynamic linker will already have a bunch // of work to do. So it is not worth hiding the specialization in the dylib, given that // it adds deployment target restrictions. - const char* what() const noexcept override + _CCCL_HOST_API const char* what() const noexcept override { return "bad access to cuda::std::expected"; } diff --git a/libcudacxx/include/cuda/std/__optional/optional.h b/libcudacxx/include/cuda/std/__optional/optional.h index 5e82dd8a5dca..a147317e3686 100644 --- a/libcudacxx/include/cuda/std/__optional/optional.h +++ b/libcudacxx/include/cuda/std/__optional/optional.h @@ -209,8 +209,8 @@ class optional : private __optional_move_assign_base<_Tp> return *this; } - constexpr optional& operator=(const optional&) = default; - constexpr optional& operator=(optional&&) = default; + _CCCL_HIDE_FROM_ABI constexpr optional& operator=(const optional&) = default; + _CCCL_HIDE_FROM_ABI constexpr optional& operator=(optional&&) = default; _CCCL_EXEC_CHECK_DISABLE _CCCL_TEMPLATE(class _Up = value_type) diff --git a/libcudacxx/include/cuda/std/__optional/optional_base.h b/libcudacxx/include/cuda/std/__optional/optional_base.h index 951b4e7ecc0b..c72fd58a80b6 100644 --- a/libcudacxx/include/cuda/std/__optional/optional_base.h +++ b/libcudacxx/include/cuda/std/__optional/optional_base.h @@ -375,7 +375,7 @@ struct __optional_copy_assign_base<_Tp, __smf_availability::__available> : __opt return *this; } - __optional_copy_assign_base& operator=(__optional_copy_assign_base&&) = default; + _CCCL_HIDE_FROM_ABI __optional_copy_assign_base& operator=(__optional_copy_assign_base&&) = default; }; template diff --git a/libcudacxx/include/cuda/std/__pstl/cuda/sort.h b/libcudacxx/include/cuda/std/__pstl/cuda/sort.h index 1b59392c9fac..c1f6b7d13371 100644 --- a/libcudacxx/include/cuda/std/__pstl/cuda/sort.h +++ b/libcudacxx/include/cuda/std/__pstl/cuda/sort.h @@ -78,7 +78,7 @@ struct __pstl_dispatch<__pstl_algorithm::__sort, __execution_backend::__cuda> cudaError_t (*)(void*, size_t&, CUB_NS_QUALIFIER::DoubleBuffer<_Tp>&, size_t, int, int, const _Policy&); template - [[nodiscard]] static _CCCL_CONSTEVAL _DeviceRadixSort<_Tp, _Policy> __select_radix_impl() noexcept + [[nodiscard]] _CCCL_HOST_API static _CCCL_CONSTEVAL _DeviceRadixSort<_Tp, _Policy> __select_radix_impl() noexcept { if constexpr (__is_one_of_v, less<>, less<_Tp>>) { diff --git a/libcudacxx/include/cuda/std/__random/bernoulli_distribution.h b/libcudacxx/include/cuda/std/__random/bernoulli_distribution.h index 56d7d5c24d85..4a3ce6da134e 100644 --- a/libcudacxx/include/cuda/std/__random/bernoulli_distribution.h +++ b/libcudacxx/include/cuda/std/__random/bernoulli_distribution.h @@ -132,7 +132,7 @@ class bernoulli_distribution #endif // _CCCL_STD_VER <= 2017 #if _CCCL_HOSTED() template - friend ::std::basic_ostream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_ostream<_CharT, _Traits>& operator<<(::std::basic_ostream<_CharT, _Traits>& __os, const bernoulli_distribution& __x) { using ostream_type = ::std::basic_ostream<_CharT, _Traits>; @@ -146,7 +146,7 @@ class bernoulli_distribution } template - friend ::std::basic_istream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_istream<_CharT, _Traits>& operator>>(::std::basic_istream<_CharT, _Traits>& __is, bernoulli_distribution& __x) { using istream_type = ::std::basic_istream<_CharT, _Traits>; diff --git a/libcudacxx/include/cuda/std/__random/binomial_distribution.h b/libcudacxx/include/cuda/std/__random/binomial_distribution.h index a137b09fedf7..2c46f2002cd1 100644 --- a/libcudacxx/include/cuda/std/__random/binomial_distribution.h +++ b/libcudacxx/include/cuda/std/__random/binomial_distribution.h @@ -216,7 +216,7 @@ class binomial_distribution #if _CCCL_HOSTED() template - friend ::std::basic_ostream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_ostream<_CharT, _Traits>& operator<<(::std::basic_ostream<_CharT, _Traits>& __os, const binomial_distribution& __x) { using ostream_type = ::std::basic_ostream<_CharT, _Traits>; @@ -230,7 +230,7 @@ class binomial_distribution } template - friend ::std::basic_istream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_istream<_CharT, _Traits>& operator>>(::std::basic_istream<_CharT, _Traits>& __is, binomial_distribution& __x) { using istream_type = ::std::basic_istream<_CharT, _Traits>; diff --git a/libcudacxx/include/cuda/std/__random/cauchy_distribution.h b/libcudacxx/include/cuda/std/__random/cauchy_distribution.h index 7f6b541ec0f3..6161475b3798 100644 --- a/libcudacxx/include/cuda/std/__random/cauchy_distribution.h +++ b/libcudacxx/include/cuda/std/__random/cauchy_distribution.h @@ -80,7 +80,7 @@ class cauchy_distribution public: // constructor and reset functions - constexpr cauchy_distribution() noexcept = default; + _CCCL_HIDE_FROM_ABI constexpr cauchy_distribution() noexcept = default; _CCCL_HOST_DEVICE_API constexpr explicit cauchy_distribution(result_type __a, result_type __b = 1) noexcept : __p_{param_type{__a, __b}} {} @@ -147,7 +147,7 @@ class cauchy_distribution #if _CCCL_HOSTED() template - friend ::std::basic_ostream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_ostream<_CharT, _Traits>& operator<<(::std::basic_ostream<_CharT, _Traits>& __os, const cauchy_distribution& __x) { using ostream_type = ::std::basic_ostream<_CharT, _Traits>; @@ -167,7 +167,7 @@ class cauchy_distribution } template - friend ::std::basic_istream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_istream<_CharT, _Traits>& operator>>(::std::basic_istream<_CharT, _Traits>& __is, cauchy_distribution& __x) { using istream_type = ::std::basic_istream<_CharT, _Traits>; diff --git a/libcudacxx/include/cuda/std/__random/chi_squared_distribution.h b/libcudacxx/include/cuda/std/__random/chi_squared_distribution.h index 583d09c951ac..dad98ef0c2de 100644 --- a/libcudacxx/include/cuda/std/__random/chi_squared_distribution.h +++ b/libcudacxx/include/cuda/std/__random/chi_squared_distribution.h @@ -44,7 +44,7 @@ class chi_squared_distribution public: using distribution_type = chi_squared_distribution; - constexpr param_type() noexcept = default; + _CCCL_HIDE_FROM_ABI constexpr param_type() noexcept = default; _CCCL_HOST_DEVICE_API constexpr explicit param_type(result_type __n) noexcept : __n_{__n} @@ -74,7 +74,7 @@ class chi_squared_distribution public: // constructor and reset functions - constexpr chi_squared_distribution() noexcept = default; + _CCCL_HIDE_FROM_ABI constexpr chi_squared_distribution() noexcept = default; _CCCL_HOST_DEVICE_API constexpr explicit chi_squared_distribution(result_type __n) noexcept : __p_{param_type{__n}} @@ -136,7 +136,7 @@ class chi_squared_distribution #if _CCCL_HOSTED() template - friend ::std::basic_ostream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_ostream<_CharT, _Traits>& operator<<(::std::basic_ostream<_CharT, _Traits>& __os, const chi_squared_distribution& __x) { using ostream_type = ::std::basic_ostream<_CharT, _Traits>; @@ -154,7 +154,7 @@ class chi_squared_distribution } template - friend ::std::basic_istream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_istream<_CharT, _Traits>& operator>>(::std::basic_istream<_CharT, _Traits>& __is, chi_squared_distribution& __x) { using istream_type = ::std::basic_istream<_CharT, _Traits>; diff --git a/libcudacxx/include/cuda/std/__random/exponential_distribution.h b/libcudacxx/include/cuda/std/__random/exponential_distribution.h index cc503f3c396a..f53951471a76 100644 --- a/libcudacxx/include/cuda/std/__random/exponential_distribution.h +++ b/libcudacxx/include/cuda/std/__random/exponential_distribution.h @@ -140,7 +140,7 @@ class exponential_distribution #if _CCCL_HOSTED() template - friend ::std::basic_ostream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_ostream<_CharT, _Traits>& operator<<(::std::basic_ostream<_CharT, _Traits>& __os, const exponential_distribution& __x) { using ostream_type = ::std::basic_ostream<_CharT, _Traits>; @@ -158,7 +158,7 @@ class exponential_distribution } template - friend ::std::basic_istream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_istream<_CharT, _Traits>& operator>>(::std::basic_istream<_CharT, _Traits>& __is, exponential_distribution& __x) { using istream_type = ::std::basic_istream<_CharT, _Traits>; diff --git a/libcudacxx/include/cuda/std/__random/extreme_value_distribution.h b/libcudacxx/include/cuda/std/__random/extreme_value_distribution.h index 00739f87a82a..6958e08cd28f 100644 --- a/libcudacxx/include/cuda/std/__random/extreme_value_distribution.h +++ b/libcudacxx/include/cuda/std/__random/extreme_value_distribution.h @@ -150,7 +150,7 @@ class extreme_value_distribution #if _CCCL_HOSTED() template - friend ::std::basic_ostream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_ostream<_CharT, _Traits>& operator<<(::std::basic_ostream<_CharT, _Traits>& __os, const extreme_value_distribution& __x) { using ostream_type = ::std::basic_ostream<_CharT, _Traits>; @@ -170,7 +170,7 @@ class extreme_value_distribution } template - friend ::std::basic_istream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_istream<_CharT, _Traits>& operator>>(::std::basic_istream<_CharT, _Traits>& __is, extreme_value_distribution& __x) { using istream_type = ::std::basic_istream<_CharT, _Traits>; diff --git a/libcudacxx/include/cuda/std/__random/fisher_f_distribution.h b/libcudacxx/include/cuda/std/__random/fisher_f_distribution.h index 1897f2fb6e99..0176bacb06db 100644 --- a/libcudacxx/include/cuda/std/__random/fisher_f_distribution.h +++ b/libcudacxx/include/cuda/std/__random/fisher_f_distribution.h @@ -81,7 +81,7 @@ class fisher_f_distribution public: // constructor and reset functions - constexpr fisher_f_distribution() = default; + _CCCL_HIDE_FROM_ABI constexpr fisher_f_distribution() = default; _CCCL_HOST_DEVICE_API constexpr explicit fisher_f_distribution(result_type __m, result_type __n = result_type{1}) : __p_{param_type{__m, __n}} @@ -149,7 +149,7 @@ class fisher_f_distribution #if _CCCL_HOSTED() template - friend ::std::basic_ostream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_ostream<_CharT, _Traits>& operator<<(::std::basic_ostream<_CharT, _Traits>& __os, const fisher_f_distribution& __x) { using ostream_type = ::std::basic_ostream<_CharT, _Traits>; @@ -169,7 +169,7 @@ class fisher_f_distribution } template - friend ::std::basic_istream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_istream<_CharT, _Traits>& operator>>(::std::basic_istream<_CharT, _Traits>& __is, fisher_f_distribution& __x) { using istream_type = ::std::basic_istream<_CharT, _Traits>; diff --git a/libcudacxx/include/cuda/std/__random/gamma_distribution.h b/libcudacxx/include/cuda/std/__random/gamma_distribution.h index 02b339665dad..dd051ed33f76 100644 --- a/libcudacxx/include/cuda/std/__random/gamma_distribution.h +++ b/libcudacxx/include/cuda/std/__random/gamma_distribution.h @@ -85,7 +85,7 @@ class gamma_distribution public: // constructors and reset functions - constexpr gamma_distribution() = default; + _CCCL_HIDE_FROM_ABI constexpr gamma_distribution() = default; _CCCL_HOST_DEVICE_API constexpr explicit gamma_distribution( result_type __alpha, result_type __beta = result_type{1}) noexcept : __p_{param_type{__alpha, __beta}} @@ -212,7 +212,7 @@ class gamma_distribution #if _CCCL_HOSTED() template - friend ::std::basic_ostream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_ostream<_CharT, _Traits>& operator<<(::std::basic_ostream<_CharT, _Traits>& __os, const gamma_distribution& __x) { using ostream_type = ::std::basic_ostream<_CharT, _Traits>; @@ -232,7 +232,7 @@ class gamma_distribution } template - friend ::std::basic_istream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_istream<_CharT, _Traits>& operator>>(::std::basic_istream<_CharT, _Traits>& __is, gamma_distribution& __x) { using istream_type = ::std::basic_istream<_CharT, _Traits>; diff --git a/libcudacxx/include/cuda/std/__random/geometric_distribution.h b/libcudacxx/include/cuda/std/__random/geometric_distribution.h index 290cb0bfdcb2..1af12e944b73 100644 --- a/libcudacxx/include/cuda/std/__random/geometric_distribution.h +++ b/libcudacxx/include/cuda/std/__random/geometric_distribution.h @@ -46,7 +46,7 @@ class geometric_distribution public: using distribution_type = geometric_distribution; - param_type() noexcept = default; + _CCCL_HIDE_FROM_ABI param_type() noexcept = default; _CCCL_HOST_DEVICE_API constexpr explicit param_type(double __p) noexcept : __p_{__p} @@ -76,7 +76,7 @@ class geometric_distribution public: // constructors and reset functions - constexpr geometric_distribution() noexcept = default; + _CCCL_HIDE_FROM_ABI constexpr geometric_distribution() noexcept = default; _CCCL_HOST_DEVICE_API constexpr explicit geometric_distribution(double __p) noexcept : __p_{__p} {} @@ -136,7 +136,7 @@ class geometric_distribution #if _CCCL_HOSTED() template - friend ::std::basic_ostream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_ostream<_CharT, _Traits>& operator<<(::std::basic_ostream<_CharT, _Traits>& __os, const geometric_distribution& __x) { using _Ostream = ::std::basic_ostream<_CharT, _Traits>; @@ -153,7 +153,7 @@ class geometric_distribution } template - friend ::std::basic_istream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_istream<_CharT, _Traits>& operator>>(::std::basic_istream<_CharT, _Traits>& __is, geometric_distribution& __x) { using _Istream = ::std::basic_istream<_CharT, _Traits>; diff --git a/libcudacxx/include/cuda/std/__random/lognormal_distribution.h b/libcudacxx/include/cuda/std/__random/lognormal_distribution.h index 419cfb159ee2..40103ce1488b 100644 --- a/libcudacxx/include/cuda/std/__random/lognormal_distribution.h +++ b/libcudacxx/include/cuda/std/__random/lognormal_distribution.h @@ -82,7 +82,7 @@ class lognormal_distribution public: // constructor and reset functions - constexpr lognormal_distribution() noexcept = default; + _CCCL_HIDE_FROM_ABI constexpr lognormal_distribution() noexcept = default; _CCCL_HOST_DEVICE_API constexpr explicit lognormal_distribution( result_type __m, result_type __s = result_type{1}) noexcept : __nd_{__m, __s} @@ -153,14 +153,14 @@ class lognormal_distribution #if _CCCL_HOSTED() template - friend ::std::basic_ostream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_ostream<_CharT, _Traits>& operator<<(::std::basic_ostream<_CharT, _Traits>& __os, const lognormal_distribution& __x) { return __os << __x.__nd_; } template - friend ::std::basic_istream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_istream<_CharT, _Traits>& operator>>(::std::basic_istream<_CharT, _Traits>& __is, lognormal_distribution& __x) { return __is >> __x.__nd_; diff --git a/libcudacxx/include/cuda/std/__random/negative_binomial_distribution.h b/libcudacxx/include/cuda/std/__random/negative_binomial_distribution.h index 8a4724ce0588..7a2999b094cd 100644 --- a/libcudacxx/include/cuda/std/__random/negative_binomial_distribution.h +++ b/libcudacxx/include/cuda/std/__random/negative_binomial_distribution.h @@ -82,7 +82,7 @@ class negative_binomial_distribution public: // constructor and reset functions - constexpr negative_binomial_distribution() noexcept = default; + _CCCL_HIDE_FROM_ABI constexpr negative_binomial_distribution() noexcept = default; _CCCL_HOST_DEVICE_API constexpr explicit negative_binomial_distribution(result_type __k, double __p = 0.5) noexcept : __p_{__k, __p} @@ -170,7 +170,7 @@ class negative_binomial_distribution #if _CCCL_HOSTED() template - friend ::std::basic_ostream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_ostream<_CharT, _Traits>& operator<<(::std::basic_ostream<_CharT, _Traits>& __os, const negative_binomial_distribution& __x) { using _Ostream = ::std::basic_ostream<_CharT, _Traits>; @@ -187,7 +187,7 @@ class negative_binomial_distribution } template - friend ::std::basic_istream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_istream<_CharT, _Traits>& operator>>(::std::basic_istream<_CharT, _Traits>& __is, negative_binomial_distribution& __x) { using _Istream = ::std::basic_istream<_CharT, _Traits>; diff --git a/libcudacxx/include/cuda/std/__random/normal_distribution.h b/libcudacxx/include/cuda/std/__random/normal_distribution.h index 695359d566f9..542975f62e47 100644 --- a/libcudacxx/include/cuda/std/__random/normal_distribution.h +++ b/libcudacxx/include/cuda/std/__random/normal_distribution.h @@ -179,7 +179,7 @@ class normal_distribution #if _CCCL_HOSTED() template - friend ::std::basic_ostream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_ostream<_CharT, _Traits>& operator<<(::std::basic_ostream<_CharT, _Traits>& __os, const normal_distribution& __x) { _CharT __sp = __os.widen(' '); @@ -199,7 +199,7 @@ class normal_distribution } template - friend ::std::basic_istream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_istream<_CharT, _Traits>& operator>>(::std::basic_istream<_CharT, _Traits>& __is, normal_distribution& __x) { using _Istream = ::std::basic_istream<_CharT, _Traits>; diff --git a/libcudacxx/include/cuda/std/__random/poisson_distribution.h b/libcudacxx/include/cuda/std/__random/poisson_distribution.h index 3f1bab46a59c..224c00b53700 100644 --- a/libcudacxx/include/cuda/std/__random/poisson_distribution.h +++ b/libcudacxx/include/cuda/std/__random/poisson_distribution.h @@ -311,7 +311,7 @@ class poisson_distribution #if _CCCL_HOSTED() template - friend ::std::basic_ostream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_ostream<_CharT, _Traits>& operator<<(::std::basic_ostream<_CharT, _Traits>& __os, const poisson_distribution& __x) { using ostream_type = ::std::basic_ostream<_CharT, _Traits>; @@ -324,7 +324,7 @@ class poisson_distribution } template - friend ::std::basic_istream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_istream<_CharT, _Traits>& operator>>(::std::basic_istream<_CharT, _Traits>& __is, poisson_distribution& __x) { using istream_type = ::std::basic_istream<_CharT, _Traits>; diff --git a/libcudacxx/include/cuda/std/__random/student_t_distribution.h b/libcudacxx/include/cuda/std/__random/student_t_distribution.h index 6dd99d46350e..9a54d414f3ac 100644 --- a/libcudacxx/include/cuda/std/__random/student_t_distribution.h +++ b/libcudacxx/include/cuda/std/__random/student_t_distribution.h @@ -77,7 +77,7 @@ class student_t_distribution public: // constructor and reset functions - constexpr student_t_distribution() noexcept = default; + _CCCL_HIDE_FROM_ABI constexpr student_t_distribution() noexcept = default; _CCCL_HOST_DEVICE_API constexpr explicit student_t_distribution(result_type __n) noexcept : __p_{param_type{__n}} @@ -143,7 +143,7 @@ class student_t_distribution #if _CCCL_HOSTED() template - friend ::std::basic_ostream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_ostream<_CharT, _Traits>& operator<<(::std::basic_ostream<_CharT, _Traits>& __os, const student_t_distribution& __x) { using _Ostream = ::std::basic_ostream<_CharT, _Traits>; @@ -160,7 +160,7 @@ class student_t_distribution } template - friend ::std::basic_istream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_istream<_CharT, _Traits>& operator>>(::std::basic_istream<_CharT, _Traits>& __is, student_t_distribution& __x) { using _Istream = ::std::basic_istream<_CharT, _Traits>; diff --git a/libcudacxx/include/cuda/std/__random/uniform_int_distribution.h b/libcudacxx/include/cuda/std/__random/uniform_int_distribution.h index d627f41e1f68..79b6225a4df1 100644 --- a/libcudacxx/include/cuda/std/__random/uniform_int_distribution.h +++ b/libcudacxx/include/cuda/std/__random/uniform_int_distribution.h @@ -212,7 +212,7 @@ class uniform_int_distribution public: // constructors and reset functions - constexpr uniform_int_distribution() noexcept = default; + _CCCL_HIDE_FROM_ABI constexpr uniform_int_distribution() noexcept = default; _CCCL_HOST_DEVICE_API explicit constexpr uniform_int_distribution( result_type __a, result_type __b = numeric_limits::max()) noexcept : __p_(param_type(__a, __b)) @@ -305,7 +305,7 @@ class uniform_int_distribution #if _CCCL_HOSTED() template - friend ::std::basic_ostream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_ostream<_CharT, _Traits>& operator<<(::std::basic_ostream<_CharT, _Traits>& __os, const uniform_int_distribution& __x) { using _Ostream = ::std::basic_ostream<_CharT, _Traits>; @@ -320,7 +320,7 @@ class uniform_int_distribution } template - friend ::std::basic_istream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_istream<_CharT, _Traits>& operator>>(::std::basic_istream<_CharT, _Traits>& __is, uniform_int_distribution& __x) { using _Istream = ::std::basic_istream<_CharT, _Traits>; diff --git a/libcudacxx/include/cuda/std/__random/uniform_real_distribution.h b/libcudacxx/include/cuda/std/__random/uniform_real_distribution.h index e9f6a40518a5..98ae876b1a18 100644 --- a/libcudacxx/include/cuda/std/__random/uniform_real_distribution.h +++ b/libcudacxx/include/cuda/std/__random/uniform_real_distribution.h @@ -81,7 +81,7 @@ class uniform_real_distribution public: // constructors and reset functions - constexpr uniform_real_distribution() noexcept = default; + _CCCL_HIDE_FROM_ABI constexpr uniform_real_distribution() noexcept = default; _CCCL_HOST_DEVICE_API constexpr explicit uniform_real_distribution(result_type __a, result_type __b = 1) noexcept : __p_{param_type(__a, __b)} {} @@ -149,7 +149,7 @@ class uniform_real_distribution #if _CCCL_HOSTED() template - friend ::std::basic_ostream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_ostream<_CharT, _Traits>& operator<<(::std::basic_ostream<_CharT, _Traits>& __os, const uniform_real_distribution& __x) { using _Ostream = ::std::basic_ostream<_CharT, _Traits>; @@ -166,7 +166,7 @@ class uniform_real_distribution } template - friend ::std::basic_istream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_istream<_CharT, _Traits>& operator>>(::std::basic_istream<_CharT, _Traits>& __is, uniform_real_distribution& __x) { using _Istream = ::std::basic_istream<_CharT, _Traits>; diff --git a/libcudacxx/include/cuda/std/__random/weibull_distribution.h b/libcudacxx/include/cuda/std/__random/weibull_distribution.h index a23c5da8249e..bf64b6b50309 100644 --- a/libcudacxx/include/cuda/std/__random/weibull_distribution.h +++ b/libcudacxx/include/cuda/std/__random/weibull_distribution.h @@ -82,7 +82,7 @@ class weibull_distribution public: // constructor and reset functions - constexpr weibull_distribution() noexcept = default; + _CCCL_HIDE_FROM_ABI constexpr weibull_distribution() noexcept = default; _CCCL_HOST_DEVICE_API constexpr explicit weibull_distribution( result_type __a, result_type __b = result_type{1}) noexcept : __p_{param_type{__a, __b}} @@ -148,7 +148,7 @@ class weibull_distribution #if _CCCL_HOSTED() template - friend ::std::basic_ostream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_ostream<_CharT, _Traits>& operator<<(::std::basic_ostream<_CharT, _Traits>& __os, const weibull_distribution& __x) { using _Ostream = ::std::basic_ostream<_CharT, _Traits>; @@ -165,7 +165,7 @@ class weibull_distribution } template - friend ::std::basic_istream<_CharT, _Traits>& + _CCCL_HOST_API friend ::std::basic_istream<_CharT, _Traits>& operator>>(::std::basic_istream<_CharT, _Traits>& __is, weibull_distribution& __x) { using _Istream = ::std::basic_istream<_CharT, _Traits>; diff --git a/libcudacxx/include/cuda/std/__tuple_dir/ignore.h b/libcudacxx/include/cuda/std/__tuple_dir/ignore.h index 0dd1ce898715..5304a070be3d 100644 --- a/libcudacxx/include/cuda/std/__tuple_dir/ignore.h +++ b/libcudacxx/include/cuda/std/__tuple_dir/ignore.h @@ -26,7 +26,7 @@ _CCCL_BEGIN_NAMESPACE_CUDA_STD struct __ignore_t { - __ignore_t() = default; + _CCCL_HIDE_FROM_ABI __ignore_t() = default; template _CCCL_API constexpr __ignore_t(const _Tp&, const _Ts&...) noexcept diff --git a/libcudacxx/include/cuda/std/__tuple_dir/sfinae_helpers.h b/libcudacxx/include/cuda/std/__tuple_dir/sfinae_helpers.h index bea39775c044..37d60d73dc7c 100644 --- a/libcudacxx/include/cuda/std/__tuple_dir/sfinae_helpers.h +++ b/libcudacxx/include/cuda/std/__tuple_dir/sfinae_helpers.h @@ -57,31 +57,31 @@ struct _CCCL_DECLSPEC_EMPTY_BASES __tuple_impl_sfinae_helper template struct _CCCL_DECLSPEC_EMPTY_BASES __tuple_impl_sfinae_helper<_Impl, false, true> { - __tuple_impl_sfinae_helper() = default; - __tuple_impl_sfinae_helper(const __tuple_impl_sfinae_helper&) = default; - __tuple_impl_sfinae_helper(__tuple_impl_sfinae_helper&&) = default; - __tuple_impl_sfinae_helper& operator=(const __tuple_impl_sfinae_helper&) = delete; - __tuple_impl_sfinae_helper& operator=(__tuple_impl_sfinae_helper&&) = default; + _CCCL_HIDE_FROM_ABI __tuple_impl_sfinae_helper() = default; + _CCCL_HIDE_FROM_ABI __tuple_impl_sfinae_helper(const __tuple_impl_sfinae_helper&) = default; + _CCCL_HIDE_FROM_ABI __tuple_impl_sfinae_helper(__tuple_impl_sfinae_helper&&) = default; + __tuple_impl_sfinae_helper& operator=(const __tuple_impl_sfinae_helper&) = delete; + _CCCL_HIDE_FROM_ABI __tuple_impl_sfinae_helper& operator=(__tuple_impl_sfinae_helper&&) = default; }; template struct _CCCL_DECLSPEC_EMPTY_BASES __tuple_impl_sfinae_helper<_Impl, true, false> { - __tuple_impl_sfinae_helper() = default; - __tuple_impl_sfinae_helper(const __tuple_impl_sfinae_helper&) = default; - __tuple_impl_sfinae_helper(__tuple_impl_sfinae_helper&&) = default; - __tuple_impl_sfinae_helper& operator=(const __tuple_impl_sfinae_helper&) = default; - __tuple_impl_sfinae_helper& operator=(__tuple_impl_sfinae_helper&&) = delete; + _CCCL_HIDE_FROM_ABI __tuple_impl_sfinae_helper() = default; + _CCCL_HIDE_FROM_ABI __tuple_impl_sfinae_helper(const __tuple_impl_sfinae_helper&) = default; + _CCCL_HIDE_FROM_ABI __tuple_impl_sfinae_helper(__tuple_impl_sfinae_helper&&) = default; + _CCCL_HIDE_FROM_ABI __tuple_impl_sfinae_helper& operator=(const __tuple_impl_sfinae_helper&) = default; + __tuple_impl_sfinae_helper& operator=(__tuple_impl_sfinae_helper&&) = delete; }; template struct _CCCL_DECLSPEC_EMPTY_BASES __tuple_impl_sfinae_helper<_Impl, false, false> { - __tuple_impl_sfinae_helper() = default; - __tuple_impl_sfinae_helper(const __tuple_impl_sfinae_helper&) = default; - __tuple_impl_sfinae_helper(__tuple_impl_sfinae_helper&&) = default; - __tuple_impl_sfinae_helper& operator=(const __tuple_impl_sfinae_helper&) = delete; - __tuple_impl_sfinae_helper& operator=(__tuple_impl_sfinae_helper&&) = delete; + _CCCL_HIDE_FROM_ABI __tuple_impl_sfinae_helper() = default; + _CCCL_HIDE_FROM_ABI __tuple_impl_sfinae_helper(const __tuple_impl_sfinae_helper&) = default; + _CCCL_HIDE_FROM_ABI __tuple_impl_sfinae_helper(__tuple_impl_sfinae_helper&&) = default; + __tuple_impl_sfinae_helper& operator=(const __tuple_impl_sfinae_helper&) = delete; + __tuple_impl_sfinae_helper& operator=(__tuple_impl_sfinae_helper&&) = delete; }; _CCCL_END_NAMESPACE_CUDA_STD diff --git a/libcudacxx/include/cuda/std/__tuple_dir/tuple_leaf.h b/libcudacxx/include/cuda/std/__tuple_dir/tuple_leaf.h index c0ea5d235a40..74f00d4edc3d 100644 --- a/libcudacxx/include/cuda/std/__tuple_dir/tuple_leaf.h +++ b/libcudacxx/include/cuda/std/__tuple_dir/tuple_leaf.h @@ -241,9 +241,9 @@ class __tuple_leaf<_Ip, _Hp, __tuple_leaf_specialization::__synthesize_assignmen } _CCCL_EXEC_CHECK_DISABLE - __tuple_leaf(const __tuple_leaf& __t) = default; + _CCCL_HIDE_FROM_ABI __tuple_leaf(const __tuple_leaf& __t) = default; _CCCL_EXEC_CHECK_DISABLE - __tuple_leaf(__tuple_leaf&& __t) = default; + _CCCL_HIDE_FROM_ABI __tuple_leaf(__tuple_leaf&& __t) = default; // Do not use = default here. The value type may be a reference, in which case the defaulted // assignment constructor is implicitly deleted (a quirk in the C++ spec meant). diff --git a/libcudacxx/include/cuda/std/__utility/typeid.h b/libcudacxx/include/cuda/std/__utility/typeid.h index b69be9c38092..e3bfc57f16f6 100644 --- a/libcudacxx/include/cuda/std/__utility/typeid.h +++ b/libcudacxx/include/cuda/std/__utility/typeid.h @@ -374,7 +374,7 @@ struct __type_info friend struct __type_info_ptr_; friend struct __type_info_ref_; - __type_info(__type_info const&) = default; // needed by __type_info_ptr_::operator*() before C++17 + _CCCL_HIDE_FROM_ABI __type_info(__type_info const&) = default; // needed by __type_info_ptr_::operator*() before C++17 __type_info_impl (*__pfn_)() noexcept; }; diff --git a/libcudacxx/include/cuda/std/__variant/variant_base.h b/libcudacxx/include/cuda/std/__variant/variant_base.h index 6e2a195386f6..7709dfd454db 100644 --- a/libcudacxx/include/cuda/std/__variant/variant_base.h +++ b/libcudacxx/include/cuda/std/__variant/variant_base.h @@ -56,16 +56,16 @@ struct _CCCL_TYPE_VISIBILITY_DEFAULT __alt : __value(::cuda::std::forward<_Args>(__args)...) {} _CCCL_EXEC_CHECK_DISABLE - constexpr __alt(const __alt&) = default; + _CCCL_HIDE_FROM_ABI constexpr __alt(const __alt&) = default; _CCCL_EXEC_CHECK_DISABLE - constexpr __alt(__alt&&) = default; + _CCCL_HIDE_FROM_ABI constexpr __alt(__alt&&) = default; _CCCL_EXEC_CHECK_DISABLE - constexpr __alt& operator=(const __alt&) = default; + _CCCL_HIDE_FROM_ABI constexpr __alt& operator=(const __alt&) = default; _CCCL_EXEC_CHECK_DISABLE - constexpr __alt& operator=(__alt&&) = default; + _CCCL_HIDE_FROM_ABI constexpr __alt& operator=(__alt&&) = default; _CCCL_EXEC_CHECK_DISABLE - ~__alt() = default; + _CCCL_HIDE_FROM_ABI ~__alt() = default; __value_type __value; }; diff --git a/libcudacxx/include/nv/target b/libcudacxx/include/nv/target index b5565725d25d..702ac3579299 100644 --- a/libcudacxx/include/nv/target +++ b/libcudacxx/include/nv/target @@ -88,7 +88,7 @@ struct _NV_BITSET_ATTRIBUTE target_description { base_int_t targets; - constexpr target_description(base_int_t a) + constexpr target_description(base_int_t a) // NOLINT(cccl-hide-from-abi) : targets(a) {} }; @@ -121,12 +121,12 @@ enum class sm_selector : base_int_t sm_121 = 121, }; -constexpr base_int_t toint(sm_selector a) +constexpr base_int_t toint(sm_selector a) // NOLINT(cccl-hide-from-abi) { return static_cast(a); } -constexpr base_int_t bitexact(sm_selector a) +constexpr base_int_t bitexact(sm_selector a) // NOLINT(cccl-hide-from-abi) { return toint(a) == 35 ? sm_35_bit : toint(a) == 37 ? sm_37_bit @@ -154,7 +154,7 @@ constexpr base_int_t bitexact(sm_selector a) : 0; } -constexpr base_int_t bitrounddown(sm_selector a) +constexpr base_int_t bitrounddown(sm_selector a) // NOLINT(cccl-hide-from-abi) { return toint(a) >= 121 ? sm_121_bit : toint(a) >= 120 ? sm_120_bit @@ -184,29 +184,29 @@ constexpr base_int_t bitrounddown(sm_selector a) // Public API for NVIDIA GPUs -constexpr target_description is_exactly(sm_selector a) +constexpr target_description is_exactly(sm_selector a) // NOLINT(cccl-hide-from-abi) { return target_description(bitexact(a)); } -constexpr target_description provides(sm_selector a) +constexpr target_description provides(sm_selector a) // NOLINT(cccl-hide-from-abi) { return target_description(~(bitrounddown(a) - 1) & all_devices); } // Boolean operations on target sets -constexpr target_description operator&&(target_description a, target_description b) +constexpr target_description operator&&(target_description a, target_description b) // NOLINT(cccl-hide-from-abi) { return target_description(a.targets & b.targets); } -constexpr target_description operator||(target_description a, target_description b) +constexpr target_description operator||(target_description a, target_description b) // NOLINT(cccl-hide-from-abi) { return target_description(a.targets | b.targets); } -constexpr target_description operator!(target_description a) +constexpr target_description operator!(target_description a) // NOLINT(cccl-hide-from-abi) { return target_description(~a.targets & (all_devices | all_hosts)); } diff --git a/nvbench_helper/nvbench_helper/device_side_benchmark.cuh b/nvbench_helper/nvbench_helper/device_side_benchmark.cuh index 9c5020be78df..4ff9884557ba 100644 --- a/nvbench_helper/nvbench_helper/device_side_benchmark.cuh +++ b/nvbench_helper/nvbench_helper/device_side_benchmark.cuh @@ -11,7 +11,7 @@ #include template -__device__ __forceinline__ T generate_random_data() +_CCCL_DEVICE_API __forceinline__ T generate_random_data() { constexpr auto size = static_cast(cuda::ceil_div(sizeof(T), sizeof(uint32_t))); uint32_t data[size]; @@ -27,7 +27,7 @@ __device__ __forceinline__ T generate_random_data() // When benchmarking algorithms sensitive to data distribution, each thread should use a different seed to get different // data. template -__device__ __forceinline__ T generate_random_data(uint32_t& seed) +_CCCL_DEVICE_API __forceinline__ T generate_random_data(uint32_t& seed) { constexpr auto size = static_cast(cuda::ceil_div(sizeof(T), sizeof(uint32_t))); uint32_t data[size]; @@ -45,7 +45,7 @@ __device__ __forceinline__ T generate_random_data(uint32_t& seed) __device__ static int device_var[16]; template -__device__ __forceinline__ void sink(T value) +_CCCL_DEVICE_API __forceinline__ void sink(T value) { if (cuda::ptx::get_sreg_smid() == static_cast(-1)) { @@ -54,7 +54,7 @@ __device__ __forceinline__ void sink(T value) } template -__device__ __forceinline__ void sink(T (&values)[Size]) +_CCCL_DEVICE_API __forceinline__ void sink(T (&values)[Size]) { if (cuda::ptx::get_sreg_smid() == static_cast(-1)) { @@ -70,7 +70,7 @@ __device__ __forceinline__ void sink(T (&values)[Size]) } template -__launch_bounds__(ThreadsPerBlock) __global__ static void benchmark_kernel(const ActionT action) +__launch_bounds__(ThreadsPerBlock) _CCCL_KERNEL_ATTRIBUTES static void benchmark_kernel(const ActionT action) { auto data = generate_random_data(); if constexpr (Unroll) @@ -93,7 +93,7 @@ __launch_bounds__(ThreadsPerBlock) __global__ static void benchmark_kernel(const // benchmark results. // For keys-only benchmarks, set ValueT to void. template -__global__ static void benchmark_kernel(int num_iterations, const ActionT action, Args... args) +_CCCL_KERNEL_ATTRIBUTES static void benchmark_kernel(int num_iterations, const ActionT action, Args... args) { constexpr int warp_threads = 32; constexpr bool has_values = !cuda::std::is_void_v; diff --git a/nvbench_helper/nvbench_helper/nvbench_helper.cuh b/nvbench_helper/nvbench_helper/nvbench_helper.cuh index 0e31bd3cd644..0fd7dfcd8632 100644 --- a/nvbench_helper/nvbench_helper/nvbench_helper.cuh +++ b/nvbench_helper/nvbench_helper/nvbench_helper.cuh @@ -54,11 +54,11 @@ NVBENCH_DECLARE_TYPE_STRINGS(cub::detail::arg_max, "arg_max", "cub::detail::arg_ template struct nvbench::type_strings> { - static std::string input_string() + _CCCL_HOST_API static std::string input_string() { return std::to_string(I); } - static std::string description() + _CCCL_HOST_API static std::string description() { return "integral_constant<" + type_strings::description() + ", " + std::to_string(I) + ">"; } @@ -124,16 +124,16 @@ class value_wrapper_t T m_val{}; public: - explicit value_wrapper_t(T val) + _CCCL_HOST_API explicit value_wrapper_t(T val) : m_val(val) {} - T get() const + _CCCL_HOST_API T get() const { return m_val; } - value_wrapper_t& operator++() + _CCCL_HOST_API value_wrapper_t& operator++() { m_val++; return *this; @@ -146,7 +146,7 @@ public: using value_wrapper_t::value_wrapper_t; using value_wrapper_t::operator++; - seed_t() + _CCCL_HOST_API seed_t() : value_wrapper_t(42) {} }; @@ -162,7 +162,7 @@ enum class bit_entropy }; NVBENCH_DECLARE_TYPE_STRINGS(bit_entropy, "BE", "bit entropy"); -[[nodiscard]] inline double entropy_to_probability(bit_entropy entropy) +[[nodiscard]] _CCCL_HOST_API inline double entropy_to_probability(bit_entropy entropy) { switch (entropy) { @@ -183,7 +183,7 @@ NVBENCH_DECLARE_TYPE_STRINGS(bit_entropy, "BE", "bit entropy"); } } -[[nodiscard]] inline bit_entropy str_to_entropy(std::string str) +[[nodiscard]] _CCCL_HOST_API inline bit_entropy str_to_entropy(std::string str) { if (str == "1.000") { @@ -215,7 +215,7 @@ NVBENCH_DECLARE_TYPE_STRINGS(bit_entropy, "BE", "bit entropy"); // Creates an interpolated value of type T between min (at = 0.0) and max (at = 1.0). template -[[nodiscard]] T lerp_min_max(double at) noexcept +[[nodiscard]] _CCCL_HOST_API T lerp_min_max(double at) noexcept { if (at == 1.0) { @@ -231,7 +231,7 @@ template struct is_even_after_truncation { template - [[nodiscard]] __host__ __device__ constexpr bool operator()(const T& val) const noexcept + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool operator()(const T& val) const noexcept { return cuda::__is_even{}(static_cast(val)); } @@ -276,7 +276,7 @@ struct generator_base_t const bit_entropy m_entropy{bit_entropy::_1_000}; template - thrust::device_vector generate(T min, T max) + _CCCL_HOST_API thrust::device_vector generate(T min, T max) { thrust::device_vector vec(m_elements); const cuda::std::span span(thrust::raw_pointer_cast(vec.data()), m_elements); @@ -296,7 +296,7 @@ struct vector_generator_t : generator_base_t const T m_min{cuda::std::numeric_limits::min()}; const T m_max{cuda::std::numeric_limits::max()}; - operator thrust::device_vector() + _CCCL_HOST_API operator thrust::device_vector() { return generator_base_t::generate(m_min, m_max); } @@ -306,7 +306,7 @@ template <> struct vector_generator_t : generator_base_t { template - operator thrust::device_vector() + _CCCL_HOST_API operator thrust::device_vector() { return generator_base_t::generate(cuda::std::numeric_limits::min(), cuda::std::numeric_limits::max()); } @@ -314,7 +314,7 @@ struct vector_generator_t : generator_base_t // This overload is needed because numeric limits is not specialized for complex, making // the min and max values for complex equal zero. template - operator thrust::device_vector>() + _CCCL_HOST_API operator thrust::device_vector>() { const auto min = cuda::std::complex{cuda::std::numeric_limits::min(), cuda::std::numeric_limits::min()}; const auto max = cuda::std::complex{cuda::std::numeric_limits::max(), cuda::std::numeric_limits::max()}; @@ -331,7 +331,7 @@ struct uniform_key_segments_generator_t const std::size_t m_max_segment_size{0}; template - operator thrust::device_vector() + _CCCL_HOST_API operator thrust::device_vector() { thrust::device_vector keys_vec(m_total_elements); const cuda::std::span keys(thrust::raw_pointer_cast(keys_vec.data()), keys_vec.size()); @@ -353,7 +353,7 @@ struct uniform_segment_offsets_generator_t const std::size_t m_max_segment_size{0}; template - operator thrust::device_vector() + _CCCL_HOST_API operator thrust::device_vector() { thrust::device_vector offsets_vec(m_total_elements + 2); const cuda::std::span offsets(thrust::raw_pointer_cast(offsets_vec.data()), offsets_vec.size()); @@ -377,7 +377,7 @@ struct power_law_segment_offsets_generator_t const std::size_t m_segments{0}; template - operator thrust::device_vector() + _CCCL_HOST_API operator thrust::device_vector() { thrust::device_vector offsets_vec(m_segments + 1); const cuda::std::span offsets(thrust::raw_pointer_cast(offsets_vec.data()), offsets_vec.size()); @@ -393,7 +393,7 @@ struct power_law_segment_offsets_generator_t struct gen_uniform_key_segments_t { - uniform_key_segments_generator_t + _CCCL_HOST_API uniform_key_segments_generator_t operator()(std::size_t total_elements, std::size_t min_segment_size, std::size_t max_segment_size) const { return {seed_t{}, total_elements, min_segment_size, max_segment_size}; @@ -402,7 +402,7 @@ struct gen_uniform_key_segments_t struct gen_uniform_segment_offsets_t { - uniform_segment_offsets_generator_t + _CCCL_HOST_API uniform_segment_offsets_generator_t operator()(std::size_t total_elements, std::size_t min_segment_size, std::size_t max_segment_size) const { return {seed_t{}, total_elements, min_segment_size, max_segment_size}; @@ -417,7 +417,7 @@ struct gen_uniform_t struct gen_power_law_segment_offsets_t { - power_law_segment_offsets_generator_t operator()(std::size_t elements, std::size_t segments) const + _CCCL_HOST_API power_law_segment_offsets_generator_t operator()(std::size_t elements, std::size_t segments) const { return {seed_t{}, elements, segments}; } @@ -430,13 +430,14 @@ struct gen_power_law_t struct gen_t { - vector_generator_t operator()(std::size_t elements, bit_entropy entropy = bit_entropy::_1_000) const + _CCCL_HOST_API vector_generator_t + operator()(std::size_t elements, bit_entropy entropy = bit_entropy::_1_000) const { return {seed_t{}, elements, entropy}; } template - vector_generator_t operator()( + _CCCL_HOST_API vector_generator_t operator()( std::size_t elements, bit_entropy entropy = bit_entropy::_1_000, T min = cuda::std::numeric_limits::min, @@ -453,7 +454,7 @@ struct gen_t inline detail::gen_t generate; template -void do_not_optimize(const T& val) +_CCCL_HOST_API void do_not_optimize(const T& val) { detail::do_not_optimize(&val); } @@ -461,13 +462,13 @@ void do_not_optimize(const T& val) struct less_t { template - __host__ __device__ bool operator()(const DataType& lhs, const DataType& rhs) const + _CCCL_HOST_DEVICE_API bool operator()(const DataType& lhs, const DataType& rhs) const { return lhs < rhs; } template - __host__ __device__ bool operator()(const cuda::std::complex& lhs, const cuda::std::complex& rhs) const + _CCCL_HOST_DEVICE_API bool operator()(const cuda::std::complex& lhs, const cuda::std::complex& rhs) const { double magnitude_0 = cuda::std::abs(lhs); double magnitude_1 = cuda::std::abs(rhs); @@ -513,21 +514,21 @@ struct less_t struct max_t { template - __host__ __device__ DataType operator()(const DataType& lhs, const DataType& rhs) const + _CCCL_HOST_DEVICE_API DataType operator()(const DataType& lhs, const DataType& rhs) const { const less_t less{}; return less(lhs, rhs) ? rhs : lhs; } #if _CCCL_HAS_NVFP16() && _CCCL_CTK_AT_LEAST(12, 2) - __host__ __device__ __half operator()(__half lhs, __half rhs) const + _CCCL_HOST_DEVICE_API __half operator()(__half lhs, __half rhs) const { return static_cast(lhs) < static_cast(rhs) ? rhs : lhs; } #endif // _CCCL_HAS_NVFP16() && _CCCL_CTK_AT_LEAST(12, 2) #if _CCCL_HAS_NVBF16() && _CCCL_CTK_AT_LEAST(12, 2) - __host__ __device__ __nv_bfloat16 operator()(__nv_bfloat16 lhs, __nv_bfloat16 rhs) const + _CCCL_HOST_DEVICE_API __nv_bfloat16 operator()(__nv_bfloat16 lhs, __nv_bfloat16 rhs) const { return static_cast(lhs) < static_cast(rhs) ? rhs : lhs; } @@ -539,7 +540,7 @@ struct less_then_t { T m_val; - [[nodiscard]] __device__ bool operator()(const T& val) const noexcept + [[nodiscard]] _CCCL_DEVICE_API bool operator()(const T& val) const noexcept { return val < m_val; } @@ -555,13 +556,13 @@ struct caching_allocator_t { using value_type = char; - caching_allocator_t() = default; - ~caching_allocator_t() + _CCCL_HIDE_FROM_ABI caching_allocator_t() = default; + _CCCL_HOST_API ~caching_allocator_t() { free_all(); } - char* allocate(std::ptrdiff_t num_bytes) + _CCCL_HOST_API char* allocate(std::ptrdiff_t num_bytes) { #if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA if (first_async_stream != cuda::invalid_stream) @@ -589,7 +590,7 @@ struct caching_allocator_t return result; } - void deallocate(char* ptr, size_t, [[maybe_unused]] bool check_stream = true) + _CCCL_HOST_API void deallocate(char* ptr, size_t, [[maybe_unused]] bool check_stream = true) { #if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA if (check_stream && first_async_stream != cuda::invalid_stream) @@ -612,17 +613,17 @@ struct caching_allocator_t } #if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA - void* allocate_sync(size_t num_bytes, size_t) + _CCCL_HOST_API void* allocate_sync(size_t num_bytes, size_t) { return allocate(static_cast(num_bytes)); } - void deallocate_sync(void* ptr, size_t num_bytes, size_t) + _CCCL_HOST_API void deallocate_sync(void* ptr, size_t num_bytes, size_t) { deallocate(static_cast(ptr), num_bytes); } - void* allocate(cuda::stream_ref __stream, size_t num_bytes, size_t) + _CCCL_HOST_API void* allocate(cuda::stream_ref __stream, size_t num_bytes, size_t) { if (first_async_stream == cuda::invalid_stream) { @@ -656,7 +657,7 @@ struct caching_allocator_t return result; } - void deallocate(cuda::stream_ref __stream, void* ptr, size_t num_bytes, size_t) + _CCCL_HOST_API void deallocate(cuda::stream_ref __stream, void* ptr, size_t num_bytes, size_t) { if (first_async_stream != __stream) { @@ -680,7 +681,7 @@ private: cuda::stream_ref first_async_stream{cuda::invalid_stream}; // just to detect wrong usage patterns #endif // THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA - void free_all() + _CCCL_HOST_API void free_all() { for (auto i : free_blocks) { @@ -693,7 +694,7 @@ private: } } - value_type* do_allocate(std::size_t num_bytes) + _CCCL_HOST_API value_type* do_allocate(std::size_t num_bytes) { value_type* result{}; #if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA @@ -708,7 +709,7 @@ private: return result; } - void do_deallocate(value_type* ptr) // NOLINT(readability-non-const-parameter) + _CCCL_HOST_API void do_deallocate(value_type* ptr) // NOLINT(readability-non-const-parameter) { #if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA static_cast(cudaFree(ptr)); @@ -717,49 +718,49 @@ private: #endif } - friend bool operator==(const caching_allocator_t& lhs, const caching_allocator_t& rhs) + _CCCL_HOST_API friend bool operator==(const caching_allocator_t& lhs, const caching_allocator_t& rhs) { return lhs.free_blocks == rhs.free_blocks && lhs.allocated_blocks == rhs.allocated_blocks; } - friend bool operator!=(const caching_allocator_t& lhs, const caching_allocator_t& rhs) + _CCCL_HOST_API friend bool operator!=(const caching_allocator_t& lhs, const caching_allocator_t& rhs) { return lhs.free_blocks == rhs.free_blocks && lhs.allocated_blocks == rhs.allocated_blocks; } #if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA - friend constexpr void get_property(const caching_allocator_t&, cuda::mr::device_accessible) noexcept {} + _CCCL_HOST_API friend constexpr void get_property(const caching_allocator_t&, cuda::mr::device_accessible) noexcept {} #endif // THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA }; #if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA -inline auto policy(caching_allocator_t& alloc) +_CCCL_HOST_API inline auto policy(caching_allocator_t& alloc) { return thrust::cuda::par(alloc); } -inline auto cuda_policy(caching_allocator_t& alloc) +_CCCL_HOST_API inline auto cuda_policy(caching_allocator_t& alloc) { return cuda::execution::gpu.with(cuda::mr::get_memory_resource, alloc); } #else -inline auto policy(caching_allocator_t&) +_CCCL_HOST_API inline auto policy(caching_allocator_t&) { return thrust::device; } #endif #if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA -inline auto policy(caching_allocator_t& alloc, nvbench::launch& launch) +_CCCL_HOST_API inline auto policy(caching_allocator_t& alloc, nvbench::launch& launch) { return thrust::cuda::par(alloc).on(launch.get_stream()); } -inline auto cuda_policy(caching_allocator_t& alloc, nvbench::launch& launch) +_CCCL_HOST_API inline auto cuda_policy(caching_allocator_t& alloc, nvbench::launch& launch) { return cuda::execution::gpu.with(cuda::mr::get_memory_resource, alloc) .with(cuda::get_stream, launch.get_stream().get_stream()); } #else -inline auto policy(caching_allocator_t&, nvbench::launch&) +_CCCL_HOST_API inline auto policy(caching_allocator_t&, nvbench::launch&) { return thrust::device; } @@ -768,7 +769,7 @@ inline auto policy(caching_allocator_t&, nvbench::launch&) #if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA // Returns an environment for benchmarking using alloc as MR, launch's stream, and any additional envs passed in. template -auto cub_bench_env(caching_allocator_t& alloc, nvbench::launch& launch, MoreEnvs... envs) +_CCCL_HOST_API auto cub_bench_env(caching_allocator_t& alloc, nvbench::launch& launch, MoreEnvs... envs) { return cuda::std::execution::env{ cuda::stream_ref{launch.get_stream().get_stream()}, diff --git a/thrust/benchmarks/bench/set_operations/base.cuh b/thrust/benchmarks/bench/set_operations/base.cuh index b1cf4b2b16a2..6ea05f65f846 100644 --- a/thrust/benchmarks/bench/set_operations/base.cuh +++ b/thrust/benchmarks/bench/set_operations/base.cuh @@ -11,7 +11,7 @@ #include "nvbench_helper.cuh" template -void basic(nvbench::state& state, nvbench::type_list, OpT op) +_CCCL_HOST_API void basic(nvbench::state& state, nvbench::type_list, OpT op) { const auto elements = static_cast(state.get_int64("Elements")); const auto size_ratio = static_cast(state.get_int64("SizeRatio")); diff --git a/thrust/benchmarks/bench/set_operations/by_key.cuh b/thrust/benchmarks/bench/set_operations/by_key.cuh index 9a946e4127ca..add34abbde87 100644 --- a/thrust/benchmarks/bench/set_operations/by_key.cuh +++ b/thrust/benchmarks/bench/set_operations/by_key.cuh @@ -11,7 +11,7 @@ #include "nvbench_helper.cuh" template -void basic(nvbench::state& state, nvbench::type_list, OpT op) +_CCCL_HOST_API void basic(nvbench::state& state, nvbench::type_list, OpT op) { const auto elements = static_cast(state.get_int64("Elements")); const auto size_ratio = static_cast(state.get_int64("SizeRatio")); diff --git a/thrust/testing/copy_construct_test.h b/thrust/testing/copy_construct_test.h index 0c6a1a09803f..48054d81fa0d 100644 --- a/thrust/testing/copy_construct_test.h +++ b/thrust/testing/copy_construct_test.h @@ -9,16 +9,16 @@ // system they claim to target. struct CopyConstructTest { - CopyConstructTest() = default; + _CCCL_HIDE_FROM_ABI CopyConstructTest() = default; - _CCCL_HOST_DEVICE CopyConstructTest(const CopyConstructTest&) + _CCCL_HOST_DEVICE_API CopyConstructTest(const CopyConstructTest&) { NV_IF_TARGET(NV_IS_DEVICE, (copy_constructed_on_device = true; copy_constructed_on_host = false;), (copy_constructed_on_device = false; copy_constructed_on_host = true;)); } - CopyConstructTest& operator=(const CopyConstructTest&) = default; + _CCCL_HIDE_FROM_ABI CopyConstructTest& operator=(const CopyConstructTest&) = default; bool copy_constructed_on_host{false}; bool copy_constructed_on_device{false}; @@ -26,7 +26,7 @@ struct CopyConstructTest struct is_copy_constructed_on_device { - _CCCL_HOST_DEVICE bool operator()(const CopyConstructTest& t) const + _CCCL_HOST_DEVICE_API bool operator()(const CopyConstructTest& t) const { return t.copy_constructed_on_device; } @@ -34,7 +34,7 @@ struct is_copy_constructed_on_device struct is_copy_constructed_on_host { - _CCCL_HOST_DEVICE bool operator()(const CopyConstructTest& t) const + _CCCL_HOST_DEVICE_API bool operator()(const CopyConstructTest& t) const { return t.copy_constructed_on_host; } diff --git a/thrust/testing/unittest/random.h b/thrust/testing/unittest/random.h index 40ab907ac1a5..acd53c0e4be0 100644 --- a/thrust/testing/unittest/random.h +++ b/thrust/testing/unittest/random.h @@ -11,7 +11,7 @@ namespace unittest { namespace detail { -inline unsigned int hash(unsigned int a) +_CCCL_HOST_API inline unsigned int hash(unsigned int a) { a = (a + 0x7ed55d16) + (a << 12); a = (a ^ 0xc761c23c) ^ (a >> 19); @@ -26,7 +26,7 @@ inline unsigned int hash(unsigned int a) template struct generate_random_integer { - T operator()(unsigned int i) const + _CCCL_HOST_API T operator()(unsigned int i) const { THRUST_NS_QUALIFIER::default_random_engine rng(detail::hash(i)); if constexpr (::cuda::std::is_same_v) @@ -58,7 +58,7 @@ struct generate_random_integer template struct generate_random_sample { - T operator()(unsigned int i) const + _CCCL_HOST_API T operator()(unsigned int i) const { THRUST_NS_QUALIFIER::default_random_engine rng(detail::hash(i)); THRUST_NS_QUALIFIER::uniform_int_distribution dist(0, 20); @@ -68,7 +68,7 @@ struct generate_random_sample }; template -THRUST_NS_QUALIFIER::host_vector random_integers(const size_t N) +_CCCL_HOST_API THRUST_NS_QUALIFIER::host_vector random_integers(const size_t N) { THRUST_NS_QUALIFIER::host_vector vec(N); THRUST_NS_QUALIFIER::transform( @@ -81,13 +81,13 @@ THRUST_NS_QUALIFIER::host_vector random_integers(const size_t N) } template -T random_integer() +_CCCL_HOST_API T random_integer() { return generate_random_integer()(0); } template -THRUST_NS_QUALIFIER::host_vector random_samples(const size_t N) +_CCCL_HOST_API THRUST_NS_QUALIFIER::host_vector random_samples(const size_t N) { THRUST_NS_QUALIFIER::host_vector vec(N); THRUST_NS_QUALIFIER::transform( diff --git a/thrust/testing/unittest/special_types.h b/thrust/testing/unittest/special_types.h index 497b089d8928..6d8bb1112f25 100644 --- a/thrust/testing/unittest/special_types.h +++ b/thrust/testing/unittest/special_types.h @@ -14,7 +14,7 @@ struct FixedVector { T data[N]; - _CCCL_HOST_DEVICE FixedVector() + _CCCL_HOST_DEVICE_API FixedVector() { for (unsigned int i = 0; i < N; i++) { @@ -22,7 +22,7 @@ struct FixedVector } } - _CCCL_HOST_DEVICE explicit FixedVector(T init) + _CCCL_HOST_DEVICE_API explicit FixedVector(T init) { for (unsigned int i = 0; i < N; i++) { @@ -30,7 +30,7 @@ struct FixedVector } } - _CCCL_HOST_DEVICE + _CCCL_HOST_DEVICE_API #if _CCCL_COMPILER(NVHPC) __attribute__((noinline)) #endif @@ -44,7 +44,7 @@ struct FixedVector return output; } - _CCCL_HOST_DEVICE bool operator<(const FixedVector& bs) const + _CCCL_HOST_DEVICE_API bool operator<(const FixedVector& bs) const { for (unsigned int i = 0; i < N; i++) { @@ -60,7 +60,7 @@ struct FixedVector return false; } - _CCCL_HOST_DEVICE bool operator==(const FixedVector& bs) const + _CCCL_HOST_DEVICE_API bool operator==(const FixedVector& bs) const { for (unsigned int i = 0; i < N; i++) { @@ -79,37 +79,37 @@ struct key_value using key_type = Key; using value_type = Value; - _CCCL_HOST_DEVICE key_value() + _CCCL_HOST_DEVICE_API key_value() : key() , value() {} - _CCCL_HOST_DEVICE key_value(key_type k, value_type v) + _CCCL_HOST_DEVICE_API key_value(key_type k, value_type v) : key(k) , value(v) {} - _CCCL_HOST_DEVICE bool operator<(const key_value& rhs) const + _CCCL_HOST_DEVICE_API bool operator<(const key_value& rhs) const { return key < rhs.key; } - _CCCL_HOST_DEVICE bool operator>(const key_value& rhs) const + _CCCL_HOST_DEVICE_API bool operator>(const key_value& rhs) const { return key > rhs.key; } - _CCCL_HOST_DEVICE bool operator==(const key_value& rhs) const + _CCCL_HOST_DEVICE_API bool operator==(const key_value& rhs) const { return key == rhs.key && value == rhs.value; } - _CCCL_HOST_DEVICE bool operator!=(const key_value& rhs) const + _CCCL_HOST_DEVICE_API bool operator!=(const key_value& rhs) const { return !(*this == rhs); } - friend std::ostream& operator<<(std::ostream& os, const key_value& kv) + _CCCL_HOST_API friend std::ostream& operator<<(std::ostream& os, const key_value& kv) { // cast 1-byte integral (char) types so they print as numbers, not characters // (clang-tidy bugprone-unintended-char-ostream-output) @@ -140,18 +140,18 @@ struct key_value struct user_swappable { - _CCCL_HOST_DEVICE user_swappable(bool swapped = false) + _CCCL_HOST_DEVICE_API user_swappable(bool swapped = false) : was_swapped(swapped) {} bool was_swapped; - friend _CCCL_HOST_DEVICE bool operator==(const user_swappable& x, const user_swappable& y) + friend _CCCL_HOST_DEVICE_API bool operator==(const user_swappable& x, const user_swappable& y) { return x.was_swapped == y.was_swapped; } - friend _CCCL_HOST_DEVICE void swap(user_swappable& x, user_swappable& y) noexcept + friend _CCCL_HOST_DEVICE_API void swap(user_swappable& x, user_swappable& y) noexcept { x.was_swapped = true; y.was_swapped = false; @@ -165,7 +165,7 @@ class custom_numeric { public: // fill() initializes every element. - _CCCL_HOST_DEVICE custom_numeric() // NOLINT(cppcoreguidelines-pro-type-member-init) + _CCCL_HOST_DEVICE_API custom_numeric() // NOLINT(cppcoreguidelines-pro-type-member-init) { fill(0); } @@ -182,24 +182,24 @@ class custom_numeric // Allow construction from any integral numeric. // fill() initializes every element. template >> - _CCCL_HOST_DEVICE custom_numeric(const T& i) // NOLINT(cppcoreguidelines-pro-type-member-init) + _CCCL_HOST_DEVICE_API custom_numeric(const T& i) // NOLINT(cppcoreguidelines-pro-type-member-init) { fill(static_cast(i)); } // fill() initializes every element. - _CCCL_HOST_DEVICE custom_numeric(const custom_numeric& other) // NOLINT(cppcoreguidelines-pro-type-member-init) + _CCCL_HOST_DEVICE_API custom_numeric(const custom_numeric& other) // NOLINT(cppcoreguidelines-pro-type-member-init) { fill(other.value[0]); } - _CCCL_HOST_DEVICE custom_numeric& operator=(int val) + _CCCL_HOST_DEVICE_API custom_numeric& operator=(int val) { fill(val); return *this; } - _CCCL_HOST_DEVICE custom_numeric& operator=(const custom_numeric& other) + _CCCL_HOST_DEVICE_API custom_numeric& operator=(const custom_numeric& other) { if (this != &other) { @@ -207,22 +207,22 @@ class custom_numeric } return *this; } - _CCCL_HOST_DEVICE operator void*() const + _CCCL_HOST_DEVICE_API operator void*() const { return reinterpret_cast(static_cast(value[0])); // NOLINT(performance-no-int-to-ptr) } -#define DEFINE_OPERATOR(op) \ - _CCCL_HOST_DEVICE custom_numeric& operator op() \ - { \ - fill(op value[0]); \ - return *this; \ - } \ - _CCCL_HOST_DEVICE custom_numeric operator op(int) const \ - { \ - custom_numeric ret(*this); \ - op ret; \ - return ret; \ +#define DEFINE_OPERATOR(op) \ + _CCCL_HOST_DEVICE_API custom_numeric& operator op() \ + { \ + fill(op value[0]); \ + return *this; \ + } \ + _CCCL_HOST_DEVICE_API custom_numeric operator op(int) const \ + { \ + custom_numeric ret(*this); \ + op ret; \ + return ret; \ } DEFINE_OPERATOR(++) @@ -230,10 +230,10 @@ class custom_numeric #undef DEFINE_OPERATOR -#define DEFINE_OPERATOR(op) \ - _CCCL_HOST_DEVICE custom_numeric operator op() const \ - { \ - return custom_numeric(op value[0]); \ +#define DEFINE_OPERATOR(op) \ + _CCCL_HOST_DEVICE_API custom_numeric operator op() const \ + { \ + return custom_numeric(op value[0]); \ } DEFINE_OPERATOR(+) @@ -242,10 +242,10 @@ class custom_numeric #undef DEFINE_OPERATOR -#define DEFINE_OPERATOR(op) \ - _CCCL_HOST_DEVICE custom_numeric operator op(const custom_numeric& other) const \ - { \ - return custom_numeric(value[0] op other.value[0]); \ +#define DEFINE_OPERATOR(op) \ + _CCCL_HOST_DEVICE_API custom_numeric operator op(const custom_numeric& other) const \ + { \ + return custom_numeric(value[0] op other.value[0]); \ } DEFINE_OPERATOR(+) @@ -263,11 +263,11 @@ class custom_numeric #define CONCAT(X, Y) X##Y -#define DEFINE_OPERATOR(op) \ - _CCCL_HOST_DEVICE custom_numeric& operator CONCAT(op, =)(const custom_numeric& other) \ - { \ - fill(value[0] op other.value[0]); \ - return *this; \ +#define DEFINE_OPERATOR(op) \ + _CCCL_HOST_DEVICE_API custom_numeric& operator CONCAT(op, =)(const custom_numeric& other) \ + { \ + fill(value[0] op other.value[0]); \ + return *this; \ } DEFINE_OPERATOR(+) @@ -284,10 +284,10 @@ class custom_numeric #undef DEFINE_OPERATOR #undef CONCAT -#define DEFINE_OPERATOR(op) \ - _CCCL_HOST_DEVICE friend bool operator op(const custom_numeric& lhs, const custom_numeric& rhs) \ - { \ - return lhs.value[0] op rhs.value[0]; \ +#define DEFINE_OPERATOR(op) \ + _CCCL_HOST_DEVICE_API friend bool operator op(const custom_numeric& lhs, const custom_numeric& rhs) \ + { \ + return lhs.value[0] op rhs.value[0]; \ } DEFINE_OPERATOR(==) @@ -304,14 +304,14 @@ class custom_numeric // Mixed comparisons against int. Because custom_numeric is implicitly convertible to bool, the built-in integer // comparison operators would otherwise compete with the homogeneous ones above and make comparisons against integer // literals (e.g. ASSERT_EQUAL(0, some_custom_numeric)) ambiguous. Providing exact int overloads resolves this. -#define DEFINE_OPERATOR(op) \ - _CCCL_HOST_DEVICE friend bool operator op(const custom_numeric& lhs, int rhs) \ - { \ - return lhs.value[0] op rhs; \ - } \ - _CCCL_HOST_DEVICE friend bool operator op(int lhs, const custom_numeric& rhs) \ - { \ - return lhs op rhs.value[0]; \ +#define DEFINE_OPERATOR(op) \ + _CCCL_HOST_DEVICE_API friend bool operator op(const custom_numeric& lhs, int rhs) \ + { \ + return lhs.value[0] op rhs; \ + } \ + _CCCL_HOST_DEVICE_API friend bool operator op(int lhs, const custom_numeric& rhs) \ + { \ + return lhs op rhs.value[0]; \ } DEFINE_OPERATOR(==) @@ -323,7 +323,7 @@ class custom_numeric #undef DEFINE_OPERATOR - friend std::ostream& operator<<(std::ostream& os, const custom_numeric& val) + _CCCL_HOST_API friend std::ostream& operator<<(std::ostream& os, const custom_numeric& val) { return os << "custom_numeric{" << val.value[0] << "}"; } @@ -331,7 +331,7 @@ class custom_numeric private: int value[5]; - _CCCL_HOST_DEVICE void fill(int val) + _CCCL_HOST_DEVICE_API void fill(int val) { for (int& v : value) { @@ -359,18 +359,18 @@ _CCCL_END_NAMESPACE_CUDA_STD struct my_system : THRUST_NS_QUALIFIER::device_execution_policy { - my_system(int) {} + _CCCL_HOST_API my_system(int) {} - my_system(const my_system& other) + _CCCL_HOST_API my_system(const my_system& other) : num_copies(other.num_copies + 1) {} - void validate_dispatch() + _CCCL_HOST_API void validate_dispatch() { correctly_dispatched = (num_copies == 0); } - bool is_valid() const + _CCCL_HOST_API bool is_valid() const { return correctly_dispatched; } diff --git a/thrust/testing/unittest/unittest.h b/thrust/testing/unittest/unittest.h index 5bfeeb386ec7..0deceadcaa4b 100644 --- a/thrust/testing/unittest/unittest.h +++ b/thrust/testing/unittest/unittest.h @@ -79,7 +79,7 @@ struct has_value_type> : ::cuda:: // device_vector: copy to host first template -std::vector to_vec(thrust::device_vector const& vec) +_CCCL_HOST_API std::vector to_vec(thrust::device_vector const& vec) { thrust::host_vector temp = vec; return std::vector{temp.begin(), temp.end()}; @@ -87,14 +87,14 @@ std::vector to_vec(thrust::device_vector const& vec) // any other host-accessible container (host_vector, universal_vector, std::vector, vector_base, ...) template ::value, int> = 0> -std::vector to_vec(C const& c) +_CCCL_HOST_API std::vector to_vec(C const& c) { return std::vector{c.begin(), c.end()}; } // scalar: wrap into a one-element vector template ::value, int> = 0> -std::vector to_vec(T const& x) +_CCCL_HOST_API std::vector to_vec(T const& x) { return std::vector{x}; } @@ -102,13 +102,13 @@ std::vector to_vec(T const& x) // Catch2's Approx matcher only understands arithmetic element types. For complex numbers we flatten the vector into // its interleaved real/imaginary components so the matcher can compare them component-wise. template -std::vector to_approx(std::vector v) +_CCCL_HOST_API std::vector to_approx(std::vector v) { return v; } template