From b842394957f220fd7adefdc564109366885c5d8d Mon Sep 17 00:00:00 2001 From: Alex Voicu Date: Fri, 8 Dec 2017 04:22:57 +0000 Subject: [PATCH] This introduces LipoProteinLipase (lpl), a simple tool for creating fat binaries. It represents a direct replacement of the creaky hccgenco.sh script, which had various issues. The format it uses is that of a code object bundle, generated by the Clang Offload Bundler. The output is always suffixed with the ".adipose" extension. It is shared with HCC. The hipcc script and associated tests are modified to use lpl. Help can be obtained by invoking lpl --help. A more computer-sciency / corporate friendly name is likely to be beneficial, which is a reason for choosing easily searchable/replaceable names such as lpl or adipose. --- LPL/CMakeLists.txt | 12 + LPL/clara/clara.hpp | 1231 +++++++++ LPL/lpl.cpp | 56 + LPL/lpl.hpp | 246 ++ LPL/pstreams/pstream.h | 2255 +++++++++++++++++ bin/hipcc | 6 +- include/hip/hcc_detail/code_object_bundle.hpp | 50 +- .../0_Intro/module_api/launchKernelHcc.cpp | 3 +- samples/0_Intro/module_api/runKernel.cpp | 3 +- samples/0_Intro/module_api/vcpy_kernel.cpp | 2 +- .../0_Intro/module_api_global/runKernel.cpp | 7 +- .../0_Intro/module_api_global/vcpy_kernel.cpp | 4 +- src/code_object_bundle.cpp | 40 +- src/hip_module.cpp | 62 +- src/hsa_helpers.hpp | 32 + src/program_state.cpp | 42 +- .../synchronization/copy_coherency.cpp | 1 - 17 files changed, 3958 insertions(+), 94 deletions(-) create mode 100644 LPL/CMakeLists.txt create mode 100644 LPL/clara/clara.hpp create mode 100644 LPL/lpl.cpp create mode 100644 LPL/lpl.hpp create mode 100644 LPL/pstreams/pstream.h diff --git a/LPL/CMakeLists.txt b/LPL/CMakeLists.txt new file mode 100644 index 0000000000..26e3f6f0de --- /dev/null +++ b/LPL/CMakeLists.txt @@ -0,0 +1,12 @@ +add_executable(lpl lpl.cpp) +set_target_properties( + lpl PROPERTIES + CXX_STANDARD 11 + CXX_STANDARD_REQUIRED ON + CXX_EXTENSIONS OFF + RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) +target_include_directories(lpl PUBLIC ${PROJECT_SOURCE_DIR}/src) +# Install LPL if platform is hcc +if (HIP_PLATFORM STREQUAL "hcc") + install(TARGETS lpl RUNTIME DESTINATION bin) +endif () \ No newline at end of file diff --git a/LPL/clara/clara.hpp b/LPL/clara/clara.hpp new file mode 100644 index 0000000000..aa429e7a19 --- /dev/null +++ b/LPL/clara/clara.hpp @@ -0,0 +1,1231 @@ +// v1.0-develop.2 +// See https://github.com/philsquared/Clara + +#ifndef CLARA_HPP_INCLUDED +#define CLARA_HPP_INCLUDED + +#ifndef CLARA_CONFIG_CONSOLE_WIDTH +#define CLARA_CONFIG_CONSOLE_WIDTH 80 +#endif + +#ifndef CLARA_TEXTFLOW_CONFIG_CONSOLE_WIDTH +#define CLARA_TEXTFLOW_CONFIG_CONSOLE_WIDTH CLARA_CONFIG_CONSOLE_WIDTH +#endif + +// ----------- #included from clara_textflow.hpp ----------- + +// TextFlowCpp +// +// A single-header library for wrapping and laying out basic text, by Phil Nash +// +// This work is licensed under the BSD 2-Clause license. +// See the accompanying LICENSE file, or the one at https://opensource.org/licenses/BSD-2-Clause +// +// This project is hosted at https://github.com/philsquared/textflowcpp + +#ifndef CLARA_TEXTFLOW_HPP_INCLUDED +#define CLARA_TEXTFLOW_HPP_INCLUDED + +#include +#include +#include +#include + +#ifndef CLARA_TEXTFLOW_CONFIG_CONSOLE_WIDTH +#define CLARA_TEXTFLOW_CONFIG_CONSOLE_WIDTH 80 +#endif + + +namespace clara { namespace TextFlow { + + inline auto isWhitespace( char c ) -> bool { + static std::string chars = " \t\n\r"; + return chars.find( c ) != std::string::npos; + } + inline auto isBreakableBefore( char c ) -> bool { + static std::string chars = "[({<|"; + return chars.find( c ) != std::string::npos; + } + inline auto isBreakableAfter( char c ) -> bool { + static std::string chars = "])}>.,:;*+-=&/\\"; + return chars.find( c ) != std::string::npos; + } + + class Columns; + + class Column { + std::vector m_strings; + size_t m_width = CLARA_TEXTFLOW_CONFIG_CONSOLE_WIDTH; + size_t m_indent = 0; + size_t m_initialIndent = std::string::npos; + + public: + class iterator { + friend Column; + + Column const& m_column; + size_t m_stringIndex = 0; + size_t m_pos = 0; + + size_t m_len = 0; + size_t m_end = 0; + bool m_suffix = false; + + iterator( Column const& column, size_t stringIndex ) + : m_column( column ), + m_stringIndex( stringIndex ) + {} + + auto line() const -> std::string const& { return m_column.m_strings[m_stringIndex]; } + + auto isBoundary( size_t at ) const -> bool { + assert( at > 0 ); + assert( at <= line().size() ); + + return at == line().size() || + ( isWhitespace( line()[at] ) && !isWhitespace( line()[at-1] ) ) || + isBreakableBefore( line()[at] ) || + isBreakableAfter( line()[at-1] ); + } + + void calcLength() { + assert( m_stringIndex < m_column.m_strings.size() ); + + m_suffix = false; + auto width = m_column.m_width-indent(); + m_end = m_pos; + while( m_end < line().size() && line()[m_end] != '\n' ) + ++m_end; + + if( m_end < m_pos + width ) { + m_len = m_end - m_pos; + } + else { + size_t len = width; + while (len > 0 && !isBoundary(m_pos + len)) + --len; + while (len > 0 && isWhitespace( line()[m_pos + len - 1] )) + --len; + + if (len > 0) { + m_len = len; + } else { + m_suffix = true; + m_len = width - 1; + } + } + } + + auto indent() const -> size_t { + auto initial = m_pos == 0 && m_stringIndex == 0 ? m_column.m_initialIndent : std::string::npos; + return initial == std::string::npos ? m_column.m_indent : initial; + } + + auto addIndentAndSuffix(std::string const &plain) const -> std::string { + return std::string( indent(), ' ' ) + (m_suffix ? plain + "-" : plain); + } + + public: + explicit iterator( Column const& column ) : m_column( column ) { + assert( m_column.m_width > m_column.m_indent ); + assert( m_column.m_initialIndent == std::string::npos || m_column.m_width > m_column.m_initialIndent ); + calcLength(); + if( m_len == 0 ) + m_stringIndex++; // Empty string + } + + auto operator *() const -> std::string { + assert( m_stringIndex < m_column.m_strings.size() ); + assert( m_pos <= m_end ); + if( m_pos + m_column.m_width < m_end ) + return addIndentAndSuffix(line().substr(m_pos, m_len)); + else + return addIndentAndSuffix(line().substr(m_pos, m_end - m_pos)); + } + + auto operator ++() -> iterator& { + m_pos += m_len; + if( m_pos < line().size() && line()[m_pos] == '\n' ) + m_pos += 1; + else + while( m_pos < line().size() && isWhitespace( line()[m_pos] ) ) + ++m_pos; + + if( m_pos == line().size() ) { + m_pos = 0; + ++m_stringIndex; + } + if( m_stringIndex < m_column.m_strings.size() ) + calcLength(); + return *this; + } + auto operator ++(int) -> iterator { + iterator prev( *this ); + operator++(); + return prev; + } + + auto operator ==( iterator const& other ) const -> bool { + return + m_pos == other.m_pos && + m_stringIndex == other.m_stringIndex && + &m_column == &other.m_column; + } + auto operator !=( iterator const& other ) const -> bool { + return !operator==( other ); + } + }; + using const_iterator = iterator; + + explicit Column( std::string const& text ) { m_strings.push_back( text ); } + + auto width( size_t newWidth ) -> Column& { + assert( newWidth > 0 ); + m_width = newWidth; + return *this; + } + auto indent( size_t newIndent ) -> Column& { + m_indent = newIndent; + return *this; + } + auto initialIndent( size_t newIndent ) -> Column& { + m_initialIndent = newIndent; + return *this; + } + + auto width() const -> size_t { return m_width; } + auto begin() const -> iterator { return iterator( *this ); } + auto end() const -> iterator { return { *this, m_strings.size() }; } + + inline friend std::ostream& operator << ( std::ostream& os, Column const& col ) { + bool first = true; + for( auto line : col ) { + if( first ) + first = false; + else + os << "\n"; + os << line; + } + return os; + } + + auto operator + ( Column const& other ) -> Columns; + + auto toString() const -> std::string { + std::ostringstream oss; + oss << *this; + return oss.str(); + } + }; + + class Spacer : public Column { + + public: + explicit Spacer( size_t spaceWidth ) : Column( "" ) { + width( spaceWidth ); + } + }; + + class Columns { + std::vector m_columns; + + public: + + class iterator { + friend Columns; + struct EndTag {}; + + std::vector const& m_columns; + std::vector m_iterators; + size_t m_activeIterators; + + iterator( Columns const& columns, EndTag ) + : m_columns( columns.m_columns ), + m_activeIterators( 0 ) + { + m_iterators.reserve( m_columns.size() ); + + for( auto const& col : m_columns ) + m_iterators.push_back( col.end() ); + } + + public: + explicit iterator( Columns const& columns ) + : m_columns( columns.m_columns ), + m_activeIterators( m_columns.size() ) + { + m_iterators.reserve( m_columns.size() ); + + for( auto const& col : m_columns ) + m_iterators.push_back( col.begin() ); + } + + auto operator ==( iterator const& other ) const -> bool { + return m_iterators == other.m_iterators; + } + auto operator !=( iterator const& other ) const -> bool { + return m_iterators != other.m_iterators; + } + auto operator *() const -> std::string { + std::string row, padding; + + for( size_t i = 0; i < m_columns.size(); ++i ) { + auto width = m_columns[i].width(); + if( m_iterators[i] != m_columns[i].end() ) { + std::string col = *m_iterators[i]; + row += padding + col; + if( col.size() < width ) + padding = std::string( width - col.size(), ' ' ); + else + padding = ""; + } + else { + padding += std::string( width, ' ' ); + } + } + return row; + } + auto operator ++() -> iterator& { + for( size_t i = 0; i < m_columns.size(); ++i ) { + if (m_iterators[i] != m_columns[i].end()) + ++m_iterators[i]; + } + return *this; + } + auto operator ++(int) -> iterator { + iterator prev( *this ); + operator++(); + return prev; + } + }; + using const_iterator = iterator; + + auto begin() const -> iterator { return iterator( *this ); } + auto end() const -> iterator { return { *this, iterator::EndTag() }; } + + auto operator += ( Column const& col ) -> Columns& { + m_columns.push_back( col ); + return *this; + } + auto operator + ( Column const& col ) -> Columns { + Columns combined = *this; + combined += col; + return combined; + } + + inline friend std::ostream& operator << ( std::ostream& os, Columns const& cols ) { + + bool first = true; + for( auto line : cols ) { + if( first ) + first = false; + else + os << "\n"; + os << line; + } + return os; + } + + auto toString() const -> std::string { + std::ostringstream oss; + oss << *this; + return oss.str(); + } + }; + + inline auto Column::operator + ( Column const& other ) -> Columns { + Columns cols; + cols += *this; + cols += other; + return cols; + } +}} // namespace clara::TextFlow + +#endif // CLARA_TEXTFLOW_HPP_INCLUDED + +// ----------- end of #include from clara_textflow.hpp ----------- +// ........... back in clara.hpp + + +#include +#include +#include + +#if !defined(CLARA_PLATFORM_WINDOWS) && ( defined(WIN32) || defined(__WIN32__) || defined(_WIN32) || defined(_MSC_VER) ) +#define CLARA_PLATFORM_WINDOWS +#endif + +namespace clara { +namespace detail { + + // Traits for extracting arg and return type of lambdas (for single argument lambdas) + template + struct UnaryLambdaTraits : UnaryLambdaTraits {}; + + template + struct UnaryLambdaTraits { + static const bool isValid = false; + }; + + template + struct UnaryLambdaTraits { + static const bool isValid = true; + using ArgType = typename std::remove_const::type>::type;; + using ReturnType = ReturnT; + }; + + class TokenStream; + + // Transport for raw args (copied from main args, or supplied via init list for testing) + class Args { + friend TokenStream; + std::string m_exeName; + std::vector m_args; + + public: + Args( int argc, char *argv[] ) { + m_exeName = argv[0]; + for( int i = 1; i < argc; ++i ) + m_args.push_back( argv[i] ); + } + + Args( std::initializer_list args ) + : m_exeName( *args.begin() ), + m_args( args.begin()+1, args.end() ) + {} + + auto exeName() const -> std::string { + return m_exeName; + } + }; + + // Wraps a token coming from a token stream. These may not directly correspond to strings as a single string + // may encode an option + its argument if the : or = form is used + enum class TokenType { + Option, Argument + }; + struct Token { + TokenType type; + std::string token; + }; + + inline auto isOptPrefix( char c ) -> bool { + return c == '-' +#ifdef CLARA_PLATFORM_WINDOWS + || c == '/' +#endif + ; + } + + // Abstracts iterators into args as a stream of tokens, with option arguments uniformly handled + class TokenStream { + using Iterator = std::vector::const_iterator; + Iterator it; + Iterator itEnd; + std::vector m_tokenBuffer; + + void loadBuffer() { + m_tokenBuffer.resize( 0 ); + + // Skip any empty strings + while( it != itEnd && it->empty() ) + ++it; + + if( it != itEnd ) { + auto const &next = *it; + if( isOptPrefix( next[0] ) ) { + auto delimiterPos = next.find_first_of( " :=" ); + if( delimiterPos != std::string::npos ) { + m_tokenBuffer.push_back( { TokenType::Option, next.substr( 0, delimiterPos ) } ); + m_tokenBuffer.push_back( { TokenType::Argument, next.substr( delimiterPos + 1 ) } ); + } else { + if( next[1] != '-' && next.size() > 2 ) { + std::string opt = "- "; + for( size_t i = 1; i < next.size(); ++i ) { + opt[1] = next[i]; + m_tokenBuffer.push_back( { TokenType::Option, opt } ); + } + } else { + m_tokenBuffer.push_back( { TokenType::Option, next } ); + } + } + } else { + m_tokenBuffer.push_back( { TokenType::Argument, next } ); + } + } + } + + public: + explicit TokenStream( Args const &args ) : TokenStream( args.m_args.begin(), args.m_args.end() ) {} + + TokenStream( Iterator it, Iterator itEnd ) : it( it ), itEnd( itEnd ) { + loadBuffer(); + } + + explicit operator bool() const { + return !m_tokenBuffer.empty() || it != itEnd; + } + + auto count() const -> size_t { return m_tokenBuffer.size() + (itEnd - it); } + + auto operator*() const -> Token { + assert( !m_tokenBuffer.empty() ); + return m_tokenBuffer.front(); + } + + auto operator->() const -> Token const * { + assert( !m_tokenBuffer.empty() ); + return &m_tokenBuffer.front(); + } + + auto operator++() -> TokenStream & { + if( m_tokenBuffer.size() >= 2 ) { + m_tokenBuffer.erase( m_tokenBuffer.begin() ); + } else { + if( it != itEnd ) + ++it; + loadBuffer(); + } + return *this; + } + }; + + + class ResultBase { + public: + enum Type { + Ok, LogicError, RuntimeError + }; + + protected: + ResultBase( Type type ) : m_type( type ) {} + virtual ~ResultBase() = default; + + virtual void enforceOk() const = 0; + + Type m_type; + }; + + template + class ResultValueBase : public ResultBase { + public: + auto value() const -> T const & { + enforceOk(); + return m_value; + } + + protected: + ResultValueBase( Type type ) : ResultBase( type ) {} + + ResultValueBase( ResultValueBase const &other ) : ResultBase( other ) { + if( m_type == ResultBase::Ok ) + new( &m_value ) T( other.m_value ); + } + + ResultValueBase( Type, T const &value ) : ResultBase( Ok ) { + new( &m_value ) T( value ); + } + + auto operator=( ResultValueBase const &other ) -> ResultValueBase & { + if( m_type == ResultBase::Ok ) + m_value.~T(); + ResultBase::operator=(other); + if( m_type == ResultBase::Ok ) + new( &m_value ) T( other.m_value ); + return *this; + } + + ~ResultValueBase() { + if( m_type == Ok ) + m_value.~T(); + } + + union { + T m_value; + }; + }; + + template<> + class ResultValueBase : public ResultBase { + protected: + using ResultBase::ResultBase; + }; + + template + class BasicResult : public ResultValueBase { + public: + template + explicit BasicResult( BasicResult const &other ) + : ResultValueBase( other.type() ), + m_errorMessage( other.errorMessage() ) + { + assert( type() != ResultBase::Ok ); + } + + template + static auto ok( U const &value ) -> BasicResult { return { ResultBase::Ok, value }; } + static auto ok() -> BasicResult { return { ResultBase::Ok }; } + static auto logicError( std::string const &message ) -> BasicResult { return { ResultBase::LogicError, message }; } + static auto runtimeError( std::string const &message ) -> BasicResult { return { ResultBase::RuntimeError, message }; } + + explicit operator bool() const { return m_type == ResultBase::Ok; } + auto type() const -> ResultBase::Type { return m_type; } + auto errorMessage() const -> std::string { return m_errorMessage; } + + protected: + virtual void enforceOk() const { + // !TBD: If no exceptions, std::terminate here or something + switch( m_type ) { + case ResultBase::LogicError: + throw std::logic_error( m_errorMessage ); + case ResultBase::RuntimeError: + throw std::runtime_error( m_errorMessage ); + case ResultBase::Ok: + break; + } + } + + std::string m_errorMessage; // Only populated if resultType is an error + + BasicResult( ResultBase::Type type, std::string const &message ) + : ResultValueBase(type), + m_errorMessage(message) + { + assert( m_type != ResultBase::Ok ); + } + + using ResultValueBase::ResultValueBase; + using ResultBase::m_type; + }; + + enum class ParseResultType { + Matched, NoMatch, ShortCircuitAll, ShortCircuitSame + }; + + class ParseState { + public: + + ParseState( ParseResultType type, TokenStream const &remainingTokens ) + : m_type(type), + m_remainingTokens( remainingTokens ) + {} + + auto type() const -> ParseResultType { return m_type; } + auto remainingTokens() const -> TokenStream { return m_remainingTokens; } + + private: + ParseResultType m_type; + TokenStream m_remainingTokens; + }; + + using Result = BasicResult; + using ParserResult = BasicResult; + using InternalParseResult = BasicResult; + + struct HelpColumns { + std::string left; + std::string right; + }; + + template + inline auto convertInto( std::string const &source, T& target ) -> ParserResult { + std::stringstream ss; + ss << source; + ss >> target; + if( ss.fail() ) + return ParserResult::runtimeError( "Unable to convert '" + source + "' to destination type" ); + else + return ParserResult::ok( ParseResultType::Matched ); + } + inline auto convertInto( std::string const &source, std::string& target ) -> ParserResult { + target = source; + return ParserResult::ok( ParseResultType::Matched ); + } + inline auto convertInto( std::string const &source, bool &target ) -> ParserResult { + std::string srcLC = source; + std::transform( srcLC.begin(), srcLC.end(), srcLC.begin(), []( char c ) { return static_cast( ::tolower(c) ); } ); + if (srcLC == "y" || srcLC == "1" || srcLC == "true" || srcLC == "yes" || srcLC == "on") + target = true; + else if (srcLC == "n" || srcLC == "0" || srcLC == "false" || srcLC == "no" || srcLC == "off") + target = false; + else + return ParserResult::runtimeError( "Expected a boolean value but did not recognise: '" + source + "'" ); + return ParserResult::ok( ParseResultType::Matched ); + } + + struct BoundRefBase { + BoundRefBase() = default; + BoundRefBase( BoundRefBase const & ) = delete; + BoundRefBase( BoundRefBase && ) = delete; + BoundRefBase &operator=( BoundRefBase const & ) = delete; + BoundRefBase &operator=( BoundRefBase && ) = delete; + + virtual ~BoundRefBase() = default; + + virtual auto isFlag() const -> bool = 0; + virtual auto isContainer() const -> bool { return false; } + virtual auto setValue( std::string const &arg ) -> ParserResult = 0; + virtual auto setFlag( bool flag ) -> ParserResult = 0; + }; + + struct BoundValueRefBase : BoundRefBase { + auto isFlag() const -> bool override { return false; } + + auto setFlag( bool ) -> ParserResult override { + return ParserResult::logicError( "Flags can only be set on boolean fields" ); + } + }; + + struct BoundFlagRefBase : BoundRefBase { + auto isFlag() const -> bool override { return true; } + + auto setValue( std::string const &arg ) -> ParserResult override { + bool flag; + auto result = convertInto( arg, flag ); + if( result ) + setFlag( flag ); + return result; + } + }; + + template + struct BoundRef : BoundValueRefBase { + T &m_ref; + + explicit BoundRef( T &ref ) : m_ref( ref ) {} + + auto setValue( std::string const &arg ) -> ParserResult override { + return convertInto( arg, m_ref ); + } + }; + + template + struct BoundRef> : BoundValueRefBase { + std::vector &m_ref; + + explicit BoundRef( std::vector &ref ) : m_ref( ref ) {} + + auto isContainer() const -> bool override { return true; } + + auto setValue( std::string const &arg ) -> ParserResult override { + T temp; + auto result = convertInto( arg, temp ); + if( result ) + m_ref.push_back( temp ); + return result; + } + }; + + struct BoundFlagRef : BoundFlagRefBase { + bool &m_ref; + + explicit BoundFlagRef( bool &ref ) : m_ref( ref ) {} + + auto setFlag( bool flag ) -> ParserResult override { + m_ref = flag; + return ParserResult::ok( ParseResultType::Matched ); + } + }; + + template + struct LambdaInvoker { + static_assert( std::is_same::value, "Lambda must return void or clara::ParserResult" ); + + template + static auto invoke( L const &lambda, ArgType const &arg ) -> ParserResult { + return lambda( arg ); + } + }; + + template<> + struct LambdaInvoker { + template + static auto invoke( L const &lambda, ArgType const &arg ) -> ParserResult { + lambda( arg ); + return ParserResult::ok( ParseResultType::Matched ); + } + }; + + template + inline auto invokeLambda( L const &lambda, std::string const &arg ) -> ParserResult { + ArgType temp; + auto result = convertInto( arg, temp ); + return !result + ? result + : LambdaInvoker::ReturnType>::invoke( lambda, temp ); + }; + + + template + struct BoundLambda : BoundValueRefBase { + L m_lambda; + + static_assert( UnaryLambdaTraits::isValid, "Supplied lambda must take exactly one argument" ); + explicit BoundLambda( L const &lambda ) : m_lambda( lambda ) {} + + auto setValue( std::string const &arg ) -> ParserResult override { + return invokeLambda::ArgType>( m_lambda, arg ); + } + }; + + template + struct BoundFlagLambda : BoundFlagRefBase { + L m_lambda; + + static_assert( UnaryLambdaTraits::isValid, "Supplied lambda must take exactly one argument" ); + static_assert( std::is_same::ArgType, bool>::value, "flags must be boolean" ); + + explicit BoundFlagLambda( L const &lambda ) : m_lambda( lambda ) {} + + auto setFlag( bool flag ) -> ParserResult override { + return LambdaInvoker::ReturnType>::invoke( m_lambda, flag ); + } + }; + + enum class Optionality { Optional, Required }; + + struct Parser; + + class ParserBase { + public: + virtual ~ParserBase() = default; + virtual auto validate() const -> Result { return Result::ok(); } + virtual auto parse( std::string const& exeName, TokenStream const &tokens) const -> InternalParseResult = 0; + virtual auto cardinality() const -> size_t { return 1; } + + auto parse( Args const &args ) const -> InternalParseResult { + return parse( args.exeName(), TokenStream( args ) ); + } + }; + + template + class ComposableParserImpl : public ParserBase { + public: + template + auto operator|( T const &other ) const -> Parser; + }; + + // Common code and state for Args and Opts + template + class ParserRefImpl : public ComposableParserImpl { + protected: + Optionality m_optionality = Optionality::Optional; + std::shared_ptr m_ref; + std::string m_hint; + std::string m_description; + + explicit ParserRefImpl( std::shared_ptr const &ref ) : m_ref( ref ) {} + + public: + template + ParserRefImpl( T &ref, std::string const &hint ) + : m_ref( std::make_shared>( ref ) ), + m_hint( hint ) + {} + + template + ParserRefImpl( LambdaT const &ref, std::string const &hint ) + : m_ref( std::make_shared>( ref ) ), + m_hint(hint) + {} + + auto operator()( std::string const &description ) -> DerivedT & { + m_description = description; + return static_cast( *this ); + } + + auto optional() -> DerivedT & { + m_optionality = Optionality::Optional; + return static_cast( *this ); + }; + + auto required() -> DerivedT & { + m_optionality = Optionality::Required; + return static_cast( *this ); + }; + + auto isOptional() const -> bool { + return m_optionality == Optionality::Optional; + } + + auto cardinality() const -> size_t override { + if( m_ref->isContainer() ) + return 0; + else + return 1; + } + + auto hint() const -> std::string { return m_hint; } + }; + + class ExeName : public ComposableParserImpl { + std::shared_ptr m_name; + std::shared_ptr m_ref; + + template + static auto makeRef(LambdaT const &lambda) -> std::shared_ptr { + return std::make_shared>( lambda) ; + } + + public: + ExeName() : m_name( std::make_shared( "" ) ) {} + + explicit ExeName( std::string &ref ) : ExeName() { + m_ref = std::make_shared>( ref ); + } + + template + explicit ExeName( LambdaT const& lambda ) : ExeName() { + m_ref = std::make_shared>( lambda ); + } + + // The exe name is not parsed out of the normal tokens, but is handled specially + auto parse( std::string const&, TokenStream const &tokens ) const -> InternalParseResult override { + return InternalParseResult::ok( ParseState( ParseResultType::NoMatch, tokens ) ); + } + + auto name() const -> std::string { return *m_name; } + auto set( std::string const& newName ) -> ParserResult { + + auto lastSlash = newName.find_last_of( "\\/" ); + auto filename = ( lastSlash == std::string::npos ) + ? newName + : newName.substr( lastSlash+1 ); + + *m_name = filename; + if( m_ref ) + return m_ref->setValue( filename ); + else + return ParserResult::ok( ParseResultType::Matched ); + } + }; + + class Arg : public ParserRefImpl { + public: + using ParserRefImpl::ParserRefImpl; + + auto parse( std::string const &, TokenStream const &tokens ) const -> InternalParseResult override { + auto validationResult = validate(); + if( !validationResult ) + return InternalParseResult( validationResult ); + + auto remainingTokens = tokens; + auto const &token = *remainingTokens; + if( token.type != TokenType::Argument ) + return InternalParseResult::ok( ParseState( ParseResultType::NoMatch, remainingTokens ) ); + + auto result = m_ref->setValue( remainingTokens->token ); + if( !result ) + return InternalParseResult( result ); + else + return InternalParseResult::ok( ParseState( ParseResultType::Matched, ++remainingTokens ) ); + } + }; + + inline auto normaliseOpt( std::string const &optName ) -> std::string { +#ifdef CLARA_PLATFORM_WINDOWS + if( optName[0] == '/' ) + return "-" + optName.substr( 1 ); + else +#endif + return optName; + } + + class Opt : public ParserRefImpl { + protected: + std::vector m_optNames; + + public: + template + explicit Opt( LambdaT const &ref ) : ParserRefImpl( std::make_shared>( ref ) ) {} + + explicit Opt( bool &ref ) : ParserRefImpl( std::make_shared( ref ) ) {} + + template + Opt( LambdaT const &ref, std::string const &hint ) : ParserRefImpl( ref, hint ) {} + + template + Opt( T &ref, std::string const &hint ) : ParserRefImpl( ref, hint ) {} + + auto operator[]( std::string const &optName ) -> Opt & { + m_optNames.push_back( optName ); + return *this; + } + + auto getHelpColumns() const -> std::vector { + std::ostringstream oss; + bool first = true; + for( auto const &opt : m_optNames ) { + if (first) + first = false; + else + oss << ", "; + oss << opt; + } + if( !m_hint.empty() ) + oss << " <" << m_hint << ">"; + return { { oss.str(), m_description } }; + } + + auto isMatch( std::string const &optToken ) const -> bool { + auto normalisedToken = normaliseOpt( optToken ); + for( auto const &name : m_optNames ) { + if( normaliseOpt( name ) == normalisedToken ) + return true; + } + return false; + } + + using ParserBase::parse; + + auto parse( std::string const&, TokenStream const &tokens ) const -> InternalParseResult override { + auto validationResult = validate(); + if( !validationResult ) + return InternalParseResult( validationResult ); + + auto remainingTokens = tokens; + if( remainingTokens && remainingTokens->type == TokenType::Option ) { + auto const &token = *remainingTokens; + if( isMatch(token.token ) ) { + if( m_ref->isFlag() ) { + auto result = m_ref->setFlag( true ); + if( !result ) + return InternalParseResult( result ); + if( result.value() == ParseResultType::ShortCircuitAll ) + return InternalParseResult::ok( ParseState( result.value(), remainingTokens ) ); + } else { + ++remainingTokens; + if( !remainingTokens ) + return InternalParseResult::runtimeError( "Expected argument following " + token.token ); + auto const &argToken = *remainingTokens; + if( argToken.type != TokenType::Argument ) + return InternalParseResult::runtimeError( "Expected argument following " + token.token ); + auto result = m_ref->setValue( argToken.token ); + if( !result ) + return InternalParseResult( result ); + if( result.value() == ParseResultType::ShortCircuitAll ) + return InternalParseResult::ok( ParseState( result.value(), remainingTokens ) ); + } + return InternalParseResult::ok( ParseState( ParseResultType::Matched, ++remainingTokens ) ); + } + } + return InternalParseResult::ok( ParseState( ParseResultType::NoMatch, remainingTokens ) ); + } + + auto validate() const -> Result override { + if( m_optNames.empty() ) + return Result::logicError( "No options supplied to Opt" ); + for( auto const &name : m_optNames ) { + if( name.empty() ) + return Result::logicError( "Option name cannot be empty" ); +#ifdef CLARA_PLATFORM_WINDOWS + if( name[0] != '-' && name[0] != '/' ) + return Result::logicError( "Option name must begin with '-' or '/'" ); +#else + if( name[0] != '-' ) + return Result::logicError( "Option name must begin with '-'" ); +#endif + } + return ParserRefImpl::validate(); + } + }; + + struct Help : Opt { + Help( bool &showHelpFlag ) + : Opt([&]( bool flag ) { + showHelpFlag = flag; + return ParserResult::ok( ParseResultType::ShortCircuitAll ); + }) + { + static_cast( *this ) + ("display usage information") + ["-?"]["-h"]["--help"] + .optional(); + } + }; + + + struct Parser : ParserBase { + + mutable ExeName m_exeName; + std::vector m_options; + std::vector m_args; + + auto operator|=( ExeName const &exeName ) -> Parser & { + m_exeName = exeName; + return *this; + } + + auto operator|=( Arg const &arg ) -> Parser & { + m_args.push_back(arg); + return *this; + } + + auto operator|=( Opt const &opt ) -> Parser & { + m_options.push_back(opt); + return *this; + } + + auto operator|=( Parser const &other ) -> Parser & { + m_options.insert(m_options.end(), other.m_options.begin(), other.m_options.end()); + m_args.insert(m_args.end(), other.m_args.begin(), other.m_args.end()); + return *this; + } + + template + auto operator|( T const &other ) const -> Parser { + return Parser( *this ) |= other; + } + + auto getHelpColumns() const -> std::vector { + std::vector cols; + for (auto const &o : m_options) { + auto childCols = o.getHelpColumns(); + cols.insert( cols.end(), childCols.begin(), childCols.end() ); + } + return cols; + } + + void writeToStream( std::ostream &os ) const { + if (!m_exeName.name().empty()) { + os << "usage:\n" << " " << m_exeName.name() << " "; + bool required = true, first = true; + for( auto const &arg : m_args ) { + if (first) + first = false; + else + os << " "; + if( arg.isOptional() && required ) { + os << "["; + required = false; + } + os << "<" << arg.hint() << ">"; + if( arg.cardinality() == 0 ) + os << " ... "; + } + if( !required ) + os << "]"; + if( !m_options.empty() ) + os << " options"; + os << "\n\nwhere options are:" << std::endl; + } + + auto rows = getHelpColumns(); + size_t consoleWidth = CLARA_CONFIG_CONSOLE_WIDTH; + size_t optWidth = 0; + for( auto const &cols : rows ) + optWidth = (std::max)(optWidth, cols.left.size() + 2); + + for( auto const &cols : rows ) { + auto row = + TextFlow::Column( cols.left ).width( optWidth ).indent( 2 ) + + TextFlow::Spacer(4) + + TextFlow::Column( cols.right ).width( consoleWidth - 7 - optWidth ); + os << row << std::endl; + } + } + + friend auto operator<<( std::ostream &os, Parser const &parser ) -> std::ostream& { + parser.writeToStream( os ); + return os; + } + + auto validate() const -> Result override { + for( auto const &opt : m_options ) { + auto result = opt.validate(); + if( !result ) + return result; + } + for( auto const &arg : m_args ) { + auto result = arg.validate(); + if( !result ) + return result; + } + return Result::ok(); + } + + using ParserBase::parse; + + auto parse( std::string const& exeName, TokenStream const &tokens ) const -> InternalParseResult override { + + struct ParserInfo { + ParserBase const* parser = nullptr; + size_t count = 0; + }; + const size_t totalParsers = m_options.size() + m_args.size(); + assert( totalParsers < 512 ); + // ParserInfo parseInfos[totalParsers]; // <-- this is what we really want to do + ParserInfo parseInfos[512]; + + { + size_t i = 0; + for (auto const &opt : m_options) parseInfos[i++].parser = &opt; + for (auto const &arg : m_args) parseInfos[i++].parser = &arg; + } + + m_exeName.set( exeName ); + + auto result = InternalParseResult::ok( ParseState( ParseResultType::NoMatch, tokens ) ); + while( result.value().remainingTokens() ) { + bool tokenParsed = false; + + for( size_t i = 0; i < totalParsers; ++i ) { + auto& parseInfo = parseInfos[i]; + if( parseInfo.parser->cardinality() == 0 || parseInfo.count < parseInfo.parser->cardinality() ) { + result = parseInfo.parser->parse(exeName, result.value().remainingTokens()); + if (!result) + return result; + if (result.value().type() != ParseResultType::NoMatch) { + tokenParsed = true; + ++parseInfo.count; + break; + } + } + } + + if( result.value().type() == ParseResultType::ShortCircuitAll ) + return result; + if( !tokenParsed ) + return InternalParseResult::runtimeError( "Unrecognised token: " + result.value().remainingTokens()->token ); + } + // !TBD Check missing required options + return result; + } + }; + + template + template + auto ComposableParserImpl::operator|( T const &other ) const -> Parser { + return Parser() | static_cast( *this ) | other; + } +} // namespace detail + + +// A Combined parser +using detail::Parser; + +// A parser for options +using detail::Opt; + +// A parser for arguments +using detail::Arg; + +// Wrapper for argc, argv from main() +using detail::Args; + +// Specifies the name of the executable +using detail::ExeName; + +// Convenience wrapper for option parser that specifies the help option +using detail::Help; + +// enum of result types from a parse +using detail::ParseResultType; + +// Result type for parser operation +using detail::ParserResult; + + +} // namespace clara + +#endif // CLARA_HPP_INCLUDED diff --git a/LPL/lpl.cpp b/LPL/lpl.cpp new file mode 100644 index 0000000000..0b99f758c2 --- /dev/null +++ b/LPL/lpl.cpp @@ -0,0 +1,56 @@ +#include "lpl.hpp" + +#include +#include +#include +#include +#include +#include + +using namespace clara; +using namespace hip_impl; +using namespace std; + +int main(int argc, char** argv) +{ + try { + if (!hipcc_and_lpl_colocated()) { + throw runtime_error{ + "The LPL executable and hipcc must be in the same directory."}; + } + + bool help = false; + string flags; + string output; + vector sources; + string targets; + + auto cmd = cmdline_parser(help, sources, targets, flags, output); + + const auto r = cmd.parse(Args{argc, argv}); + + if (!r) throw runtime_error{r.errorMessage()}; + + if (help) cout << cmd << endl; + else { + if (sources.empty()) throw runtime_error{"No inputs specified."}; + + auto tmp = tokenize_targets(targets); + if (tmp.empty()) { + tmp.assign(amdgpu_targets().cbegin(), amdgpu_targets().cend()); + } + else validate_targets(tmp); + + if (output.empty()) for (auto&& x : tmp) output += x; + + generate_fat_binary(sources, tmp, flags, output); + } + } + catch (const exception& ex) { + cerr << ex.what() << endl; + + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} \ No newline at end of file diff --git a/LPL/lpl.hpp b/LPL/lpl.hpp new file mode 100644 index 0000000000..f3d0c537b4 --- /dev/null +++ b/LPL/lpl.hpp @@ -0,0 +1,246 @@ +#include "clara/clara.hpp" +#include "pstreams/pstream.h" +#include "../src/elfio/elfio.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace hip_impl +{ + inline + const std::unordered_set& amdgpu_targets() + { // The evolving list lives at: + // https://www.llvm.org/docs/AMDGPUUsage.html#processors. + static const std::unordered_set r{ + "gfx701", "gfx801", "gfx802", "gfx803", "gfx900"}; + + return r; + } + + inline + const std::string& fat_binary_extension() + { + static const std::string r{".adipose"}; + + return r; + } + + inline + const std::string& kernel_section() + { + static const std::string r{".kernel"}; + + return r; + } + + inline + const std::string& path_to_self() + { + static constexpr const char self[] = "/proc/self/exe"; + + static std::string r(PATH_MAX, '\0'); + static std::once_flag f; + + std::call_once(f, []() { + decltype(readlink(self, &r.front(), r.size())) read_cnt; + do { + read_cnt = readlink(self, &r.front(), r.size()); + } while (read_cnt == -1); + + r.resize(read_cnt); + }); + + return r; + } + + inline + const std::string& path_to_hipcc() + { + assert(!path_to_self().empty()); + + static const auto r = path_to_self().substr( + 0, path_to_self().find_last_of('/')) += "/hipcc"; + + return r; + } + + inline + std::string make_hipcc_call( + const std::vector& sources, + const std::vector& targets, + const std::string& flags, + const std::string& hipcc_output) + { + assert(!sources.empty() && !targets.empty() && !hipcc_output.empty()); + + std::string r{path_to_hipcc() + ' '}; + + for (auto&& x : sources) r += x + ' '; + r += "-o " + hipcc_output + ' '; + for (auto&& x : targets) r += "--amdgpu-target=" + x + ' '; + r += flags + " -fPIC -shared"; + + return r; + } + + inline + void copy_kernel_section_to_fat_binary( + const std::string& tmp, const std::string& output) + { + ELFIO::elfio reader; + if (!reader.load(tmp)) { + throw std::runtime_error{ + "The result of the compilation is inaccessible."}; + } + + const auto it = std::find_if( + reader.sections.begin(), + reader.sections.end(), + [](const ELFIO::section* x) { + return x->get_name() == kernel_section(); + }); + + std::ofstream out{output + fat_binary_extension()}; + + if (it == reader.sections.end()) { + std::cerr << "Warning: no kernels were generated; fat binary shall " + "be empty." << std::endl; + } + else { + std::copy_n( + (*it)->get_data(), + (*it)->get_size(), + std::ostreambuf_iterator{out}); + } + } + + inline + void generate_fat_binary( + const std::vector& sources, + const std::vector& targets, + const std::string& flags, + const std::string& output) + { + static const auto d = [](const std::string* f) { remove(f->c_str()); }; + + std::unique_ptr tmp{&output, d}; + + redi::ipstream hipcc{ + make_hipcc_call(sources, targets, flags, *tmp), + redi::pstream::pstderr}; + + if (!hipcc.is_open()) { + throw std::runtime_error{"Compiler invocation failed."}; + } + + std::string log; + while (std::getline(hipcc, log)) std::cout << log << '\n'; + + hipcc.close(); + + if (hipcc.rdbuf()->exited() && + hipcc.rdbuf()->status() != EXIT_SUCCESS) { + throw std::runtime_error{"Compilation failed."}; + } + + copy_kernel_section_to_fat_binary(*tmp, output); + } + + inline + bool file_exists(const std::string& path_to) + { + return static_cast(std::ifstream{path_to}); + } + + inline + bool hipcc_and_lpl_colocated() + { + if (path_to_self().empty()) return false; + + return file_exists(path_to_hipcc()); + } + + inline + std::vector tokenize_targets(const std::string& x) + { // TODO: move to regular expressions once we clarify the need to support + // ancient standard library implementations. + if (x.empty()) return {}; + + static constexpr const char valid_characters[] = "gfx0123456789,"; + + if (x.find_first_not_of(valid_characters) != std::string::npos) { + throw std::runtime_error{"Invalid target string: " + x}; + } + + std::vector r; + + auto it = x.cbegin(); + do { + auto it1 = std::find(it, x.cend(), ','); + r.emplace_back(it, it1); + + if (it1 == x.cend()) break; + + it = ++it1; + } while (true); + + return r; + } + + inline + void validate_targets(const std::vector& x) + { + assert(!x.empty()); + + for (auto&& t : x) { + static const std::string digits{"0123456789"}; + static const std::string pre{"gfx"}; + + if (t.find(pre) != 0 || + t.find_first_not_of(digits, pre.size()) != std::string::npos) { + throw std::runtime_error{"Invalid target: " + t}; + } + + if (amdgpu_targets().find(t) == amdgpu_targets().cend()) { + std::cerr << "Warning: target " << t + << " has not been validated yet; it may be invalid." + << std::endl; + } + } + } + + inline + clara::Parser cmdline_parser( + bool& help, + std::vector& sources, + std::string& targets, + std::string& flags, + std::string& output) + { + return + clara::Opt{flags, "\"-v -DMACRO etc.\""} + ["-f"]["--flags"]( + "flags for compilation; must be valid for hipcc.") | + clara::Help{help} | + clara::Opt{output, "filename"} + ["-o"]["--output"]( + "name of fat-binary output file; the binary format of the " + "file is documented at: https://reviews.llvm.org/D13909.") | + clara::Arg{sources, "a.cpp b.cpp etc."}( + "inputs for compilation; must contain valid C++ code.") | + clara::Opt{targets, "gfx803,gfx900 etc."} + ["-t"]["--targets"]( + "targets for AMDGPU lowering; must be one of the processors" + " with ROCm support from " + "https://www.llvm.org/docs/AMDGPUUsage.html#processors."); + } +} \ No newline at end of file diff --git a/LPL/pstreams/pstream.h b/LPL/pstreams/pstream.h new file mode 100644 index 0000000000..28cbeadb1e --- /dev/null +++ b/LPL/pstreams/pstream.h @@ -0,0 +1,2255 @@ +// PStreams - POSIX Process I/O for C++ + +// Copyright (C) 2001 - 2017 Jonathan Wakely +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) +// + +/** + * @file pstream.h + * @brief Declares all PStreams classes. + * @author Jonathan Wakely + * + * Defines classes redi::ipstream, redi::opstream, redi::pstream + * and redi::rpstream. + */ + +#ifndef REDI_PSTREAM_H_SEEN +#define REDI_PSTREAM_H_SEEN + +#include +#include +#include +#include +#include +#include +#include // for min() +#include // for errno +#include // for size_t, NULL +#include // for exit() +#include // for pid_t +#include // for waitpid() +#include // for ioctl() and FIONREAD +#if defined(__sun) +# include // for FIONREAD on Solaris 2.5 +#endif +#include // for pipe() fork() exec() and filedes functions +#include // for kill() +#include // for fcntl() +#if REDI_EVISCERATE_PSTREAMS +# include // for FILE, fdopen() +#endif + + +/// The library version. +#define PSTREAMS_VERSION 0x0101 // 1.0.1 + +/** + * @namespace redi + * @brief All PStreams classes are declared in namespace redi. + * + * Like the standard iostreams, PStreams is a set of class templates, + * taking a character type and traits type. As with the standard streams + * they are most likely to be used with @c char and the default + * traits type, so typedefs for this most common case are provided. + * + * The @c pstream_common class template is not intended to be used directly, + * it is used internally to provide the common functionality for the + * other stream classes. + */ +namespace redi +{ + /// Common base class providing constants and typenames. + struct pstreams + { + /// Type used to specify how to connect to the process. + typedef std::ios_base::openmode pmode; + + /// Type used to hold the arguments for a command. + typedef std::vector argv_type; + + /// Type used for file descriptors. + typedef int fd_type; + + static const pmode pstdin = std::ios_base::out; ///< Write to stdin + static const pmode pstdout = std::ios_base::in; ///< Read from stdout + static const pmode pstderr = std::ios_base::app; ///< Read from stderr + + /// Create a new process group for the child process. + static const pmode newpg = std::ios_base::trunc; + + protected: + enum { bufsz = 32 }; ///< Size of pstreambuf buffers. + enum { pbsz = 2 }; ///< Number of putback characters kept. + }; + + /// Class template for stream buffer. + template > + class basic_pstreambuf + : public std::basic_streambuf + , public pstreams + { + public: + // Type definitions for dependent types + typedef CharT char_type; + typedef Traits traits_type; + typedef typename traits_type::int_type int_type; + typedef typename traits_type::off_type off_type; + typedef typename traits_type::pos_type pos_type; + /** @deprecated use pstreams::fd_type instead. */ + typedef fd_type fd_t; + + /// Default constructor. + basic_pstreambuf(); + + /// Constructor that initialises the buffer with @a cmd. + basic_pstreambuf(const std::string& cmd, pmode mode); + + /// Constructor that initialises the buffer with @a file and @a argv. + basic_pstreambuf( const std::string& file, + const argv_type& argv, + pmode mode ); + + /// Destructor. + ~basic_pstreambuf(); + + /// Initialise the stream buffer with @a cmd. + basic_pstreambuf* + open(const std::string& cmd, pmode mode); + + /// Initialise the stream buffer with @a file and @a argv. + basic_pstreambuf* + open(const std::string& file, const argv_type& argv, pmode mode); + + /// Close the stream buffer and wait for the process to exit. + basic_pstreambuf* + close(); + + /// Send a signal to the process. + basic_pstreambuf* + kill(int signal = SIGTERM); + + /// Send a signal to the process' process group. + basic_pstreambuf* + killpg(int signal = SIGTERM); + + /// Close the pipe connected to the process' stdin. + void + peof(); + + /// Change active input source. + bool + read_err(bool readerr = true); + + /// Report whether the stream buffer has been initialised. + bool + is_open() const; + + /// Report whether the process has exited. + bool + exited(); + +#if REDI_EVISCERATE_PSTREAMS + /// Obtain FILE pointers for each of the process' standard streams. + std::size_t + fopen(FILE*& in, FILE*& out, FILE*& err); +#endif + + /// Return the exit status of the process. + int + status() const; + + /// Return the error number (errno) for the most recent failed operation. + int + error() const; + + protected: + /// Transfer characters to the pipe when character buffer overflows. + int_type + overflow(int_type c); + + /// Transfer characters from the pipe when the character buffer is empty. + int_type + underflow(); + + /// Make a character available to be returned by the next extraction. + int_type + pbackfail(int_type c = traits_type::eof()); + + /// Write any buffered characters to the stream. + int + sync(); + + /// Insert multiple characters into the pipe. + std::streamsize + xsputn(const char_type* s, std::streamsize n); + + /// Insert a sequence of characters into the pipe. + std::streamsize + write(const char_type* s, std::streamsize n); + + /// Extract a sequence of characters from the pipe. + std::streamsize + read(char_type* s, std::streamsize n); + + /// Report how many characters can be read from active input without blocking. + std::streamsize + showmanyc(); + + protected: + /// Enumerated type to indicate whether stdout or stderr is to be read. + enum buf_read_src { rsrc_out = 0, rsrc_err = 1 }; + + /// Initialise pipes and fork process. + pid_t + fork(pmode mode); + + /// Wait for the child process to exit. + int + wait(bool nohang = false); + + /// Return the file descriptor for the output pipe. + fd_type& + wpipe(); + + /// Return the file descriptor for the active input pipe. + fd_type& + rpipe(); + + /// Return the file descriptor for the specified input pipe. + fd_type& + rpipe(buf_read_src which); + + void + create_buffers(pmode mode); + + void + destroy_buffers(pmode mode); + + /// Writes buffered characters to the process' stdin pipe. + bool + empty_buffer(); + + bool + fill_buffer(bool non_blocking = false); + + /// Return the active input buffer. + char_type* + rbuffer(); + + buf_read_src + switch_read_buffer(buf_read_src); + + private: + basic_pstreambuf(const basic_pstreambuf&); + basic_pstreambuf& operator=(const basic_pstreambuf&); + + void + init_rbuffers(); + + pid_t ppid_; // pid of process + fd_type wpipe_; // pipe used to write to process' stdin + fd_type rpipe_[2]; // two pipes to read from, stdout and stderr + char_type* wbuffer_; + char_type* rbuffer_[2]; + char_type* rbufstate_[3]; + /// Index into rpipe_[] to indicate active source for read operations. + buf_read_src rsrc_; + int status_; // hold exit status of child process + int error_; // hold errno if fork() or exec() fails + }; + + /// Class template for common base class. + template > + class pstream_common + : virtual public std::basic_ios + , virtual public pstreams + { + protected: + typedef basic_pstreambuf streambuf_type; + + typedef pstreams::pmode pmode; + typedef pstreams::argv_type argv_type; + + /// Default constructor. + pstream_common(); + + /// Constructor that initialises the stream by starting a process. + pstream_common(const std::string& cmd, pmode mode); + + /// Constructor that initialises the stream by starting a process. + pstream_common(const std::string& file, const argv_type& argv, pmode mode); + + /// Pure virtual destructor. + virtual + ~pstream_common() = 0; + + /// Start a process. + void + do_open(const std::string& cmd, pmode mode); + + /// Start a process. + void + do_open(const std::string& file, const argv_type& argv, pmode mode); + + public: + /// Close the pipe. + void + close(); + + /// Report whether the stream's buffer has been initialised. + bool + is_open() const; + + /// Return the command used to initialise the stream. + const std::string& + command() const; + + /// Return a pointer to the stream buffer. + streambuf_type* + rdbuf() const; + +#if REDI_EVISCERATE_PSTREAMS + /// Obtain FILE pointers for each of the process' standard streams. + std::size_t + fopen(FILE*& in, FILE*& out, FILE*& err); +#endif + + protected: + std::string command_; ///< The command used to start the process. + streambuf_type buf_; ///< The stream buffer. + }; + + + /** + * @class basic_ipstream + * @brief Class template for Input PStreams. + * + * Reading from an ipstream reads the command's standard output and/or + * standard error (depending on how the ipstream is opened) + * and the command's standard input is the same as that of the process + * that created the object, unless altered by the command itself. + */ + + template > + class basic_ipstream + : public std::basic_istream + , public pstream_common + , virtual public pstreams + { + typedef std::basic_istream istream_type; + typedef pstream_common pbase_type; + + using pbase_type::buf_; // declare name in this scope + + // Ensure a basic_ipstream will read from at least one pipe + pmode readable(pmode mode) + { + if (!(mode & (pstdout|pstderr))) + mode |= pstdout; + return mode; + } + + public: + /// Type used to specify how to connect to the process. + typedef typename pbase_type::pmode pmode; + + /// Type used to hold the arguments for a command. + typedef typename pbase_type::argv_type argv_type; + + /// Default constructor, creates an uninitialised stream. + basic_ipstream() + : istream_type(NULL), pbase_type() + { } + + /** + * @brief Constructor that initialises the stream by starting a process. + * + * Initialises the stream buffer by calling do_open() with the supplied + * arguments. + * + * @param cmd a string containing a shell command. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, pmode) + */ + explicit + basic_ipstream(const std::string& cmd, pmode mode = pstdout) + : istream_type(NULL), pbase_type(cmd, readable(mode)) + { } + + /** + * @brief Constructor that initialises the stream by starting a process. + * + * Initialises the stream buffer by calling do_open() with the supplied + * arguments. + * + * @param file a string containing the pathname of a program to execute. + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + basic_ipstream( const std::string& file, + const argv_type& argv, + pmode mode = pstdout ) + : istream_type(NULL), pbase_type(file, argv, readable(mode)) + { } + + /** + * @brief Constructor that initialises the stream by starting a process. + * + * Initialises the stream buffer by calling + * @c do_open(argv[0],argv,mode|pstdout) + * + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + explicit + basic_ipstream(const argv_type& argv, pmode mode = pstdout) + : istream_type(NULL), pbase_type(argv.at(0), argv, readable(mode)) + { } + +#if __cplusplus >= 201103L + template + explicit + basic_ipstream(std::initializer_list args, pmode mode = pstdout) + : basic_ipstream(argv_type(args.begin(), args.end()), mode) + { } +#endif + + /** + * @brief Destructor. + * + * Closes the stream and waits for the child to exit. + */ + ~basic_ipstream() + { } + + /** + * @brief Start a process. + * + * Calls do_open( @a cmd , @a mode|pstdout ). + * + * @param cmd a string containing a shell command. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, pmode) + */ + void + open(const std::string& cmd, pmode mode = pstdout) + { + this->do_open(cmd, readable(mode)); + } + + /** + * @brief Start a process. + * + * Calls do_open( @a file , @a argv , @a mode|pstdout ). + * + * @param file a string containing the pathname of a program to execute. + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + void + open( const std::string& file, + const argv_type& argv, + pmode mode = pstdout ) + { + this->do_open(file, argv, readable(mode)); + } + + /** + * @brief Set streambuf to read from process' @c stdout. + * @return @c *this + */ + basic_ipstream& + out() + { + this->buf_.read_err(false); + return *this; + } + + /** + * @brief Set streambuf to read from process' @c stderr. + * @return @c *this + */ + basic_ipstream& + err() + { + this->buf_.read_err(true); + return *this; + } + }; + + + /** + * @class basic_opstream + * @brief Class template for Output PStreams. + * + * Writing to an open opstream writes to the standard input of the command; + * the command's standard output is the same as that of the process that + * created the pstream object, unless altered by the command itself. + */ + + template > + class basic_opstream + : public std::basic_ostream + , public pstream_common + , virtual public pstreams + { + typedef std::basic_ostream ostream_type; + typedef pstream_common pbase_type; + + using pbase_type::buf_; // declare name in this scope + + public: + /// Type used to specify how to connect to the process. + typedef typename pbase_type::pmode pmode; + + /// Type used to hold the arguments for a command. + typedef typename pbase_type::argv_type argv_type; + + /// Default constructor, creates an uninitialised stream. + basic_opstream() + : ostream_type(NULL), pbase_type() + { } + + /** + * @brief Constructor that initialises the stream by starting a process. + * + * Initialises the stream buffer by calling do_open() with the supplied + * arguments. + * + * @param cmd a string containing a shell command. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, pmode) + */ + explicit + basic_opstream(const std::string& cmd, pmode mode = pstdin) + : ostream_type(NULL), pbase_type(cmd, mode|pstdin) + { } + + /** + * @brief Constructor that initialises the stream by starting a process. + * + * Initialises the stream buffer by calling do_open() with the supplied + * arguments. + * + * @param file a string containing the pathname of a program to execute. + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + basic_opstream( const std::string& file, + const argv_type& argv, + pmode mode = pstdin ) + : ostream_type(NULL), pbase_type(file, argv, mode|pstdin) + { } + + /** + * @brief Constructor that initialises the stream by starting a process. + * + * Initialises the stream buffer by calling + * @c do_open(argv[0],argv,mode|pstdin) + * + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + explicit + basic_opstream(const argv_type& argv, pmode mode = pstdin) + : ostream_type(NULL), pbase_type(argv.at(0), argv, mode|pstdin) + { } + +#if __cplusplus >= 201103L + /** + * @brief Constructor that initialises the stream by starting a process. + * + * @param args a list of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + template + explicit + basic_opstream(std::initializer_list args, pmode mode = pstdin) + : basic_opstream(argv_type(args.begin(), args.end()), mode) + { } +#endif + + /** + * @brief Destructor + * + * Closes the stream and waits for the child to exit. + */ + ~basic_opstream() { } + + /** + * @brief Start a process. + * + * Calls do_open( @a cmd , @a mode|pstdin ). + * + * @param cmd a string containing a shell command. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, pmode) + */ + void + open(const std::string& cmd, pmode mode = pstdin) + { + this->do_open(cmd, mode|pstdin); + } + + /** + * @brief Start a process. + * + * Calls do_open( @a file , @a argv , @a mode|pstdin ). + * + * @param file a string containing the pathname of a program to execute. + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + void + open( const std::string& file, + const argv_type& argv, + pmode mode = pstdin) + { + this->do_open(file, argv, mode|pstdin); + } + }; + + + /** + * @class basic_pstream + * @brief Class template for Bidirectional PStreams. + * + * Writing to a pstream opened with @c pmode @c pstdin writes to the + * standard input of the command. + * Reading from a pstream opened with @c pmode @c pstdout and/or @c pstderr + * reads the command's standard output and/or standard error. + * Any of the process' @c stdin, @c stdout or @c stderr that is not + * connected to the pstream (as specified by the @c pmode) + * will be the same as the process that created the pstream object, + * unless altered by the command itself. + */ + template > + class basic_pstream + : public std::basic_iostream + , public pstream_common + , virtual public pstreams + { + typedef std::basic_iostream iostream_type; + typedef pstream_common pbase_type; + + using pbase_type::buf_; // declare name in this scope + + public: + /// Type used to specify how to connect to the process. + typedef typename pbase_type::pmode pmode; + + /// Type used to hold the arguments for a command. + typedef typename pbase_type::argv_type argv_type; + + /// Default constructor, creates an uninitialised stream. + basic_pstream() + : iostream_type(NULL), pbase_type() + { } + + /** + * @brief Constructor that initialises the stream by starting a process. + * + * Initialises the stream buffer by calling do_open() with the supplied + * arguments. + * + * @param cmd a string containing a shell command. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, pmode) + */ + explicit + basic_pstream(const std::string& cmd, pmode mode = pstdout|pstdin) + : iostream_type(NULL), pbase_type(cmd, mode) + { } + + /** + * @brief Constructor that initialises the stream by starting a process. + * + * Initialises the stream buffer by calling do_open() with the supplied + * arguments. + * + * @param file a string containing the pathname of a program to execute. + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + basic_pstream( const std::string& file, + const argv_type& argv, + pmode mode = pstdout|pstdin ) + : iostream_type(NULL), pbase_type(file, argv, mode) + { } + + /** + * @brief Constructor that initialises the stream by starting a process. + * + * Initialises the stream buffer by calling + * @c do_open(argv[0],argv,mode) + * + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + explicit + basic_pstream(const argv_type& argv, pmode mode = pstdout|pstdin) + : iostream_type(NULL), pbase_type(argv.at(0), argv, mode) + { } + +#if __cplusplus >= 201103L + /** + * @brief Constructor that initialises the stream by starting a process. + * + * @param l a list of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + template + explicit + basic_pstream(std::initializer_list l, pmode mode = pstdout|pstdin) + : basic_pstream(argv_type(l.begin(), l.end()), mode) + { } +#endif + + /** + * @brief Destructor + * + * Closes the stream and waits for the child to exit. + */ + ~basic_pstream() { } + + /** + * @brief Start a process. + * + * Calls do_open( @a cnd , @a mode ). + * + * @param cmd a string containing a shell command. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, pmode) + */ + void + open(const std::string& cmd, pmode mode = pstdout|pstdin) + { + this->do_open(cmd, mode); + } + + /** + * @brief Start a process. + * + * Calls do_open( @a file , @a argv , @a mode ). + * + * @param file a string containing the pathname of a program to execute. + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + void + open( const std::string& file, + const argv_type& argv, + pmode mode = pstdout|pstdin ) + { + this->do_open(file, argv, mode); + } + + /** + * @brief Set streambuf to read from process' @c stdout. + * @return @c *this + */ + basic_pstream& + out() + { + this->buf_.read_err(false); + return *this; + } + + /** + * @brief Set streambuf to read from process' @c stderr. + * @return @c *this + */ + basic_pstream& + err() + { + this->buf_.read_err(true); + return *this; + } + }; + + + /** + * @class basic_rpstream + * @brief Class template for Restricted PStreams. + * + * Writing to an rpstream opened with @c pmode @c pstdin writes to the + * standard input of the command. + * It is not possible to read directly from an rpstream object, to use + * an rpstream as in istream you must call either basic_rpstream::out() + * or basic_rpstream::err(). This is to prevent accidental reads from + * the wrong input source. If the rpstream was not opened with @c pmode + * @c pstderr then the class cannot read the process' @c stderr, and + * basic_rpstream::err() will return an istream that reads from the + * process' @c stdout, and vice versa. + * Reading from an rpstream opened with @c pmode @c pstdout and/or + * @c pstderr reads the command's standard output and/or standard error. + * Any of the process' @c stdin, @c stdout or @c stderr that is not + * connected to the pstream (as specified by the @c pmode) + * will be the same as the process that created the pstream object, + * unless altered by the command itself. + */ + + template > + class basic_rpstream + : public std::basic_ostream + , private std::basic_istream + , private pstream_common + , virtual public pstreams + { + typedef std::basic_ostream ostream_type; + typedef std::basic_istream istream_type; + typedef pstream_common pbase_type; + + using pbase_type::buf_; // declare name in this scope + + public: + /// Type used to specify how to connect to the process. + typedef typename pbase_type::pmode pmode; + + /// Type used to hold the arguments for a command. + typedef typename pbase_type::argv_type argv_type; + + /// Default constructor, creates an uninitialised stream. + basic_rpstream() + : ostream_type(NULL), istream_type(NULL), pbase_type() + { } + + /** + * @brief Constructor that initialises the stream by starting a process. + * + * Initialises the stream buffer by calling do_open() with the supplied + * arguments. + * + * @param cmd a string containing a shell command. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, pmode) + */ + explicit + basic_rpstream(const std::string& cmd, pmode mode = pstdout|pstdin) + : ostream_type(NULL) , istream_type(NULL) , pbase_type(cmd, mode) + { } + + /** + * @brief Constructor that initialises the stream by starting a process. + * + * Initialises the stream buffer by calling do_open() with the supplied + * arguments. + * + * @param file a string containing the pathname of a program to execute. + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + basic_rpstream( const std::string& file, + const argv_type& argv, + pmode mode = pstdout|pstdin ) + : ostream_type(NULL), istream_type(NULL), pbase_type(file, argv, mode) + { } + + /** + * @brief Constructor that initialises the stream by starting a process. + * + * Initialises the stream buffer by calling + * @c do_open(argv[0],argv,mode) + * + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + explicit + basic_rpstream(const argv_type& argv, pmode mode = pstdout|pstdin) + : ostream_type(NULL), istream_type(NULL), + pbase_type(argv.at(0), argv, mode) + { } + +#if __cplusplus >= 201103L + /** + * @brief Constructor that initialises the stream by starting a process. + * + * @param l a list of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + template + explicit + basic_rpstream(std::initializer_list l, pmode mode = pstdout|pstdin) + : basic_rpstream(argv_type(l.begin(), l.end()), mode) + { } +#endif + + /// Destructor + ~basic_rpstream() { } + + /** + * @brief Start a process. + * + * Calls do_open( @a cmd , @a mode ). + * + * @param cmd a string containing a shell command. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, pmode) + */ + void + open(const std::string& cmd, pmode mode = pstdout|pstdin) + { + this->do_open(cmd, mode); + } + + /** + * @brief Start a process. + * + * Calls do_open( @a file , @a argv , @a mode ). + * + * @param file a string containing the pathname of a program to execute. + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + void + open( const std::string& file, + const argv_type& argv, + pmode mode = pstdout|pstdin ) + { + this->do_open(file, argv, mode); + } + + /** + * @brief Obtain a reference to the istream that reads + * the process' @c stdout. + * @return @c *this + */ + istream_type& + out() + { + this->buf_.read_err(false); + return *this; + } + + /** + * @brief Obtain a reference to the istream that reads + * the process' @c stderr. + * @return @c *this + */ + istream_type& + err() + { + this->buf_.read_err(true); + return *this; + } + }; + + + /// Type definition for common template specialisation. + typedef basic_pstreambuf pstreambuf; + /// Type definition for common template specialisation. + typedef basic_ipstream ipstream; + /// Type definition for common template specialisation. + typedef basic_opstream opstream; + /// Type definition for common template specialisation. + typedef basic_pstream pstream; + /// Type definition for common template specialisation. + typedef basic_rpstream rpstream; + + + /** + * When inserted into an output pstream the manipulator calls + * basic_pstreambuf::peof() to close the output pipe, + * causing the child process to receive the end-of-file indicator + * on subsequent reads from its @c stdin stream. + * + * @brief Manipulator to close the pipe connected to the process' stdin. + * @param s An output PStream class. + * @return The stream object the manipulator was invoked on. + * @see basic_pstreambuf::peof() + * @relates basic_opstream basic_pstream basic_rpstream + */ + template + inline std::basic_ostream& + peof(std::basic_ostream& s) + { + typedef basic_pstreambuf pstreambuf_type; + if (pstreambuf_type* p = dynamic_cast(s.rdbuf())) + p->peof(); + return s; + } + + + /* + * member definitions for pstreambuf + */ + + + /** + * @class basic_pstreambuf + * Provides underlying streambuf functionality for the PStreams classes. + */ + + /** Creates an uninitialised stream buffer. */ + template + inline + basic_pstreambuf::basic_pstreambuf() + : ppid_(-1) // initialise to -1 to indicate no process run yet. + , wpipe_(-1) + , wbuffer_(NULL) + , rsrc_(rsrc_out) + , status_(-1) + , error_(0) + { + init_rbuffers(); + } + + /** + * Initialises the stream buffer by calling open() with the supplied + * arguments. + * + * @param cmd a string containing a shell command. + * @param mode the I/O mode to use when opening the pipe. + * @see open() + */ + template + inline + basic_pstreambuf::basic_pstreambuf(const std::string& cmd, pmode mode) + : ppid_(-1) // initialise to -1 to indicate no process run yet. + , wpipe_(-1) + , wbuffer_(NULL) + , rsrc_(rsrc_out) + , status_(-1) + , error_(0) + { + init_rbuffers(); + open(cmd, mode); + } + + /** + * Initialises the stream buffer by calling open() with the supplied + * arguments. + * + * @param file a string containing the name of a program to execute. + * @param argv a vector of argument strings passsed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see open() + */ + template + inline + basic_pstreambuf::basic_pstreambuf( const std::string& file, + const argv_type& argv, + pmode mode ) + : ppid_(-1) // initialise to -1 to indicate no process run yet. + , wpipe_(-1) + , wbuffer_(NULL) + , rsrc_(rsrc_out) + , status_(-1) + , error_(0) + { + init_rbuffers(); + open(file, argv, mode); + } + + /** + * Closes the stream by calling close(). + * @see close() + */ + template + inline + basic_pstreambuf::~basic_pstreambuf() + { + close(); + } + + /** + * Starts a new process by passing @a command to the shell (/bin/sh) + * and opens pipes to the process with the specified @a mode. + * + * If @a mode contains @c pstdout the initial read source will be + * the child process' stdout, otherwise if @a mode contains @c pstderr + * the initial read source will be the child's stderr. + * + * Will duplicate the actions of the shell in searching for an + * executable file if the specified file name does not contain a slash (/) + * character. + * + * @warning + * There is no way to tell whether the shell command succeeded, this + * function will always succeed unless resource limits (such as + * memory usage, or number of processes or open files) are exceeded. + * This means is_open() will return true even if @a command cannot + * be executed. + * Use pstreambuf::open(const std::string&, const argv_type&, pmode) + * if you need to know whether the command failed to execute. + * + * @param command a string containing a shell command. + * @param mode a bitwise OR of one or more of @c out, @c in, @c err. + * @return NULL if the shell could not be started or the + * pipes could not be opened, @c this otherwise. + * @see execl(3) + */ + template + basic_pstreambuf* + basic_pstreambuf::open(const std::string& command, pmode mode) + { + const char * shell_path = "/bin/sh"; +#if 0 + const std::string argv[] = { "sh", "-c", command }; + return this->open(shell_path, argv_type(argv, argv+3), mode); +#else + basic_pstreambuf* ret = NULL; + + if (!is_open()) + { + switch(fork(mode)) + { + case 0 : + // this is the new process, exec command + ::execl(shell_path, "sh", "-c", command.c_str(), (char*)NULL); + + // can only reach this point if exec() failed + + // parent can get exit code from waitpid() + ::_exit(errno); + // using std::exit() would make static dtors run twice + + case -1 : + // couldn't fork, error already handled in pstreambuf::fork() + break; + + default : + // this is the parent process + // activate buffers + create_buffers(mode); + ret = this; + } + } + return ret; +#endif + } + + /** + * @brief Helper function to close a file descriptor. + * + * Inspects @a fd and calls close(3) if it has a non-negative value. + * + * @param fd a file descriptor. + * @relates basic_pstreambuf + */ + inline void + close_fd(pstreams::fd_type& fd) + { + if (fd >= 0 && ::close(fd) == 0) + fd = -1; + } + + /** + * @brief Helper function to close an array of file descriptors. + * + * Calls @c close_fd() on each member of the array. + * The length of the array is determined automatically by + * template argument deduction to avoid errors. + * + * @param fds an array of file descriptors. + * @relates basic_pstreambuf + */ + template + inline void + close_fd_array(pstreams::fd_type (&fds)[N]) + { + for (std::size_t i = 0; i < N; ++i) + close_fd(fds[i]); + } + + /** + * Starts a new process by executing @a file with the arguments in + * @a argv and opens pipes to the process with the specified @a mode. + * + * By convention @c argv[0] should be the file name of the file being + * executed. + * + * If @a mode contains @c pstdout the initial read source will be + * the child process' stdout, otherwise if @a mode contains @c pstderr + * the initial read source will be the child's stderr. + * + * Will duplicate the actions of the shell in searching for an + * executable file if the specified file name does not contain a slash (/) + * character. + * + * Iff @a file is successfully executed then is_open() will return true. + * Otherwise, pstreambuf::error() can be used to obtain the value of + * @c errno that was set by execvp(3) in the child process. + * + * The exit status of the new process will be returned by + * pstreambuf::status() after pstreambuf::exited() returns true. + * + * @param file a string containing the pathname of a program to execute. + * @param argv a vector of argument strings passed to the new program. + * @param mode a bitwise OR of one or more of @c out, @c in and @c err. + * @return NULL if a pipe could not be opened or if the program could + * not be executed, @c this otherwise. + * @see execvp(3) + */ + template + basic_pstreambuf* + basic_pstreambuf::open( const std::string& file, + const argv_type& argv, + pmode mode ) + { + basic_pstreambuf* ret = NULL; + + if (!is_open()) + { + // constants for read/write ends of pipe + enum { RD, WR }; + + // open another pipe and set close-on-exec + fd_type ck_exec[] = { -1, -1 }; + if (-1 == ::pipe(ck_exec) + || -1 == ::fcntl(ck_exec[RD], F_SETFD, FD_CLOEXEC) + || -1 == ::fcntl(ck_exec[WR], F_SETFD, FD_CLOEXEC)) + { + error_ = errno; + close_fd_array(ck_exec); + } + else + { + switch(fork(mode)) + { + case 0 : + // this is the new process, exec command + { + char** arg_v = new char*[argv.size()+1]; + for (std::size_t i = 0; i < argv.size(); ++i) + { + const std::string& src = argv[i]; + char*& dest = arg_v[i]; + dest = new char[src.size()+1]; + dest[ src.copy(dest, src.size()) ] = '\0'; + } + arg_v[argv.size()] = NULL; + + ::execvp(file.c_str(), arg_v); + + // can only reach this point if exec() failed + + // parent can get error code from ck_exec pipe + error_ = errno; + + while (::write(ck_exec[WR], &error_, sizeof(error_)) == -1 + && errno == EINTR) + { } + + ::close(ck_exec[WR]); + ::close(ck_exec[RD]); + + ::_exit(error_); + // using std::exit() would make static dtors run twice + } + + case -1 : + // couldn't fork, error already handled in pstreambuf::fork() + close_fd_array(ck_exec); + break; + + default : + // this is the parent process + + // check child called exec() successfully + ::close(ck_exec[WR]); + switch (::read(ck_exec[RD], &error_, sizeof(error_))) + { + case 0: + // activate buffers + create_buffers(mode); + ret = this; + break; + case -1: + error_ = errno; + break; + default: + // error_ contains error code from child + // call wait() to clean up and set ppid_ to 0 + this->wait(); + break; + } + ::close(ck_exec[RD]); + } + } + } + return ret; + } + + /** + * Creates pipes as specified by @a mode and calls @c fork() to create + * a new process. If the fork is successful the parent process stores + * the child's PID and the opened pipes and the child process replaces + * its standard streams with the opened pipes. + * + * If an error occurs the error code will be set to one of the possible + * errors for @c pipe() or @c fork(). + * See your system's documentation for these error codes. + * + * @param mode an OR of pmodes specifying which of the child's + * standard streams to connect to. + * @return On success the PID of the child is returned in the parent's + * context and zero is returned in the child's context. + * On error -1 is returned and the error code is set appropriately. + */ + template + pid_t + basic_pstreambuf::fork(pmode mode) + { + pid_t pid = -1; + + // Three pairs of file descriptors, for pipes connected to the + // process' stdin, stdout and stderr + // (stored in a single array so close_fd_array() can close all at once) + fd_type fd[] = { -1, -1, -1, -1, -1, -1 }; + fd_type* const pin = fd; + fd_type* const pout = fd+2; + fd_type* const perr = fd+4; + + // constants for read/write ends of pipe + enum { RD, WR }; + + // N.B. + // For the pstreambuf pin is an output stream and + // pout and perr are input streams. + + if (!error_ && mode&pstdin && ::pipe(pin)) + error_ = errno; + + if (!error_ && mode&pstdout && ::pipe(pout)) + error_ = errno; + + if (!error_ && mode&pstderr && ::pipe(perr)) + error_ = errno; + + if (!error_) + { + pid = ::fork(); + switch (pid) + { + case 0 : + { + // this is the new process + + // for each open pipe close one end and redirect the + // respective standard stream to the other end + + if (*pin >= 0) + { + ::close(pin[WR]); + ::dup2(pin[RD], STDIN_FILENO); + ::close(pin[RD]); + } + if (*pout >= 0) + { + ::close(pout[RD]); + ::dup2(pout[WR], STDOUT_FILENO); + ::close(pout[WR]); + } + if (*perr >= 0) + { + ::close(perr[RD]); + ::dup2(perr[WR], STDERR_FILENO); + ::close(perr[WR]); + } + +#ifdef _POSIX_JOB_CONTROL + if (mode&newpg) + ::setpgid(0, 0); // Change to a new process group +#endif + + break; + } + case -1 : + { + // couldn't fork for some reason + error_ = errno; + // close any open pipes + close_fd_array(fd); + break; + } + default : + { + // this is the parent process, store process' pid + ppid_ = pid; + + // store one end of open pipes and close other end + if (*pin >= 0) + { + wpipe_ = pin[WR]; + ::close(pin[RD]); + } + if (*pout >= 0) + { + rpipe_[rsrc_out] = pout[RD]; + ::close(pout[WR]); + } + if (*perr >= 0) + { + rpipe_[rsrc_err] = perr[RD]; + ::close(perr[WR]); + } + } + } + } + else + { + // close any pipes we opened before failure + close_fd_array(fd); + } + return pid; + } + + /** + * Closes all pipes and calls wait() to wait for the process to finish. + * If an error occurs the error code will be set to one of the possible + * errors for @c waitpid(). + * See your system's documentation for these errors. + * + * @return @c this on successful close or @c NULL if there is no + * process to close or if an error occurs. + */ + template + basic_pstreambuf* + basic_pstreambuf::close() + { + const bool running = is_open(); + + sync(); // this might call wait() and reap the child process + + // rather than trying to work out whether or not we need to clean up + // just do it anyway, all cleanup functions are safe to call twice. + + destroy_buffers(pstdin|pstdout|pstderr); + + // close pipes before wait() so child gets EOF/SIGPIPE + close_fd(wpipe_); + close_fd_array(rpipe_); + + do + { + error_ = 0; + } while (wait() == -1 && error() == EINTR); + + return running ? this : NULL; + } + + /** + * Called on construction to initialise the arrays used for reading. + */ + template + inline void + basic_pstreambuf::init_rbuffers() + { + rpipe_[rsrc_out] = rpipe_[rsrc_err] = -1; + rbuffer_[rsrc_out] = rbuffer_[rsrc_err] = NULL; + rbufstate_[0] = rbufstate_[1] = rbufstate_[2] = NULL; + } + + template + void + basic_pstreambuf::create_buffers(pmode mode) + { + if (mode & pstdin) + { + delete[] wbuffer_; + wbuffer_ = new char_type[bufsz]; + this->setp(wbuffer_, wbuffer_ + bufsz); + } + if (mode & pstdout) + { + delete[] rbuffer_[rsrc_out]; + rbuffer_[rsrc_out] = new char_type[bufsz]; + rsrc_ = rsrc_out; + this->setg(rbuffer_[rsrc_out] + pbsz, rbuffer_[rsrc_out] + pbsz, + rbuffer_[rsrc_out] + pbsz); + } + if (mode & pstderr) + { + delete[] rbuffer_[rsrc_err]; + rbuffer_[rsrc_err] = new char_type[bufsz]; + if (!(mode & pstdout)) + { + rsrc_ = rsrc_err; + this->setg(rbuffer_[rsrc_err] + pbsz, rbuffer_[rsrc_err] + pbsz, + rbuffer_[rsrc_err] + pbsz); + } + } + } + + template + void + basic_pstreambuf::destroy_buffers(pmode mode) + { + if (mode & pstdin) + { + this->setp(NULL, NULL); + delete[] wbuffer_; + wbuffer_ = NULL; + } + if (mode & pstdout) + { + if (rsrc_ == rsrc_out) + this->setg(NULL, NULL, NULL); + delete[] rbuffer_[rsrc_out]; + rbuffer_[rsrc_out] = NULL; + } + if (mode & pstderr) + { + if (rsrc_ == rsrc_err) + this->setg(NULL, NULL, NULL); + delete[] rbuffer_[rsrc_err]; + rbuffer_[rsrc_err] = NULL; + } + } + + template + typename basic_pstreambuf::buf_read_src + basic_pstreambuf::switch_read_buffer(buf_read_src src) + { + if (rsrc_ != src) + { + char_type* tmpbufstate[] = {this->eback(), this->gptr(), this->egptr()}; + this->setg(rbufstate_[0], rbufstate_[1], rbufstate_[2]); + for (std::size_t i = 0; i < 3; ++i) + rbufstate_[i] = tmpbufstate[i]; + rsrc_ = src; + } + return rsrc_; + } + + /** + * Suspends execution and waits for the associated process to exit, or + * until a signal is delivered whose action is to terminate the current + * process or to call a signal handling function. If the process has + * already exited (i.e. it is a "zombie" process) then wait() returns + * immediately. Waiting for the child process causes all its system + * resources to be freed. + * + * error() will return EINTR if wait() is interrupted by a signal. + * + * @param nohang true to return immediately if the process has not exited. + * @return 1 if the process has exited and wait() has not yet been called. + * 0 if @a nohang is true and the process has not exited yet. + * -1 if no process has been started or if an error occurs, + * in which case the error can be found using error(). + */ + template + int + basic_pstreambuf::wait(bool nohang) + { + int child_exited = -1; + if (is_open()) + { + int exit_status; + switch(::waitpid(ppid_, &exit_status, nohang ? WNOHANG : 0)) + { + case 0 : + // nohang was true and process has not exited + child_exited = 0; + break; + case -1 : + error_ = errno; + break; + default : + // process has exited + ppid_ = 0; + status_ = exit_status; + child_exited = 1; + // Close wpipe, would get SIGPIPE if we used it. + destroy_buffers(pstdin); + close_fd(wpipe_); + // Must free read buffers and pipes on destruction + // or next call to open()/close() + break; + } + } + return child_exited; + } + + /** + * Sends the specified signal to the process. A signal can be used to + * terminate a child process that would not exit otherwise. + * + * If an error occurs the error code will be set to one of the possible + * errors for @c kill(). See your system's documentation for these errors. + * + * @param signal A signal to send to the child process. + * @return @c this or @c NULL if @c kill() fails. + */ + template + inline basic_pstreambuf* + basic_pstreambuf::kill(int signal) + { + basic_pstreambuf* ret = NULL; + if (is_open()) + { + if (::kill(ppid_, signal)) + error_ = errno; + else + { +#if 0 + // TODO call exited() to check for exit and clean up? leave to user? + if (signal==SIGTERM || signal==SIGKILL) + this->exited(); +#endif + ret = this; + } + } + return ret; + } + + /** + * Sends the specified signal to the process group of the child process. + * A signal can be used to terminate a child process that would not exit + * otherwise, or to kill the process and its own children. + * + * If an error occurs the error code will be set to one of the possible + * errors for @c getpgid() or @c kill(). See your system's documentation + * for these errors. If the child is in the current process group then + * NULL will be returned and the error code set to EPERM. + * + * @param signal A signal to send to the child process. + * @return @c this on success or @c NULL on failure. + */ + template + inline basic_pstreambuf* + basic_pstreambuf::killpg(int signal) + { + basic_pstreambuf* ret = NULL; +#ifdef _POSIX_JOB_CONTROL + if (is_open()) + { + pid_t pgid = ::getpgid(ppid_); + if (pgid == -1) + error_ = errno; + else if (pgid == ::getpgrp()) + error_ = EPERM; // Don't commit suicide + else if (::killpg(pgid, signal)) + error_ = errno; + else + ret = this; + } +#else + error_ = ENOTSUP; +#endif + return ret; + } + + /** + * This function can call pstreambuf::wait() and so may change the + * object's state if the child process has already exited. + * + * @return True if the associated process has exited, false otherwise. + * @see basic_pstreambuf::wait() + */ + template + inline bool + basic_pstreambuf::exited() + { + return ppid_ == 0 || wait(true)==1; + } + + + /** + * @return The exit status of the child process, or -1 if wait() + * has not yet been called to wait for the child to exit. + * @see basic_pstreambuf::wait() + */ + template + inline int + basic_pstreambuf::status() const + { + return status_; + } + + /** + * @return The error code of the most recently failed operation, or zero. + */ + template + inline int + basic_pstreambuf::error() const + { + return error_; + } + + /** + * Closes the output pipe, causing the child process to receive the + * end-of-file indicator on subsequent reads from its @c stdin stream. + */ + template + inline void + basic_pstreambuf::peof() + { + sync(); + destroy_buffers(pstdin); + close_fd(wpipe_); + } + + /** + * Unlike pstreambuf::exited(), this function will not call wait() and + * so will not change the object's state. This means that once a child + * process is executed successfully this function will continue to + * return true even after the process exits (until wait() is called.) + * + * @return true if a previous call to open() succeeded and wait() has + * not been called and determined that the process has exited, + * false otherwise. + */ + template + inline bool + basic_pstreambuf::is_open() const + { + return ppid_ > 0; + } + + /** + * Toggle the stream used for reading. If @a readerr is @c true then the + * process' @c stderr output will be used for subsequent extractions, if + * @a readerr is false the the process' stdout will be used. + * @param readerr @c true to read @c stderr, @c false to read @c stdout. + * @return @c true if the requested stream is open and will be used for + * subsequent extractions, @c false otherwise. + */ + template + inline bool + basic_pstreambuf::read_err(bool readerr) + { + buf_read_src src = readerr ? rsrc_err : rsrc_out; + if (rpipe_[src]>=0) + { + switch_read_buffer(src); + return true; + } + return false; + } + + /** + * Called when the internal character buffer is not present or is full, + * to transfer the buffer contents to the pipe. + * + * @param c a character to be written to the pipe. + * @return @c traits_type::eof() if an error occurs, otherwise if @a c + * is not equal to @c traits_type::eof() it will be buffered and + * a value other than @c traits_type::eof() returned to indicate + * success. + */ + template + typename basic_pstreambuf::int_type + basic_pstreambuf::overflow(int_type c) + { + if (!empty_buffer()) + return traits_type::eof(); + else if (!traits_type::eq_int_type(c, traits_type::eof())) + return this->sputc(c); + else + return traits_type::not_eof(c); + } + + + template + int + basic_pstreambuf::sync() + { + return !exited() && empty_buffer() ? 0 : -1; + } + + /** + * @param s character buffer. + * @param n buffer length. + * @return the number of characters written. + */ + template + std::streamsize + basic_pstreambuf::xsputn(const char_type* s, std::streamsize n) + { + std::streamsize done = 0; + while (done < n) + { + if (std::streamsize nbuf = this->epptr() - this->pptr()) + { + nbuf = std::min(nbuf, n - done); + traits_type::copy(this->pptr(), s + done, nbuf); + this->pbump(nbuf); + done += nbuf; + } + else if (!empty_buffer()) + break; + } + return done; + } + + /** + * @return true if the buffer was emptied, false otherwise. + */ + template + bool + basic_pstreambuf::empty_buffer() + { + const std::streamsize count = this->pptr() - this->pbase(); + if (count > 0) + { + const std::streamsize written = this->write(this->wbuffer_, count); + if (written > 0) + { + if (const std::streamsize unwritten = count - written) + traits_type::move(this->pbase(), this->pbase()+written, unwritten); + this->pbump(-written); + return true; + } + } + return false; + } + + /** + * Called when the internal character buffer is is empty, to re-fill it + * from the pipe. + * + * @return The first available character in the buffer, + * or @c traits_type::eof() in case of failure. + */ + template + typename basic_pstreambuf::int_type + basic_pstreambuf::underflow() + { + if (this->gptr() < this->egptr() || fill_buffer()) + return traits_type::to_int_type(*this->gptr()); + else + return traits_type::eof(); + } + + /** + * Attempts to make @a c available as the next character to be read by + * @c sgetc(). + * + * @param c a character to make available for extraction. + * @return @a c if the character can be made available, + * @c traits_type::eof() otherwise. + */ + template + typename basic_pstreambuf::int_type + basic_pstreambuf::pbackfail(int_type c) + { + if (this->gptr() != this->eback()) + { + this->gbump(-1); + if (!traits_type::eq_int_type(c, traits_type::eof())) + *this->gptr() = traits_type::to_char_type(c); + return traits_type::not_eof(c); + } + else + return traits_type::eof(); + } + + template + std::streamsize + basic_pstreambuf::showmanyc() + { + int avail = 0; + if (sizeof(char_type) == 1) + avail = fill_buffer(true) ? this->egptr() - this->gptr() : -1; +#ifdef FIONREAD + else + { + if (::ioctl(rpipe(), FIONREAD, &avail) == -1) + avail = -1; + else if (avail) + avail /= sizeof(char_type); + } +#endif + return std::streamsize(avail); + } + + /** + * @return true if the buffer was filled, false otherwise. + */ + template + bool + basic_pstreambuf::fill_buffer(bool non_blocking) + { + const std::streamsize pb1 = this->gptr() - this->eback(); + const std::streamsize pb2 = pbsz; + const std::streamsize npb = std::min(pb1, pb2); + + char_type* const rbuf = rbuffer(); + + if (npb) + traits_type::move(rbuf + pbsz - npb, this->gptr() - npb, npb); + + std::streamsize rc = -1; + + if (non_blocking) + { + const int flags = ::fcntl(rpipe(), F_GETFL); + if (flags != -1) + { + const bool blocking = !(flags & O_NONBLOCK); + if (blocking) + ::fcntl(rpipe(), F_SETFL, flags | O_NONBLOCK); // set non-blocking + + error_ = 0; + rc = read(rbuf + pbsz, bufsz - pbsz); + + if (rc == -1 && error_ == EAGAIN) // nothing available + rc = 0; + else if (rc == 0) // EOF + rc = -1; + + if (blocking) + ::fcntl(rpipe(), F_SETFL, flags); // restore + } + } + else + rc = read(rbuf + pbsz, bufsz - pbsz); + + if (rc > 0 || (rc == 0 && non_blocking)) + { + this->setg( rbuf + pbsz - npb, + rbuf + pbsz, + rbuf + pbsz + rc ); + return true; + } + else + { + this->setg(NULL, NULL, NULL); + return false; + } + } + + /** + * Writes up to @a n characters to the pipe from the buffer @a s. + * + * @param s character buffer. + * @param n buffer length. + * @return the number of characters written. + */ + template + inline std::streamsize + basic_pstreambuf::write(const char_type* s, std::streamsize n) + { + std::streamsize nwritten = 0; + if (wpipe() >= 0) + { + nwritten = ::write(wpipe(), s, n * sizeof(char_type)); + if (nwritten == -1) + error_ = errno; + else + nwritten /= sizeof(char_type); + } + return nwritten; + } + + /** + * Reads up to @a n characters from the pipe to the buffer @a s. + * + * @param s character buffer. + * @param n buffer length. + * @return the number of characters read. + */ + template + inline std::streamsize + basic_pstreambuf::read(char_type* s, std::streamsize n) + { + std::streamsize nread = 0; + if (rpipe() >= 0) + { + nread = ::read(rpipe(), s, n * sizeof(char_type)); + if (nread == -1) + error_ = errno; + else + nread /= sizeof(char_type); + } + return nread; + } + + /** @return a reference to the output file descriptor */ + template + inline pstreams::fd_type& + basic_pstreambuf::wpipe() + { + return wpipe_; + } + + /** @return a reference to the active input file descriptor */ + template + inline pstreams::fd_type& + basic_pstreambuf::rpipe() + { + return rpipe_[rsrc_]; + } + + /** @return a reference to the specified input file descriptor */ + template + inline pstreams::fd_type& + basic_pstreambuf::rpipe(buf_read_src which) + { + return rpipe_[which]; + } + + /** @return a pointer to the start of the active input buffer area. */ + template + inline typename basic_pstreambuf::char_type* + basic_pstreambuf::rbuffer() + { + return rbuffer_[rsrc_]; + } + + + /* + * member definitions for pstream_common + */ + + /** + * @class pstream_common + * Abstract Base Class providing common functionality for basic_ipstream, + * basic_opstream and basic_pstream. + * pstream_common manages the basic_pstreambuf stream buffer that is used + * by the derived classes to initialise an iostream class. + */ + + /** Creates an uninitialised stream. */ + template + inline + pstream_common::pstream_common() + : std::basic_ios(NULL) + , command_() + , buf_() + { + this->std::basic_ios::rdbuf(&buf_); + } + + /** + * Initialises the stream buffer by calling + * do_open( @a command , @a mode ) + * + * @param cmd a string containing a shell command. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, pmode) + */ + template + inline + pstream_common::pstream_common(const std::string& cmd, pmode mode) + : std::basic_ios(NULL) + , command_(cmd) + , buf_() + { + this->std::basic_ios::rdbuf(&buf_); + do_open(cmd, mode); + } + + /** + * Initialises the stream buffer by calling + * do_open( @a file , @a argv , @a mode ) + * + * @param file a string containing the pathname of a program to execute. + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + template + inline + pstream_common::pstream_common( const std::string& file, + const argv_type& argv, + pmode mode ) + : std::basic_ios(NULL) + , command_(file) + , buf_() + { + this->std::basic_ios::rdbuf(&buf_); + do_open(file, argv, mode); + } + + /** + * This is a pure virtual function to make @c pstream_common abstract. + * Because it is the destructor it will be called by derived classes + * and so must be defined. It is also protected, to discourage use of + * the PStreams classes through pointers or references to the base class. + * + * @sa If defining a pure virtual seems odd you should read + * http://www.gotw.ca/gotw/031.htm (and the rest of the site as well!) + */ + template + inline + pstream_common::~pstream_common() + { + } + + /** + * Calls rdbuf()->open( @a command , @a mode ) + * and sets @c failbit on error. + * + * @param cmd a string containing a shell command. + * @param mode the I/O mode to use when opening the pipe. + * @see basic_pstreambuf::open(const std::string&, pmode) + */ + template + inline void + pstream_common::do_open(const std::string& cmd, pmode mode) + { + if (!buf_.open((command_=cmd), mode)) + this->setstate(std::ios_base::failbit); + } + + /** + * Calls rdbuf()->open( @a file, @a argv, @a mode ) + * and sets @c failbit on error. + * + * @param file a string containing the pathname of a program to execute. + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see basic_pstreambuf::open(const std::string&, const argv_type&, pmode) + */ + template + inline void + pstream_common::do_open( const std::string& file, + const argv_type& argv, + pmode mode ) + { + if (!buf_.open((command_=file), argv, mode)) + this->setstate(std::ios_base::failbit); + } + + /** Calls rdbuf->close() and sets @c failbit on error. */ + template + inline void + pstream_common::close() + { + if (!buf_.close()) + this->setstate(std::ios_base::failbit); + } + + /** + * @return rdbuf()->is_open(). + * @see basic_pstreambuf::is_open() + */ + template + inline bool + pstream_common::is_open() const + { + return buf_.is_open(); + } + + /** @return a string containing the command used to initialise the stream. */ + template + inline const std::string& + pstream_common::command() const + { + return command_; + } + + /** @return a pointer to the private stream buffer member. */ + // TODO document behaviour if buffer replaced. + template + inline typename pstream_common::streambuf_type* + pstream_common::rdbuf() const + { + return const_cast(&buf_); + } + + +#if REDI_EVISCERATE_PSTREAMS + /** + * @def REDI_EVISCERATE_PSTREAMS + * If this macro has a non-zero value then certain internals of the + * @c basic_pstreambuf template class are exposed. In general this is + * a Bad Thing, as the internal implementation is largely undocumented + * and may be subject to change at any time, so this feature is only + * provided because it might make PStreams useful in situations where + * it is necessary to do Bad Things. + */ + + /** + * @warning This function exposes the internals of the stream buffer and + * should be used with caution. It is the caller's responsibility + * to flush streams etc. in order to clear any buffered data. + * The POSIX.1 function fdopen(3) is used to obtain the + * @c FILE pointers from the streambuf's private file descriptor + * members so consult your system's documentation for + * fdopen(3). + * + * @param in A FILE* that will refer to the process' stdin. + * @param out A FILE* that will refer to the process' stdout. + * @param err A FILE* that will refer to the process' stderr. + * @return An OR of zero or more of @c pstdin, @c pstdout, @c pstderr. + * + * For each open stream shared with the child process a @c FILE* is + * obtained and assigned to the corresponding parameter. For closed + * streams @c NULL is assigned to the parameter. + * The return value can be tested to see which parameters should be + * @c !NULL by masking with the corresponding @c pmode value. + * + * @see fdopen(3) + */ + template + std::size_t + basic_pstreambuf::fopen(FILE*& in, FILE*& out, FILE*& err) + { + in = out = err = NULL; + std::size_t open_files = 0; + if (wpipe() > -1) + { + if ((in = ::fdopen(wpipe(), "w"))) + { + open_files |= pstdin; + } + } + if (rpipe(rsrc_out) > -1) + { + if ((out = ::fdopen(rpipe(rsrc_out), "r"))) + { + open_files |= pstdout; + } + } + if (rpipe(rsrc_err) > -1) + { + if ((err = ::fdopen(rpipe(rsrc_err), "r"))) + { + open_files |= pstderr; + } + } + return open_files; + } + + /** + * @warning This function exposes the internals of the stream buffer and + * should be used with caution. + * + * @param in A FILE* that will refer to the process' stdin. + * @param out A FILE* that will refer to the process' stdout. + * @param err A FILE* that will refer to the process' stderr. + * @return A bitwise-or of zero or more of @c pstdin, @c pstdout, @c pstderr. + * @see basic_pstreambuf::fopen() + */ + template + inline std::size_t + pstream_common::fopen(FILE*& fin, FILE*& fout, FILE*& ferr) + { + return buf_.fopen(fin, fout, ferr); + } + +#endif // REDI_EVISCERATE_PSTREAMS + + +} // namespace redi + +/** + * @mainpage PStreams Reference + * @htmlinclude mainpage.html + */ + +#endif // REDI_PSTREAM_H_SEEN + +// vim: ts=2 sw=2 expandtab + diff --git a/bin/hipcc b/bin/hipcc index b991bd40aa..6565eaacb8 100755 --- a/bin/hipcc +++ b/bin/hipcc @@ -93,8 +93,6 @@ if ($HIP_PLATFORM eq "hcc") { $HIP_ATP_MARKER=$ENV{'HIP_ATP_MARKER'} // 1; $marker_path = "$ROCM_PATH/profiler/CXLActivityLogger"; - $ROCM_TARGET=$ENV{'ROCM_TARGET'} // "gfx803"; - # HCC* may be used to compile src/hip_hcc.o (and also feed the HIPCXXFLAGS below) $HCC = "$HCC_HOME/bin/hcc"; $HCCFLAGS = "-hc -D__HIPCC__ -I$HCC_HOME/include "; @@ -128,7 +126,7 @@ if ($HIP_PLATFORM eq "hcc") { # Force -stdlib=libc++ on UB14.04 $HOST_OSVER= `cat /etc/os-release | grep "^VERSION_ID\=" | cut -d= -f2 | tr -d '\n'`; if (($HOST_OSNAME eq "ubuntu" and $HOST_OSVER eq "\"14.04\"") - or ($HOST_OSNAME eq "\"centos\"" and $HOST_OSVER eq "\"7\"") + or ($HOST_OSNAME eq "\"centos\"" and $HOST_OSVER eq "\"7\"") or ($HOST_OSNAME eq "\"rhel\"" and $HOST_OSVER eq "\"7.4\"")) { $HIPCXXFLAGS .= " -stdlib=libc++"; $setStdLib = 1; @@ -202,7 +200,7 @@ if ($verbose & 0x4) { # Handle code object generation my $ISACMD=""; if($HIP_PLATFORM eq "hcc"){ - $ISACMD .= "set ROCM_PATH=$ROCM_PATH && set ROCM_TARGET=$ROCM_TARGET && $HIP_PATH/bin/hccgenco.sh "; + $ISACMD .= "$HIP_PATH/bin/lpl "; if($ARGV[0] eq "--genco"){ foreach $isaarg (@ARGV[1..$#ARGV]){ $ISACMD .= " "; diff --git a/include/hip/hcc_detail/code_object_bundle.hpp b/include/hip/hcc_detail/code_object_bundle.hpp index 72f9d35c73..81ecb16409 100644 --- a/include/hip/hcc_detail/code_object_bundle.hpp +++ b/include/hip/hcc_detail/code_object_bundle.hpp @@ -37,17 +37,16 @@ namespace hip_impl hsa_isa_t triple_to_hsa_isa(const std::string& triple); struct Bundled_code { - union { + union Header { struct { std::uint64_t offset; std::uint64_t bundle_sz; std::uint64_t triple_sz; }; - std::uint8_t cbuf[ - sizeof(offset) + sizeof(bundle_sz) + sizeof(triple_sz)]; - }; + char cbuf[sizeof(offset) + sizeof(bundle_sz) + sizeof(triple_sz)]; + } header; std::string triple; - std::vector blob; + std::vector blob; }; class Bundled_code_header { @@ -57,14 +56,13 @@ namespace hip_impl static constexpr auto magic_string_sz_ = sizeof(magic_string_) - 1; // DATA - union { + union Header_ { struct { - std::uint8_t bundler_magic_string_[magic_string_sz_]; + char bundler_magic_string_[magic_string_sz_]; std::uint64_t bundle_cnt_; }; - std::uint8_t cbuf_[ - sizeof(bundler_magic_string_) + sizeof(bundle_cnt_)]; - }; + char cbuf_[sizeof(bundler_magic_string_) + sizeof(bundle_cnt_)]; + } header_; std::vector bundles_; // FRIENDS - MANIPULATORS @@ -78,22 +76,24 @@ namespace hip_impl { if (f == l) return false; - std::copy_n(f, sizeof(x.cbuf_), x.cbuf_); + std::copy_n(f, sizeof(x.header_.cbuf_), x.header_.cbuf_); if (valid(x)) { - x.bundles_.resize(x.bundle_cnt_); + x.bundles_.resize(x.header_.bundle_cnt_); - auto it = f + sizeof(x.cbuf_); + auto it = f + sizeof(x.header_.cbuf_); for (auto&& y : x.bundles_) { - std::copy_n(it, sizeof(y.cbuf), y.cbuf); - it += sizeof(y.cbuf); + std::copy_n(it, sizeof(y.header.cbuf), y.header.cbuf); + it += sizeof(y.header.cbuf); - y.triple.insert(y.triple.cend(), it, it + y.triple_sz); + y.triple.assign(it, it + y.header.triple_sz); std::copy_n( - f + y.offset, y.bundle_sz, std::back_inserter(y.blob)); + f + y.header.offset, + y.header.bundle_sz, + std::back_inserter(y.blob)); - it += y.triple_sz; + it += y.header.triple_sz; } return true; @@ -103,7 +103,7 @@ namespace hip_impl } friend inline - bool read(const std::vector& blob, Bundled_code_header& x) + bool read(const std::vector& blob, Bundled_code_header& x) { return read(blob.cbegin(), blob.cend(), x); } @@ -111,7 +111,7 @@ namespace hip_impl inline bool read(std::istream& is, Bundled_code_header& x) { - return read(std::vector{ + return read(std::vector{ std::istreambuf_iterator{is}, std::istreambuf_iterator{}}, x); @@ -123,9 +123,9 @@ namespace hip_impl bool valid(const Bundled_code_header& x) { return std::equal( - x.bundler_magic_string_, - x.bundler_magic_string_ + magic_string_sz_, - x.magic_string_); + magic_string_, + magic_string_ + magic_string_sz_, + x.header_.bundler_magic_string_); } friend inline @@ -139,7 +139,9 @@ namespace hip_impl template Bundled_code_header(RandomAccessIterator f, RandomAccessIterator l); explicit - Bundled_code_header(const std::vector& blob); + Bundled_code_header(const std::vector& blob); + explicit + Bundled_code_header(const void* maybe_blob); Bundled_code_header(const Bundled_code_header&) = default; Bundled_code_header(Bundled_code_header&&) = default; ~Bundled_code_header() = default; diff --git a/samples/0_Intro/module_api/launchKernelHcc.cpp b/samples/0_Intro/module_api/launchKernelHcc.cpp index 7c90198b6a..eb10f64876 100644 --- a/samples/0_Intro/module_api/launchKernelHcc.cpp +++ b/samples/0_Intro/module_api/launchKernelHcc.cpp @@ -35,7 +35,7 @@ THE SOFTWARE. #define LEN 64 #define SIZE LEN<<2 -#define fileName "vcpy_kernel.code" +#define fileName "vcpy_kernel.code.adipose" #define kernel_name "hello_world" #define HIP_CHECK(status) \ @@ -72,7 +72,6 @@ int main(){ uint32_t one = 1; struct { - uint32_t _hidden[6]; void * _Ad; void * _Bd; } args; diff --git a/samples/0_Intro/module_api/runKernel.cpp b/samples/0_Intro/module_api/runKernel.cpp index fb34f80b7b..84dbc395a1 100644 --- a/samples/0_Intro/module_api/runKernel.cpp +++ b/samples/0_Intro/module_api/runKernel.cpp @@ -30,7 +30,7 @@ THE SOFTWARE. #define LEN 64 #define SIZE LEN<<2 -#define fileName "vcpy_kernel.code" +#define fileName "vcpy_kernel.code.adipose" #define kernel_name "hello_world" #define HIP_CHECK(status) \ @@ -68,7 +68,6 @@ int main(){ uint32_t one = 1; struct { - uint32_t _hidden[6]; void * _Ad; void * _Bd; } args; diff --git a/samples/0_Intro/module_api/vcpy_kernel.cpp b/samples/0_Intro/module_api/vcpy_kernel.cpp index b728d67c93..a65d3a2e1e 100644 --- a/samples/0_Intro/module_api/vcpy_kernel.cpp +++ b/samples/0_Intro/module_api/vcpy_kernel.cpp @@ -22,7 +22,7 @@ THE SOFTWARE. #include "hip/hip_runtime.h" -extern "C" __global__ void hello_world(hipLaunchParm lp, float *a, float *b) +extern "C" __global__ void hello_world(float *a, float *b) { int tx = hipThreadIdx_x; b[tx] = a[tx]; diff --git a/samples/0_Intro/module_api_global/runKernel.cpp b/samples/0_Intro/module_api_global/runKernel.cpp index 3f84720edf..74f4fb84e2 100644 --- a/samples/0_Intro/module_api_global/runKernel.cpp +++ b/samples/0_Intro/module_api_global/runKernel.cpp @@ -30,7 +30,7 @@ THE SOFTWARE. #define LEN 64 #define SIZE LEN*sizeof(float) -#define fileName "vcpy_kernel.code" +#define fileName "vcpy_kernel.code.adipose" float myDeviceGlobal; float myDeviceGlobalArray[16]; #define HIP_CHECK(cmd) \ @@ -80,7 +80,6 @@ int main(){ uint32_t one = 1; struct { - uint32_t _hidden[6]; // genco path + wrapper-gen pass use hidden arguments. void * _Ad; void * _Bd; } args; @@ -111,7 +110,7 @@ int main(){ HIP_LAUNCH_PARAM_END }; - { + { hipFunction_t Function; HIP_CHECK(hipModuleGetFunction(&Function, Module, "hello_world")); HIP_CHECK(hipModuleLaunchKernel(Function, 1, 1, 1, LEN, 1, 1, 0, 0, NULL, (void**)&config)); @@ -136,7 +135,7 @@ int main(){ }; } - { + { hipFunction_t Function; HIP_CHECK(hipModuleGetFunction(&Function, Module, "test_globals")); HIP_CHECK(hipModuleLaunchKernel(Function, 1, 1, 1, LEN, 1, 1, 0, 0, NULL, (void**)&config)); diff --git a/samples/0_Intro/module_api_global/vcpy_kernel.cpp b/samples/0_Intro/module_api_global/vcpy_kernel.cpp index f24a261585..8bf67b0fa0 100644 --- a/samples/0_Intro/module_api_global/vcpy_kernel.cpp +++ b/samples/0_Intro/module_api_global/vcpy_kernel.cpp @@ -27,13 +27,13 @@ THE SOFTWARE. extern float myDeviceGlobal; extern float myDeviceGlobalArray[16];; -extern "C" __global__ void hello_world(hipLaunchParm lp, const float *a, float *b) +extern "C" __global__ void hello_world(const float *a, float *b) { int tx = hipThreadIdx_x; b[tx] = a[tx]; } -extern "C" __global__ void test_globals(hipLaunchParm lp, const float *a, float *b) +extern "C" __global__ void test_globals(const float *a, float *b) { int tx = hipThreadIdx_x; b[tx] = a[tx] + myDeviceGlobal+ myDeviceGlobalArray[tx%ARRAY_SIZE] ; diff --git a/src/code_object_bundle.cpp b/src/code_object_bundle.cpp index d7d2cd1e10..aab7200533 100644 --- a/src/code_object_bundle.cpp +++ b/src/code_object_bundle.cpp @@ -2,22 +2,26 @@ #include +#include +#include #include #include #include +using namespace std; + hsa_isa_t hip_impl::triple_to_hsa_isa(const std::string& triple) { static constexpr const char prefix[] = "hcc-amdgcn--amdhsa-gfx"; - static constexpr std::size_t prefix_sz = sizeof(prefix) - 1; + static constexpr size_t prefix_sz = sizeof(prefix) - 1; hsa_isa_t r = {}; auto idx = triple.find(prefix); - if (idx != std::string::npos) { + if (idx != string::npos) { idx += prefix_sz; - std::string tmp = "AMD:AMDGPU"; + string tmp = "AMD:AMDGPU"; while (idx != triple.size()) { tmp.push_back(':'); tmp.push_back(triple[idx++]); @@ -33,7 +37,31 @@ hsa_isa_t hip_impl::triple_to_hsa_isa(const std::string& triple) constexpr const char hip_impl::Bundled_code_header::magic_string_[]; // CREATORS -hip_impl::Bundled_code_header::Bundled_code_header( - const std::vector& x) +hip_impl::Bundled_code_header::Bundled_code_header(const vector& x) : Bundled_code_header{x.cbegin(), x.cend()} -{} \ No newline at end of file +{} + +hip_impl::Bundled_code_header::Bundled_code_header(const void* p) +{ // This is a pretty terrible interface, useful only because + // hipLoadModuleData is so poorly specified (for no fault of its own). + if (!p) return; + + auto ph = static_cast(p); + + if (!equal( + magic_string_, + magic_string_ + magic_string_sz_, + ph->bundler_magic_string_)) { + return; + } + + size_t sz = sizeof(Header_) + ph->bundle_cnt_ * sizeof(Bundled_code::Header); + auto pb = static_cast(p) + sizeof(Header_); + auto n = ph->bundle_cnt_; + while (n--) { + sz += reinterpret_cast(pb)->bundle_sz; + pb += sizeof(Bundled_code::Header); + } + + read(static_cast(p), static_cast(p) + sz, *this); +} \ No newline at end of file diff --git a/src/hip_module.cpp b/src/hip_module.cpp index 45a44b3666..0b1208de91 100644 --- a/src/hip_module.cpp +++ b/src/hip_module.cpp @@ -96,23 +96,6 @@ if (hsaStatus != HSA_STATUS_SUCCESS) {\ return ihipLogStatus(hipStatus);\ } -hipError_t hipModuleLoad(hipModule_t *module, const char *fname) -{ - HIP_INIT_API(module, fname); - - if (!fname) return ihipLogStatus(hipErrorInvalidValue); - - ifstream file{fname}; - - if (!file.is_open()) return ihipLogStatus(hipErrorFileNotFound); - - vector tmp{ - istreambuf_iterator{file}, istreambuf_iterator{}}; - - return hipModuleLoadData(module, tmp.data()); -} - - hipError_t hipModuleUnload(hipModule_t hmod) { HIP_INIT_API(hmod); @@ -473,6 +456,29 @@ namespace return string{s, s + sz}; } + + string code_object_blob_for_agent( + const void* maybe_bundled_code, hsa_agent_t agent) + { + if (!maybe_bundled_code) return {}; + + Bundled_code_header tmp{maybe_bundled_code}; + + if (!valid(tmp)) return {}; + + const auto agent_isa = isa(agent); + + const auto it = find_if( + bundles(tmp).cbegin(), + bundles(tmp).cend(), + [=](const Bundled_code& x) { + return agent_isa == triple_to_hsa_isa(x.triple);; + }); + + if (it == bundles(tmp).cend()) return {}; + + return string{it->blob.cbegin(), it->blob.cend()}; + } } // Anonymous namespace, internal linkage. hipError_t ihipModuleGetFunction( @@ -526,6 +532,22 @@ hipError_t hipModuleGetGlobal(hipDeviceptr_t *dptr, size_t *bytes, return ihipLogStatus(r); } +hipError_t hipModuleLoad(hipModule_t *module, const char *fname) +{ + HIP_INIT_API(module, fname); + + if (!fname) return ihipLogStatus(hipErrorInvalidValue); + + ifstream file{fname}; + + if (!file.is_open()) return ihipLogStatus(hipErrorFileNotFound); + + vector tmp{ + istreambuf_iterator{file}, istreambuf_iterator{}}; + + return hipModuleLoadData(module, tmp.data()); +} + hipError_t hipModuleLoadData(hipModule_t *module, const void *image) { HIP_INIT_API(module, image); @@ -543,8 +565,12 @@ hipError_t hipModuleLoadData(hipModule_t *module, const void *image) nullptr, &(*module)->executable); + auto tmp = code_object_blob_for_agent(image, this_agent()); + (*module)->executable = hip_impl::load_executable( - read_elf_file_as_string(image), (*module)->executable, this_agent()); + tmp.empty() ? read_elf_file_as_string(image) : tmp, + (*module)->executable, + this_agent()); return ihipLogStatus( (*module)->executable.handle ? hipSuccess : hipErrorUnknown); diff --git a/src/hsa_helpers.hpp b/src/hsa_helpers.hpp index d8e09b7aa9..0bb8c09834 100644 --- a/src/hsa_helpers.hpp +++ b/src/hsa_helpers.hpp @@ -24,8 +24,27 @@ THE SOFTWARE. #include #include +#include #include +inline +constexpr +bool operator==(hsa_isa_t x, hsa_isa_t y) +{ + return x.handle == y.handle; +} + +namespace std +{ + template<> + struct hash { + size_t operator()(hsa_isa_t x) const + { + return hash{}(x.handle); + } + }; +} + namespace hip_impl { inline @@ -57,6 +76,19 @@ namespace hip_impl return r; } + inline + hsa_isa_t isa(hsa_agent_t x) + { + hsa_isa_t r = {}; + hsa_agent_iterate_isas(x, [](hsa_isa_t i, void* o) { + *static_cast(o) = i; // Pick the first. + + return HSA_STATUS_INFO_BREAK; + }, &r); + + return r; + } + inline std::uint64_t kernel_object(hsa_executable_symbol_t x) { diff --git a/src/program_state.cpp b/src/program_state.cpp index e867887da2..64e8e832ba 100644 --- a/src/program_state.cpp +++ b/src/program_state.cpp @@ -30,34 +30,16 @@ using namespace ELFIO; using namespace hip_impl; using namespace std; -namespace std -{ - template<> - struct hash { - size_t operator()(hsa_isa_t x) const - { - return hash{}(x.handle); - } - }; -} - -inline -constexpr -bool operator==(hsa_isa_t x, hsa_isa_t y) -{ - return x.handle == y.handle; -} - namespace { struct Symbol { - std::string name; + string name; ELFIO::Elf64_Addr value = 0; - ELFIO::Elf_Xword size = 0; - ELFIO::Elf_Half sect_idx = 0; - std::uint8_t bind = 0; - std::uint8_t type = 0; - std::uint8_t other = 0; + Elf_Xword size = 0; + Elf_Half sect_idx = 0; + uint8_t bind = 0; + uint8_t type = 0; + uint8_t other = 0; }; inline @@ -185,7 +167,7 @@ namespace } } - vector code_object_blob_for_process() + vector code_object_blob_for_process() { static constexpr const char self[] = "/proc/self/exe"; static constexpr const char kernel_section[] = ".kernel"; @@ -200,7 +182,7 @@ namespace return x->get_name() == kernel_section; }); - vector r; + vector r; if (kernels) { r.insert( r.end(), @@ -211,13 +193,13 @@ namespace return r; } - const unordered_map>>& code_object_blobs() + const unordered_map>>& code_object_blobs() { - static unordered_map>> r; + static unordered_map>> r; static once_flag f; call_once(f, []() { - static vector> blobs{ + static vector> blobs{ code_object_blob_for_process()}; dl_iterate_phdr([](dl_phdr_info* info, std::size_t, void*) { @@ -481,7 +463,7 @@ namespace hip_impl const auto code_object_dynsym = find_section_if(reader, [](const ELFIO::section* x) { - return x->get_type() == SHT_DYNSYM; + return x->get_type() == SHT_DYNSYM; }); associate_code_object_symbols_with_host_allocation( diff --git a/tests/src/runtimeApi/synchronization/copy_coherency.cpp b/tests/src/runtimeApi/synchronization/copy_coherency.cpp index b2a66f61e2..d8ff6630e8 100644 --- a/tests/src/runtimeApi/synchronization/copy_coherency.cpp +++ b/tests/src/runtimeApi/synchronization/copy_coherency.cpp @@ -69,7 +69,6 @@ void MemcpyFunction::load(const char *fileName, const char *functionName) void MemcpyFunction::launch(int * dst, const int * src, size_t numElements, hipStream_t s) { struct { - uint32_t _hidden[6]; int* _dst; const int* _src; size_t _numElements;