diff --git a/projects/hip/LPL/CMakeLists.txt b/projects/hip/LPL/CMakeLists.txt new file mode 100644 index 0000000000..26e3f6f0de --- /dev/null +++ b/projects/hip/LPL/CMakeLists.txt @@ -0,0 +1,12 @@ +add_executable(lpl lpl.cpp) +set_target_properties( + lpl PROPERTIES + CXX_STANDARD 11 + CXX_STANDARD_REQUIRED ON + CXX_EXTENSIONS OFF + RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) +target_include_directories(lpl PUBLIC ${PROJECT_SOURCE_DIR}/src) +# Install LPL if platform is hcc +if (HIP_PLATFORM STREQUAL "hcc") + install(TARGETS lpl RUNTIME DESTINATION bin) +endif () \ No newline at end of file diff --git a/projects/hip/LPL/clara/clara.hpp b/projects/hip/LPL/clara/clara.hpp new file mode 100644 index 0000000000..aa429e7a19 --- /dev/null +++ b/projects/hip/LPL/clara/clara.hpp @@ -0,0 +1,1231 @@ +// v1.0-develop.2 +// See https://github.com/philsquared/Clara + +#ifndef CLARA_HPP_INCLUDED +#define CLARA_HPP_INCLUDED + +#ifndef CLARA_CONFIG_CONSOLE_WIDTH +#define CLARA_CONFIG_CONSOLE_WIDTH 80 +#endif + +#ifndef CLARA_TEXTFLOW_CONFIG_CONSOLE_WIDTH +#define CLARA_TEXTFLOW_CONFIG_CONSOLE_WIDTH CLARA_CONFIG_CONSOLE_WIDTH +#endif + +// ----------- #included from clara_textflow.hpp ----------- + +// TextFlowCpp +// +// A single-header library for wrapping and laying out basic text, by Phil Nash +// +// This work is licensed under the BSD 2-Clause license. +// See the accompanying LICENSE file, or the one at https://opensource.org/licenses/BSD-2-Clause +// +// This project is hosted at https://github.com/philsquared/textflowcpp + +#ifndef CLARA_TEXTFLOW_HPP_INCLUDED +#define CLARA_TEXTFLOW_HPP_INCLUDED + +#include +#include +#include +#include + +#ifndef CLARA_TEXTFLOW_CONFIG_CONSOLE_WIDTH +#define CLARA_TEXTFLOW_CONFIG_CONSOLE_WIDTH 80 +#endif + + +namespace clara { namespace TextFlow { + + inline auto isWhitespace( char c ) -> bool { + static std::string chars = " \t\n\r"; + return chars.find( c ) != std::string::npos; + } + inline auto isBreakableBefore( char c ) -> bool { + static std::string chars = "[({<|"; + return chars.find( c ) != std::string::npos; + } + inline auto isBreakableAfter( char c ) -> bool { + static std::string chars = "])}>.,:;*+-=&/\\"; + return chars.find( c ) != std::string::npos; + } + + class Columns; + + class Column { + std::vector m_strings; + size_t m_width = CLARA_TEXTFLOW_CONFIG_CONSOLE_WIDTH; + size_t m_indent = 0; + size_t m_initialIndent = std::string::npos; + + public: + class iterator { + friend Column; + + Column const& m_column; + size_t m_stringIndex = 0; + size_t m_pos = 0; + + size_t m_len = 0; + size_t m_end = 0; + bool m_suffix = false; + + iterator( Column const& column, size_t stringIndex ) + : m_column( column ), + m_stringIndex( stringIndex ) + {} + + auto line() const -> std::string const& { return m_column.m_strings[m_stringIndex]; } + + auto isBoundary( size_t at ) const -> bool { + assert( at > 0 ); + assert( at <= line().size() ); + + return at == line().size() || + ( isWhitespace( line()[at] ) && !isWhitespace( line()[at-1] ) ) || + isBreakableBefore( line()[at] ) || + isBreakableAfter( line()[at-1] ); + } + + void calcLength() { + assert( m_stringIndex < m_column.m_strings.size() ); + + m_suffix = false; + auto width = m_column.m_width-indent(); + m_end = m_pos; + while( m_end < line().size() && line()[m_end] != '\n' ) + ++m_end; + + if( m_end < m_pos + width ) { + m_len = m_end - m_pos; + } + else { + size_t len = width; + while (len > 0 && !isBoundary(m_pos + len)) + --len; + while (len > 0 && isWhitespace( line()[m_pos + len - 1] )) + --len; + + if (len > 0) { + m_len = len; + } else { + m_suffix = true; + m_len = width - 1; + } + } + } + + auto indent() const -> size_t { + auto initial = m_pos == 0 && m_stringIndex == 0 ? m_column.m_initialIndent : std::string::npos; + return initial == std::string::npos ? m_column.m_indent : initial; + } + + auto addIndentAndSuffix(std::string const &plain) const -> std::string { + return std::string( indent(), ' ' ) + (m_suffix ? plain + "-" : plain); + } + + public: + explicit iterator( Column const& column ) : m_column( column ) { + assert( m_column.m_width > m_column.m_indent ); + assert( m_column.m_initialIndent == std::string::npos || m_column.m_width > m_column.m_initialIndent ); + calcLength(); + if( m_len == 0 ) + m_stringIndex++; // Empty string + } + + auto operator *() const -> std::string { + assert( m_stringIndex < m_column.m_strings.size() ); + assert( m_pos <= m_end ); + if( m_pos + m_column.m_width < m_end ) + return addIndentAndSuffix(line().substr(m_pos, m_len)); + else + return addIndentAndSuffix(line().substr(m_pos, m_end - m_pos)); + } + + auto operator ++() -> iterator& { + m_pos += m_len; + if( m_pos < line().size() && line()[m_pos] == '\n' ) + m_pos += 1; + else + while( m_pos < line().size() && isWhitespace( line()[m_pos] ) ) + ++m_pos; + + if( m_pos == line().size() ) { + m_pos = 0; + ++m_stringIndex; + } + if( m_stringIndex < m_column.m_strings.size() ) + calcLength(); + return *this; + } + auto operator ++(int) -> iterator { + iterator prev( *this ); + operator++(); + return prev; + } + + auto operator ==( iterator const& other ) const -> bool { + return + m_pos == other.m_pos && + m_stringIndex == other.m_stringIndex && + &m_column == &other.m_column; + } + auto operator !=( iterator const& other ) const -> bool { + return !operator==( other ); + } + }; + using const_iterator = iterator; + + explicit Column( std::string const& text ) { m_strings.push_back( text ); } + + auto width( size_t newWidth ) -> Column& { + assert( newWidth > 0 ); + m_width = newWidth; + return *this; + } + auto indent( size_t newIndent ) -> Column& { + m_indent = newIndent; + return *this; + } + auto initialIndent( size_t newIndent ) -> Column& { + m_initialIndent = newIndent; + return *this; + } + + auto width() const -> size_t { return m_width; } + auto begin() const -> iterator { return iterator( *this ); } + auto end() const -> iterator { return { *this, m_strings.size() }; } + + inline friend std::ostream& operator << ( std::ostream& os, Column const& col ) { + bool first = true; + for( auto line : col ) { + if( first ) + first = false; + else + os << "\n"; + os << line; + } + return os; + } + + auto operator + ( Column const& other ) -> Columns; + + auto toString() const -> std::string { + std::ostringstream oss; + oss << *this; + return oss.str(); + } + }; + + class Spacer : public Column { + + public: + explicit Spacer( size_t spaceWidth ) : Column( "" ) { + width( spaceWidth ); + } + }; + + class Columns { + std::vector m_columns; + + public: + + class iterator { + friend Columns; + struct EndTag {}; + + std::vector const& m_columns; + std::vector m_iterators; + size_t m_activeIterators; + + iterator( Columns const& columns, EndTag ) + : m_columns( columns.m_columns ), + m_activeIterators( 0 ) + { + m_iterators.reserve( m_columns.size() ); + + for( auto const& col : m_columns ) + m_iterators.push_back( col.end() ); + } + + public: + explicit iterator( Columns const& columns ) + : m_columns( columns.m_columns ), + m_activeIterators( m_columns.size() ) + { + m_iterators.reserve( m_columns.size() ); + + for( auto const& col : m_columns ) + m_iterators.push_back( col.begin() ); + } + + auto operator ==( iterator const& other ) const -> bool { + return m_iterators == other.m_iterators; + } + auto operator !=( iterator const& other ) const -> bool { + return m_iterators != other.m_iterators; + } + auto operator *() const -> std::string { + std::string row, padding; + + for( size_t i = 0; i < m_columns.size(); ++i ) { + auto width = m_columns[i].width(); + if( m_iterators[i] != m_columns[i].end() ) { + std::string col = *m_iterators[i]; + row += padding + col; + if( col.size() < width ) + padding = std::string( width - col.size(), ' ' ); + else + padding = ""; + } + else { + padding += std::string( width, ' ' ); + } + } + return row; + } + auto operator ++() -> iterator& { + for( size_t i = 0; i < m_columns.size(); ++i ) { + if (m_iterators[i] != m_columns[i].end()) + ++m_iterators[i]; + } + return *this; + } + auto operator ++(int) -> iterator { + iterator prev( *this ); + operator++(); + return prev; + } + }; + using const_iterator = iterator; + + auto begin() const -> iterator { return iterator( *this ); } + auto end() const -> iterator { return { *this, iterator::EndTag() }; } + + auto operator += ( Column const& col ) -> Columns& { + m_columns.push_back( col ); + return *this; + } + auto operator + ( Column const& col ) -> Columns { + Columns combined = *this; + combined += col; + return combined; + } + + inline friend std::ostream& operator << ( std::ostream& os, Columns const& cols ) { + + bool first = true; + for( auto line : cols ) { + if( first ) + first = false; + else + os << "\n"; + os << line; + } + return os; + } + + auto toString() const -> std::string { + std::ostringstream oss; + oss << *this; + return oss.str(); + } + }; + + inline auto Column::operator + ( Column const& other ) -> Columns { + Columns cols; + cols += *this; + cols += other; + return cols; + } +}} // namespace clara::TextFlow + +#endif // CLARA_TEXTFLOW_HPP_INCLUDED + +// ----------- end of #include from clara_textflow.hpp ----------- +// ........... back in clara.hpp + + +#include +#include +#include + +#if !defined(CLARA_PLATFORM_WINDOWS) && ( defined(WIN32) || defined(__WIN32__) || defined(_WIN32) || defined(_MSC_VER) ) +#define CLARA_PLATFORM_WINDOWS +#endif + +namespace clara { +namespace detail { + + // Traits for extracting arg and return type of lambdas (for single argument lambdas) + template + struct UnaryLambdaTraits : UnaryLambdaTraits {}; + + template + struct UnaryLambdaTraits { + static const bool isValid = false; + }; + + template + struct UnaryLambdaTraits { + static const bool isValid = true; + using ArgType = typename std::remove_const::type>::type;; + using ReturnType = ReturnT; + }; + + class TokenStream; + + // Transport for raw args (copied from main args, or supplied via init list for testing) + class Args { + friend TokenStream; + std::string m_exeName; + std::vector m_args; + + public: + Args( int argc, char *argv[] ) { + m_exeName = argv[0]; + for( int i = 1; i < argc; ++i ) + m_args.push_back( argv[i] ); + } + + Args( std::initializer_list args ) + : m_exeName( *args.begin() ), + m_args( args.begin()+1, args.end() ) + {} + + auto exeName() const -> std::string { + return m_exeName; + } + }; + + // Wraps a token coming from a token stream. These may not directly correspond to strings as a single string + // may encode an option + its argument if the : or = form is used + enum class TokenType { + Option, Argument + }; + struct Token { + TokenType type; + std::string token; + }; + + inline auto isOptPrefix( char c ) -> bool { + return c == '-' +#ifdef CLARA_PLATFORM_WINDOWS + || c == '/' +#endif + ; + } + + // Abstracts iterators into args as a stream of tokens, with option arguments uniformly handled + class TokenStream { + using Iterator = std::vector::const_iterator; + Iterator it; + Iterator itEnd; + std::vector m_tokenBuffer; + + void loadBuffer() { + m_tokenBuffer.resize( 0 ); + + // Skip any empty strings + while( it != itEnd && it->empty() ) + ++it; + + if( it != itEnd ) { + auto const &next = *it; + if( isOptPrefix( next[0] ) ) { + auto delimiterPos = next.find_first_of( " :=" ); + if( delimiterPos != std::string::npos ) { + m_tokenBuffer.push_back( { TokenType::Option, next.substr( 0, delimiterPos ) } ); + m_tokenBuffer.push_back( { TokenType::Argument, next.substr( delimiterPos + 1 ) } ); + } else { + if( next[1] != '-' && next.size() > 2 ) { + std::string opt = "- "; + for( size_t i = 1; i < next.size(); ++i ) { + opt[1] = next[i]; + m_tokenBuffer.push_back( { TokenType::Option, opt } ); + } + } else { + m_tokenBuffer.push_back( { TokenType::Option, next } ); + } + } + } else { + m_tokenBuffer.push_back( { TokenType::Argument, next } ); + } + } + } + + public: + explicit TokenStream( Args const &args ) : TokenStream( args.m_args.begin(), args.m_args.end() ) {} + + TokenStream( Iterator it, Iterator itEnd ) : it( it ), itEnd( itEnd ) { + loadBuffer(); + } + + explicit operator bool() const { + return !m_tokenBuffer.empty() || it != itEnd; + } + + auto count() const -> size_t { return m_tokenBuffer.size() + (itEnd - it); } + + auto operator*() const -> Token { + assert( !m_tokenBuffer.empty() ); + return m_tokenBuffer.front(); + } + + auto operator->() const -> Token const * { + assert( !m_tokenBuffer.empty() ); + return &m_tokenBuffer.front(); + } + + auto operator++() -> TokenStream & { + if( m_tokenBuffer.size() >= 2 ) { + m_tokenBuffer.erase( m_tokenBuffer.begin() ); + } else { + if( it != itEnd ) + ++it; + loadBuffer(); + } + return *this; + } + }; + + + class ResultBase { + public: + enum Type { + Ok, LogicError, RuntimeError + }; + + protected: + ResultBase( Type type ) : m_type( type ) {} + virtual ~ResultBase() = default; + + virtual void enforceOk() const = 0; + + Type m_type; + }; + + template + class ResultValueBase : public ResultBase { + public: + auto value() const -> T const & { + enforceOk(); + return m_value; + } + + protected: + ResultValueBase( Type type ) : ResultBase( type ) {} + + ResultValueBase( ResultValueBase const &other ) : ResultBase( other ) { + if( m_type == ResultBase::Ok ) + new( &m_value ) T( other.m_value ); + } + + ResultValueBase( Type, T const &value ) : ResultBase( Ok ) { + new( &m_value ) T( value ); + } + + auto operator=( ResultValueBase const &other ) -> ResultValueBase & { + if( m_type == ResultBase::Ok ) + m_value.~T(); + ResultBase::operator=(other); + if( m_type == ResultBase::Ok ) + new( &m_value ) T( other.m_value ); + return *this; + } + + ~ResultValueBase() { + if( m_type == Ok ) + m_value.~T(); + } + + union { + T m_value; + }; + }; + + template<> + class ResultValueBase : public ResultBase { + protected: + using ResultBase::ResultBase; + }; + + template + class BasicResult : public ResultValueBase { + public: + template + explicit BasicResult( BasicResult const &other ) + : ResultValueBase( other.type() ), + m_errorMessage( other.errorMessage() ) + { + assert( type() != ResultBase::Ok ); + } + + template + static auto ok( U const &value ) -> BasicResult { return { ResultBase::Ok, value }; } + static auto ok() -> BasicResult { return { ResultBase::Ok }; } + static auto logicError( std::string const &message ) -> BasicResult { return { ResultBase::LogicError, message }; } + static auto runtimeError( std::string const &message ) -> BasicResult { return { ResultBase::RuntimeError, message }; } + + explicit operator bool() const { return m_type == ResultBase::Ok; } + auto type() const -> ResultBase::Type { return m_type; } + auto errorMessage() const -> std::string { return m_errorMessage; } + + protected: + virtual void enforceOk() const { + // !TBD: If no exceptions, std::terminate here or something + switch( m_type ) { + case ResultBase::LogicError: + throw std::logic_error( m_errorMessage ); + case ResultBase::RuntimeError: + throw std::runtime_error( m_errorMessage ); + case ResultBase::Ok: + break; + } + } + + std::string m_errorMessage; // Only populated if resultType is an error + + BasicResult( ResultBase::Type type, std::string const &message ) + : ResultValueBase(type), + m_errorMessage(message) + { + assert( m_type != ResultBase::Ok ); + } + + using ResultValueBase::ResultValueBase; + using ResultBase::m_type; + }; + + enum class ParseResultType { + Matched, NoMatch, ShortCircuitAll, ShortCircuitSame + }; + + class ParseState { + public: + + ParseState( ParseResultType type, TokenStream const &remainingTokens ) + : m_type(type), + m_remainingTokens( remainingTokens ) + {} + + auto type() const -> ParseResultType { return m_type; } + auto remainingTokens() const -> TokenStream { return m_remainingTokens; } + + private: + ParseResultType m_type; + TokenStream m_remainingTokens; + }; + + using Result = BasicResult; + using ParserResult = BasicResult; + using InternalParseResult = BasicResult; + + struct HelpColumns { + std::string left; + std::string right; + }; + + template + inline auto convertInto( std::string const &source, T& target ) -> ParserResult { + std::stringstream ss; + ss << source; + ss >> target; + if( ss.fail() ) + return ParserResult::runtimeError( "Unable to convert '" + source + "' to destination type" ); + else + return ParserResult::ok( ParseResultType::Matched ); + } + inline auto convertInto( std::string const &source, std::string& target ) -> ParserResult { + target = source; + return ParserResult::ok( ParseResultType::Matched ); + } + inline auto convertInto( std::string const &source, bool &target ) -> ParserResult { + std::string srcLC = source; + std::transform( srcLC.begin(), srcLC.end(), srcLC.begin(), []( char c ) { return static_cast( ::tolower(c) ); } ); + if (srcLC == "y" || srcLC == "1" || srcLC == "true" || srcLC == "yes" || srcLC == "on") + target = true; + else if (srcLC == "n" || srcLC == "0" || srcLC == "false" || srcLC == "no" || srcLC == "off") + target = false; + else + return ParserResult::runtimeError( "Expected a boolean value but did not recognise: '" + source + "'" ); + return ParserResult::ok( ParseResultType::Matched ); + } + + struct BoundRefBase { + BoundRefBase() = default; + BoundRefBase( BoundRefBase const & ) = delete; + BoundRefBase( BoundRefBase && ) = delete; + BoundRefBase &operator=( BoundRefBase const & ) = delete; + BoundRefBase &operator=( BoundRefBase && ) = delete; + + virtual ~BoundRefBase() = default; + + virtual auto isFlag() const -> bool = 0; + virtual auto isContainer() const -> bool { return false; } + virtual auto setValue( std::string const &arg ) -> ParserResult = 0; + virtual auto setFlag( bool flag ) -> ParserResult = 0; + }; + + struct BoundValueRefBase : BoundRefBase { + auto isFlag() const -> bool override { return false; } + + auto setFlag( bool ) -> ParserResult override { + return ParserResult::logicError( "Flags can only be set on boolean fields" ); + } + }; + + struct BoundFlagRefBase : BoundRefBase { + auto isFlag() const -> bool override { return true; } + + auto setValue( std::string const &arg ) -> ParserResult override { + bool flag; + auto result = convertInto( arg, flag ); + if( result ) + setFlag( flag ); + return result; + } + }; + + template + struct BoundRef : BoundValueRefBase { + T &m_ref; + + explicit BoundRef( T &ref ) : m_ref( ref ) {} + + auto setValue( std::string const &arg ) -> ParserResult override { + return convertInto( arg, m_ref ); + } + }; + + template + struct BoundRef> : BoundValueRefBase { + std::vector &m_ref; + + explicit BoundRef( std::vector &ref ) : m_ref( ref ) {} + + auto isContainer() const -> bool override { return true; } + + auto setValue( std::string const &arg ) -> ParserResult override { + T temp; + auto result = convertInto( arg, temp ); + if( result ) + m_ref.push_back( temp ); + return result; + } + }; + + struct BoundFlagRef : BoundFlagRefBase { + bool &m_ref; + + explicit BoundFlagRef( bool &ref ) : m_ref( ref ) {} + + auto setFlag( bool flag ) -> ParserResult override { + m_ref = flag; + return ParserResult::ok( ParseResultType::Matched ); + } + }; + + template + struct LambdaInvoker { + static_assert( std::is_same::value, "Lambda must return void or clara::ParserResult" ); + + template + static auto invoke( L const &lambda, ArgType const &arg ) -> ParserResult { + return lambda( arg ); + } + }; + + template<> + struct LambdaInvoker { + template + static auto invoke( L const &lambda, ArgType const &arg ) -> ParserResult { + lambda( arg ); + return ParserResult::ok( ParseResultType::Matched ); + } + }; + + template + inline auto invokeLambda( L const &lambda, std::string const &arg ) -> ParserResult { + ArgType temp; + auto result = convertInto( arg, temp ); + return !result + ? result + : LambdaInvoker::ReturnType>::invoke( lambda, temp ); + }; + + + template + struct BoundLambda : BoundValueRefBase { + L m_lambda; + + static_assert( UnaryLambdaTraits::isValid, "Supplied lambda must take exactly one argument" ); + explicit BoundLambda( L const &lambda ) : m_lambda( lambda ) {} + + auto setValue( std::string const &arg ) -> ParserResult override { + return invokeLambda::ArgType>( m_lambda, arg ); + } + }; + + template + struct BoundFlagLambda : BoundFlagRefBase { + L m_lambda; + + static_assert( UnaryLambdaTraits::isValid, "Supplied lambda must take exactly one argument" ); + static_assert( std::is_same::ArgType, bool>::value, "flags must be boolean" ); + + explicit BoundFlagLambda( L const &lambda ) : m_lambda( lambda ) {} + + auto setFlag( bool flag ) -> ParserResult override { + return LambdaInvoker::ReturnType>::invoke( m_lambda, flag ); + } + }; + + enum class Optionality { Optional, Required }; + + struct Parser; + + class ParserBase { + public: + virtual ~ParserBase() = default; + virtual auto validate() const -> Result { return Result::ok(); } + virtual auto parse( std::string const& exeName, TokenStream const &tokens) const -> InternalParseResult = 0; + virtual auto cardinality() const -> size_t { return 1; } + + auto parse( Args const &args ) const -> InternalParseResult { + return parse( args.exeName(), TokenStream( args ) ); + } + }; + + template + class ComposableParserImpl : public ParserBase { + public: + template + auto operator|( T const &other ) const -> Parser; + }; + + // Common code and state for Args and Opts + template + class ParserRefImpl : public ComposableParserImpl { + protected: + Optionality m_optionality = Optionality::Optional; + std::shared_ptr m_ref; + std::string m_hint; + std::string m_description; + + explicit ParserRefImpl( std::shared_ptr const &ref ) : m_ref( ref ) {} + + public: + template + ParserRefImpl( T &ref, std::string const &hint ) + : m_ref( std::make_shared>( ref ) ), + m_hint( hint ) + {} + + template + ParserRefImpl( LambdaT const &ref, std::string const &hint ) + : m_ref( std::make_shared>( ref ) ), + m_hint(hint) + {} + + auto operator()( std::string const &description ) -> DerivedT & { + m_description = description; + return static_cast( *this ); + } + + auto optional() -> DerivedT & { + m_optionality = Optionality::Optional; + return static_cast( *this ); + }; + + auto required() -> DerivedT & { + m_optionality = Optionality::Required; + return static_cast( *this ); + }; + + auto isOptional() const -> bool { + return m_optionality == Optionality::Optional; + } + + auto cardinality() const -> size_t override { + if( m_ref->isContainer() ) + return 0; + else + return 1; + } + + auto hint() const -> std::string { return m_hint; } + }; + + class ExeName : public ComposableParserImpl { + std::shared_ptr m_name; + std::shared_ptr m_ref; + + template + static auto makeRef(LambdaT const &lambda) -> std::shared_ptr { + return std::make_shared>( lambda) ; + } + + public: + ExeName() : m_name( std::make_shared( "" ) ) {} + + explicit ExeName( std::string &ref ) : ExeName() { + m_ref = std::make_shared>( ref ); + } + + template + explicit ExeName( LambdaT const& lambda ) : ExeName() { + m_ref = std::make_shared>( lambda ); + } + + // The exe name is not parsed out of the normal tokens, but is handled specially + auto parse( std::string const&, TokenStream const &tokens ) const -> InternalParseResult override { + return InternalParseResult::ok( ParseState( ParseResultType::NoMatch, tokens ) ); + } + + auto name() const -> std::string { return *m_name; } + auto set( std::string const& newName ) -> ParserResult { + + auto lastSlash = newName.find_last_of( "\\/" ); + auto filename = ( lastSlash == std::string::npos ) + ? newName + : newName.substr( lastSlash+1 ); + + *m_name = filename; + if( m_ref ) + return m_ref->setValue( filename ); + else + return ParserResult::ok( ParseResultType::Matched ); + } + }; + + class Arg : public ParserRefImpl { + public: + using ParserRefImpl::ParserRefImpl; + + auto parse( std::string const &, TokenStream const &tokens ) const -> InternalParseResult override { + auto validationResult = validate(); + if( !validationResult ) + return InternalParseResult( validationResult ); + + auto remainingTokens = tokens; + auto const &token = *remainingTokens; + if( token.type != TokenType::Argument ) + return InternalParseResult::ok( ParseState( ParseResultType::NoMatch, remainingTokens ) ); + + auto result = m_ref->setValue( remainingTokens->token ); + if( !result ) + return InternalParseResult( result ); + else + return InternalParseResult::ok( ParseState( ParseResultType::Matched, ++remainingTokens ) ); + } + }; + + inline auto normaliseOpt( std::string const &optName ) -> std::string { +#ifdef CLARA_PLATFORM_WINDOWS + if( optName[0] == '/' ) + return "-" + optName.substr( 1 ); + else +#endif + return optName; + } + + class Opt : public ParserRefImpl { + protected: + std::vector m_optNames; + + public: + template + explicit Opt( LambdaT const &ref ) : ParserRefImpl( std::make_shared>( ref ) ) {} + + explicit Opt( bool &ref ) : ParserRefImpl( std::make_shared( ref ) ) {} + + template + Opt( LambdaT const &ref, std::string const &hint ) : ParserRefImpl( ref, hint ) {} + + template + Opt( T &ref, std::string const &hint ) : ParserRefImpl( ref, hint ) {} + + auto operator[]( std::string const &optName ) -> Opt & { + m_optNames.push_back( optName ); + return *this; + } + + auto getHelpColumns() const -> std::vector { + std::ostringstream oss; + bool first = true; + for( auto const &opt : m_optNames ) { + if (first) + first = false; + else + oss << ", "; + oss << opt; + } + if( !m_hint.empty() ) + oss << " <" << m_hint << ">"; + return { { oss.str(), m_description } }; + } + + auto isMatch( std::string const &optToken ) const -> bool { + auto normalisedToken = normaliseOpt( optToken ); + for( auto const &name : m_optNames ) { + if( normaliseOpt( name ) == normalisedToken ) + return true; + } + return false; + } + + using ParserBase::parse; + + auto parse( std::string const&, TokenStream const &tokens ) const -> InternalParseResult override { + auto validationResult = validate(); + if( !validationResult ) + return InternalParseResult( validationResult ); + + auto remainingTokens = tokens; + if( remainingTokens && remainingTokens->type == TokenType::Option ) { + auto const &token = *remainingTokens; + if( isMatch(token.token ) ) { + if( m_ref->isFlag() ) { + auto result = m_ref->setFlag( true ); + if( !result ) + return InternalParseResult( result ); + if( result.value() == ParseResultType::ShortCircuitAll ) + return InternalParseResult::ok( ParseState( result.value(), remainingTokens ) ); + } else { + ++remainingTokens; + if( !remainingTokens ) + return InternalParseResult::runtimeError( "Expected argument following " + token.token ); + auto const &argToken = *remainingTokens; + if( argToken.type != TokenType::Argument ) + return InternalParseResult::runtimeError( "Expected argument following " + token.token ); + auto result = m_ref->setValue( argToken.token ); + if( !result ) + return InternalParseResult( result ); + if( result.value() == ParseResultType::ShortCircuitAll ) + return InternalParseResult::ok( ParseState( result.value(), remainingTokens ) ); + } + return InternalParseResult::ok( ParseState( ParseResultType::Matched, ++remainingTokens ) ); + } + } + return InternalParseResult::ok( ParseState( ParseResultType::NoMatch, remainingTokens ) ); + } + + auto validate() const -> Result override { + if( m_optNames.empty() ) + return Result::logicError( "No options supplied to Opt" ); + for( auto const &name : m_optNames ) { + if( name.empty() ) + return Result::logicError( "Option name cannot be empty" ); +#ifdef CLARA_PLATFORM_WINDOWS + if( name[0] != '-' && name[0] != '/' ) + return Result::logicError( "Option name must begin with '-' or '/'" ); +#else + if( name[0] != '-' ) + return Result::logicError( "Option name must begin with '-'" ); +#endif + } + return ParserRefImpl::validate(); + } + }; + + struct Help : Opt { + Help( bool &showHelpFlag ) + : Opt([&]( bool flag ) { + showHelpFlag = flag; + return ParserResult::ok( ParseResultType::ShortCircuitAll ); + }) + { + static_cast( *this ) + ("display usage information") + ["-?"]["-h"]["--help"] + .optional(); + } + }; + + + struct Parser : ParserBase { + + mutable ExeName m_exeName; + std::vector m_options; + std::vector m_args; + + auto operator|=( ExeName const &exeName ) -> Parser & { + m_exeName = exeName; + return *this; + } + + auto operator|=( Arg const &arg ) -> Parser & { + m_args.push_back(arg); + return *this; + } + + auto operator|=( Opt const &opt ) -> Parser & { + m_options.push_back(opt); + return *this; + } + + auto operator|=( Parser const &other ) -> Parser & { + m_options.insert(m_options.end(), other.m_options.begin(), other.m_options.end()); + m_args.insert(m_args.end(), other.m_args.begin(), other.m_args.end()); + return *this; + } + + template + auto operator|( T const &other ) const -> Parser { + return Parser( *this ) |= other; + } + + auto getHelpColumns() const -> std::vector { + std::vector cols; + for (auto const &o : m_options) { + auto childCols = o.getHelpColumns(); + cols.insert( cols.end(), childCols.begin(), childCols.end() ); + } + return cols; + } + + void writeToStream( std::ostream &os ) const { + if (!m_exeName.name().empty()) { + os << "usage:\n" << " " << m_exeName.name() << " "; + bool required = true, first = true; + for( auto const &arg : m_args ) { + if (first) + first = false; + else + os << " "; + if( arg.isOptional() && required ) { + os << "["; + required = false; + } + os << "<" << arg.hint() << ">"; + if( arg.cardinality() == 0 ) + os << " ... "; + } + if( !required ) + os << "]"; + if( !m_options.empty() ) + os << " options"; + os << "\n\nwhere options are:" << std::endl; + } + + auto rows = getHelpColumns(); + size_t consoleWidth = CLARA_CONFIG_CONSOLE_WIDTH; + size_t optWidth = 0; + for( auto const &cols : rows ) + optWidth = (std::max)(optWidth, cols.left.size() + 2); + + for( auto const &cols : rows ) { + auto row = + TextFlow::Column( cols.left ).width( optWidth ).indent( 2 ) + + TextFlow::Spacer(4) + + TextFlow::Column( cols.right ).width( consoleWidth - 7 - optWidth ); + os << row << std::endl; + } + } + + friend auto operator<<( std::ostream &os, Parser const &parser ) -> std::ostream& { + parser.writeToStream( os ); + return os; + } + + auto validate() const -> Result override { + for( auto const &opt : m_options ) { + auto result = opt.validate(); + if( !result ) + return result; + } + for( auto const &arg : m_args ) { + auto result = arg.validate(); + if( !result ) + return result; + } + return Result::ok(); + } + + using ParserBase::parse; + + auto parse( std::string const& exeName, TokenStream const &tokens ) const -> InternalParseResult override { + + struct ParserInfo { + ParserBase const* parser = nullptr; + size_t count = 0; + }; + const size_t totalParsers = m_options.size() + m_args.size(); + assert( totalParsers < 512 ); + // ParserInfo parseInfos[totalParsers]; // <-- this is what we really want to do + ParserInfo parseInfos[512]; + + { + size_t i = 0; + for (auto const &opt : m_options) parseInfos[i++].parser = &opt; + for (auto const &arg : m_args) parseInfos[i++].parser = &arg; + } + + m_exeName.set( exeName ); + + auto result = InternalParseResult::ok( ParseState( ParseResultType::NoMatch, tokens ) ); + while( result.value().remainingTokens() ) { + bool tokenParsed = false; + + for( size_t i = 0; i < totalParsers; ++i ) { + auto& parseInfo = parseInfos[i]; + if( parseInfo.parser->cardinality() == 0 || parseInfo.count < parseInfo.parser->cardinality() ) { + result = parseInfo.parser->parse(exeName, result.value().remainingTokens()); + if (!result) + return result; + if (result.value().type() != ParseResultType::NoMatch) { + tokenParsed = true; + ++parseInfo.count; + break; + } + } + } + + if( result.value().type() == ParseResultType::ShortCircuitAll ) + return result; + if( !tokenParsed ) + return InternalParseResult::runtimeError( "Unrecognised token: " + result.value().remainingTokens()->token ); + } + // !TBD Check missing required options + return result; + } + }; + + template + template + auto ComposableParserImpl::operator|( T const &other ) const -> Parser { + return Parser() | static_cast( *this ) | other; + } +} // namespace detail + + +// A Combined parser +using detail::Parser; + +// A parser for options +using detail::Opt; + +// A parser for arguments +using detail::Arg; + +// Wrapper for argc, argv from main() +using detail::Args; + +// Specifies the name of the executable +using detail::ExeName; + +// Convenience wrapper for option parser that specifies the help option +using detail::Help; + +// enum of result types from a parse +using detail::ParseResultType; + +// Result type for parser operation +using detail::ParserResult; + + +} // namespace clara + +#endif // CLARA_HPP_INCLUDED diff --git a/projects/hip/LPL/lpl.cpp b/projects/hip/LPL/lpl.cpp new file mode 100644 index 0000000000..0b99f758c2 --- /dev/null +++ b/projects/hip/LPL/lpl.cpp @@ -0,0 +1,56 @@ +#include "lpl.hpp" + +#include +#include +#include +#include +#include +#include + +using namespace clara; +using namespace hip_impl; +using namespace std; + +int main(int argc, char** argv) +{ + try { + if (!hipcc_and_lpl_colocated()) { + throw runtime_error{ + "The LPL executable and hipcc must be in the same directory."}; + } + + bool help = false; + string flags; + string output; + vector sources; + string targets; + + auto cmd = cmdline_parser(help, sources, targets, flags, output); + + const auto r = cmd.parse(Args{argc, argv}); + + if (!r) throw runtime_error{r.errorMessage()}; + + if (help) cout << cmd << endl; + else { + if (sources.empty()) throw runtime_error{"No inputs specified."}; + + auto tmp = tokenize_targets(targets); + if (tmp.empty()) { + tmp.assign(amdgpu_targets().cbegin(), amdgpu_targets().cend()); + } + else validate_targets(tmp); + + if (output.empty()) for (auto&& x : tmp) output += x; + + generate_fat_binary(sources, tmp, flags, output); + } + } + catch (const exception& ex) { + cerr << ex.what() << endl; + + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} \ No newline at end of file diff --git a/projects/hip/LPL/lpl.hpp b/projects/hip/LPL/lpl.hpp new file mode 100644 index 0000000000..f3d0c537b4 --- /dev/null +++ b/projects/hip/LPL/lpl.hpp @@ -0,0 +1,246 @@ +#include "clara/clara.hpp" +#include "pstreams/pstream.h" +#include "../src/elfio/elfio.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace hip_impl +{ + inline + const std::unordered_set& amdgpu_targets() + { // The evolving list lives at: + // https://www.llvm.org/docs/AMDGPUUsage.html#processors. + static const std::unordered_set r{ + "gfx701", "gfx801", "gfx802", "gfx803", "gfx900"}; + + return r; + } + + inline + const std::string& fat_binary_extension() + { + static const std::string r{".adipose"}; + + return r; + } + + inline + const std::string& kernel_section() + { + static const std::string r{".kernel"}; + + return r; + } + + inline + const std::string& path_to_self() + { + static constexpr const char self[] = "/proc/self/exe"; + + static std::string r(PATH_MAX, '\0'); + static std::once_flag f; + + std::call_once(f, []() { + decltype(readlink(self, &r.front(), r.size())) read_cnt; + do { + read_cnt = readlink(self, &r.front(), r.size()); + } while (read_cnt == -1); + + r.resize(read_cnt); + }); + + return r; + } + + inline + const std::string& path_to_hipcc() + { + assert(!path_to_self().empty()); + + static const auto r = path_to_self().substr( + 0, path_to_self().find_last_of('/')) += "/hipcc"; + + return r; + } + + inline + std::string make_hipcc_call( + const std::vector& sources, + const std::vector& targets, + const std::string& flags, + const std::string& hipcc_output) + { + assert(!sources.empty() && !targets.empty() && !hipcc_output.empty()); + + std::string r{path_to_hipcc() + ' '}; + + for (auto&& x : sources) r += x + ' '; + r += "-o " + hipcc_output + ' '; + for (auto&& x : targets) r += "--amdgpu-target=" + x + ' '; + r += flags + " -fPIC -shared"; + + return r; + } + + inline + void copy_kernel_section_to_fat_binary( + const std::string& tmp, const std::string& output) + { + ELFIO::elfio reader; + if (!reader.load(tmp)) { + throw std::runtime_error{ + "The result of the compilation is inaccessible."}; + } + + const auto it = std::find_if( + reader.sections.begin(), + reader.sections.end(), + [](const ELFIO::section* x) { + return x->get_name() == kernel_section(); + }); + + std::ofstream out{output + fat_binary_extension()}; + + if (it == reader.sections.end()) { + std::cerr << "Warning: no kernels were generated; fat binary shall " + "be empty." << std::endl; + } + else { + std::copy_n( + (*it)->get_data(), + (*it)->get_size(), + std::ostreambuf_iterator{out}); + } + } + + inline + void generate_fat_binary( + const std::vector& sources, + const std::vector& targets, + const std::string& flags, + const std::string& output) + { + static const auto d = [](const std::string* f) { remove(f->c_str()); }; + + std::unique_ptr tmp{&output, d}; + + redi::ipstream hipcc{ + make_hipcc_call(sources, targets, flags, *tmp), + redi::pstream::pstderr}; + + if (!hipcc.is_open()) { + throw std::runtime_error{"Compiler invocation failed."}; + } + + std::string log; + while (std::getline(hipcc, log)) std::cout << log << '\n'; + + hipcc.close(); + + if (hipcc.rdbuf()->exited() && + hipcc.rdbuf()->status() != EXIT_SUCCESS) { + throw std::runtime_error{"Compilation failed."}; + } + + copy_kernel_section_to_fat_binary(*tmp, output); + } + + inline + bool file_exists(const std::string& path_to) + { + return static_cast(std::ifstream{path_to}); + } + + inline + bool hipcc_and_lpl_colocated() + { + if (path_to_self().empty()) return false; + + return file_exists(path_to_hipcc()); + } + + inline + std::vector tokenize_targets(const std::string& x) + { // TODO: move to regular expressions once we clarify the need to support + // ancient standard library implementations. + if (x.empty()) return {}; + + static constexpr const char valid_characters[] = "gfx0123456789,"; + + if (x.find_first_not_of(valid_characters) != std::string::npos) { + throw std::runtime_error{"Invalid target string: " + x}; + } + + std::vector r; + + auto it = x.cbegin(); + do { + auto it1 = std::find(it, x.cend(), ','); + r.emplace_back(it, it1); + + if (it1 == x.cend()) break; + + it = ++it1; + } while (true); + + return r; + } + + inline + void validate_targets(const std::vector& x) + { + assert(!x.empty()); + + for (auto&& t : x) { + static const std::string digits{"0123456789"}; + static const std::string pre{"gfx"}; + + if (t.find(pre) != 0 || + t.find_first_not_of(digits, pre.size()) != std::string::npos) { + throw std::runtime_error{"Invalid target: " + t}; + } + + if (amdgpu_targets().find(t) == amdgpu_targets().cend()) { + std::cerr << "Warning: target " << t + << " has not been validated yet; it may be invalid." + << std::endl; + } + } + } + + inline + clara::Parser cmdline_parser( + bool& help, + std::vector& sources, + std::string& targets, + std::string& flags, + std::string& output) + { + return + clara::Opt{flags, "\"-v -DMACRO etc.\""} + ["-f"]["--flags"]( + "flags for compilation; must be valid for hipcc.") | + clara::Help{help} | + clara::Opt{output, "filename"} + ["-o"]["--output"]( + "name of fat-binary output file; the binary format of the " + "file is documented at: https://reviews.llvm.org/D13909.") | + clara::Arg{sources, "a.cpp b.cpp etc."}( + "inputs for compilation; must contain valid C++ code.") | + clara::Opt{targets, "gfx803,gfx900 etc."} + ["-t"]["--targets"]( + "targets for AMDGPU lowering; must be one of the processors" + " with ROCm support from " + "https://www.llvm.org/docs/AMDGPUUsage.html#processors."); + } +} \ No newline at end of file diff --git a/projects/hip/LPL/pstreams/pstream.h b/projects/hip/LPL/pstreams/pstream.h new file mode 100644 index 0000000000..28cbeadb1e --- /dev/null +++ b/projects/hip/LPL/pstreams/pstream.h @@ -0,0 +1,2255 @@ +// PStreams - POSIX Process I/O for C++ + +// Copyright (C) 2001 - 2017 Jonathan Wakely +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) +// + +/** + * @file pstream.h + * @brief Declares all PStreams classes. + * @author Jonathan Wakely + * + * Defines classes redi::ipstream, redi::opstream, redi::pstream + * and redi::rpstream. + */ + +#ifndef REDI_PSTREAM_H_SEEN +#define REDI_PSTREAM_H_SEEN + +#include +#include +#include +#include +#include +#include +#include // for min() +#include // for errno +#include // for size_t, NULL +#include // for exit() +#include // for pid_t +#include // for waitpid() +#include // for ioctl() and FIONREAD +#if defined(__sun) +# include // for FIONREAD on Solaris 2.5 +#endif +#include // for pipe() fork() exec() and filedes functions +#include // for kill() +#include // for fcntl() +#if REDI_EVISCERATE_PSTREAMS +# include // for FILE, fdopen() +#endif + + +/// The library version. +#define PSTREAMS_VERSION 0x0101 // 1.0.1 + +/** + * @namespace redi + * @brief All PStreams classes are declared in namespace redi. + * + * Like the standard iostreams, PStreams is a set of class templates, + * taking a character type and traits type. As with the standard streams + * they are most likely to be used with @c char and the default + * traits type, so typedefs for this most common case are provided. + * + * The @c pstream_common class template is not intended to be used directly, + * it is used internally to provide the common functionality for the + * other stream classes. + */ +namespace redi +{ + /// Common base class providing constants and typenames. + struct pstreams + { + /// Type used to specify how to connect to the process. + typedef std::ios_base::openmode pmode; + + /// Type used to hold the arguments for a command. + typedef std::vector argv_type; + + /// Type used for file descriptors. + typedef int fd_type; + + static const pmode pstdin = std::ios_base::out; ///< Write to stdin + static const pmode pstdout = std::ios_base::in; ///< Read from stdout + static const pmode pstderr = std::ios_base::app; ///< Read from stderr + + /// Create a new process group for the child process. + static const pmode newpg = std::ios_base::trunc; + + protected: + enum { bufsz = 32 }; ///< Size of pstreambuf buffers. + enum { pbsz = 2 }; ///< Number of putback characters kept. + }; + + /// Class template for stream buffer. + template > + class basic_pstreambuf + : public std::basic_streambuf + , public pstreams + { + public: + // Type definitions for dependent types + typedef CharT char_type; + typedef Traits traits_type; + typedef typename traits_type::int_type int_type; + typedef typename traits_type::off_type off_type; + typedef typename traits_type::pos_type pos_type; + /** @deprecated use pstreams::fd_type instead. */ + typedef fd_type fd_t; + + /// Default constructor. + basic_pstreambuf(); + + /// Constructor that initialises the buffer with @a cmd. + basic_pstreambuf(const std::string& cmd, pmode mode); + + /// Constructor that initialises the buffer with @a file and @a argv. + basic_pstreambuf( const std::string& file, + const argv_type& argv, + pmode mode ); + + /// Destructor. + ~basic_pstreambuf(); + + /// Initialise the stream buffer with @a cmd. + basic_pstreambuf* + open(const std::string& cmd, pmode mode); + + /// Initialise the stream buffer with @a file and @a argv. + basic_pstreambuf* + open(const std::string& file, const argv_type& argv, pmode mode); + + /// Close the stream buffer and wait for the process to exit. + basic_pstreambuf* + close(); + + /// Send a signal to the process. + basic_pstreambuf* + kill(int signal = SIGTERM); + + /// Send a signal to the process' process group. + basic_pstreambuf* + killpg(int signal = SIGTERM); + + /// Close the pipe connected to the process' stdin. + void + peof(); + + /// Change active input source. + bool + read_err(bool readerr = true); + + /// Report whether the stream buffer has been initialised. + bool + is_open() const; + + /// Report whether the process has exited. + bool + exited(); + +#if REDI_EVISCERATE_PSTREAMS + /// Obtain FILE pointers for each of the process' standard streams. + std::size_t + fopen(FILE*& in, FILE*& out, FILE*& err); +#endif + + /// Return the exit status of the process. + int + status() const; + + /// Return the error number (errno) for the most recent failed operation. + int + error() const; + + protected: + /// Transfer characters to the pipe when character buffer overflows. + int_type + overflow(int_type c); + + /// Transfer characters from the pipe when the character buffer is empty. + int_type + underflow(); + + /// Make a character available to be returned by the next extraction. + int_type + pbackfail(int_type c = traits_type::eof()); + + /// Write any buffered characters to the stream. + int + sync(); + + /// Insert multiple characters into the pipe. + std::streamsize + xsputn(const char_type* s, std::streamsize n); + + /// Insert a sequence of characters into the pipe. + std::streamsize + write(const char_type* s, std::streamsize n); + + /// Extract a sequence of characters from the pipe. + std::streamsize + read(char_type* s, std::streamsize n); + + /// Report how many characters can be read from active input without blocking. + std::streamsize + showmanyc(); + + protected: + /// Enumerated type to indicate whether stdout or stderr is to be read. + enum buf_read_src { rsrc_out = 0, rsrc_err = 1 }; + + /// Initialise pipes and fork process. + pid_t + fork(pmode mode); + + /// Wait for the child process to exit. + int + wait(bool nohang = false); + + /// Return the file descriptor for the output pipe. + fd_type& + wpipe(); + + /// Return the file descriptor for the active input pipe. + fd_type& + rpipe(); + + /// Return the file descriptor for the specified input pipe. + fd_type& + rpipe(buf_read_src which); + + void + create_buffers(pmode mode); + + void + destroy_buffers(pmode mode); + + /// Writes buffered characters to the process' stdin pipe. + bool + empty_buffer(); + + bool + fill_buffer(bool non_blocking = false); + + /// Return the active input buffer. + char_type* + rbuffer(); + + buf_read_src + switch_read_buffer(buf_read_src); + + private: + basic_pstreambuf(const basic_pstreambuf&); + basic_pstreambuf& operator=(const basic_pstreambuf&); + + void + init_rbuffers(); + + pid_t ppid_; // pid of process + fd_type wpipe_; // pipe used to write to process' stdin + fd_type rpipe_[2]; // two pipes to read from, stdout and stderr + char_type* wbuffer_; + char_type* rbuffer_[2]; + char_type* rbufstate_[3]; + /// Index into rpipe_[] to indicate active source for read operations. + buf_read_src rsrc_; + int status_; // hold exit status of child process + int error_; // hold errno if fork() or exec() fails + }; + + /// Class template for common base class. + template > + class pstream_common + : virtual public std::basic_ios + , virtual public pstreams + { + protected: + typedef basic_pstreambuf streambuf_type; + + typedef pstreams::pmode pmode; + typedef pstreams::argv_type argv_type; + + /// Default constructor. + pstream_common(); + + /// Constructor that initialises the stream by starting a process. + pstream_common(const std::string& cmd, pmode mode); + + /// Constructor that initialises the stream by starting a process. + pstream_common(const std::string& file, const argv_type& argv, pmode mode); + + /// Pure virtual destructor. + virtual + ~pstream_common() = 0; + + /// Start a process. + void + do_open(const std::string& cmd, pmode mode); + + /// Start a process. + void + do_open(const std::string& file, const argv_type& argv, pmode mode); + + public: + /// Close the pipe. + void + close(); + + /// Report whether the stream's buffer has been initialised. + bool + is_open() const; + + /// Return the command used to initialise the stream. + const std::string& + command() const; + + /// Return a pointer to the stream buffer. + streambuf_type* + rdbuf() const; + +#if REDI_EVISCERATE_PSTREAMS + /// Obtain FILE pointers for each of the process' standard streams. + std::size_t + fopen(FILE*& in, FILE*& out, FILE*& err); +#endif + + protected: + std::string command_; ///< The command used to start the process. + streambuf_type buf_; ///< The stream buffer. + }; + + + /** + * @class basic_ipstream + * @brief Class template for Input PStreams. + * + * Reading from an ipstream reads the command's standard output and/or + * standard error (depending on how the ipstream is opened) + * and the command's standard input is the same as that of the process + * that created the object, unless altered by the command itself. + */ + + template > + class basic_ipstream + : public std::basic_istream + , public pstream_common + , virtual public pstreams + { + typedef std::basic_istream istream_type; + typedef pstream_common pbase_type; + + using pbase_type::buf_; // declare name in this scope + + // Ensure a basic_ipstream will read from at least one pipe + pmode readable(pmode mode) + { + if (!(mode & (pstdout|pstderr))) + mode |= pstdout; + return mode; + } + + public: + /// Type used to specify how to connect to the process. + typedef typename pbase_type::pmode pmode; + + /// Type used to hold the arguments for a command. + typedef typename pbase_type::argv_type argv_type; + + /// Default constructor, creates an uninitialised stream. + basic_ipstream() + : istream_type(NULL), pbase_type() + { } + + /** + * @brief Constructor that initialises the stream by starting a process. + * + * Initialises the stream buffer by calling do_open() with the supplied + * arguments. + * + * @param cmd a string containing a shell command. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, pmode) + */ + explicit + basic_ipstream(const std::string& cmd, pmode mode = pstdout) + : istream_type(NULL), pbase_type(cmd, readable(mode)) + { } + + /** + * @brief Constructor that initialises the stream by starting a process. + * + * Initialises the stream buffer by calling do_open() with the supplied + * arguments. + * + * @param file a string containing the pathname of a program to execute. + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + basic_ipstream( const std::string& file, + const argv_type& argv, + pmode mode = pstdout ) + : istream_type(NULL), pbase_type(file, argv, readable(mode)) + { } + + /** + * @brief Constructor that initialises the stream by starting a process. + * + * Initialises the stream buffer by calling + * @c do_open(argv[0],argv,mode|pstdout) + * + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + explicit + basic_ipstream(const argv_type& argv, pmode mode = pstdout) + : istream_type(NULL), pbase_type(argv.at(0), argv, readable(mode)) + { } + +#if __cplusplus >= 201103L + template + explicit + basic_ipstream(std::initializer_list args, pmode mode = pstdout) + : basic_ipstream(argv_type(args.begin(), args.end()), mode) + { } +#endif + + /** + * @brief Destructor. + * + * Closes the stream and waits for the child to exit. + */ + ~basic_ipstream() + { } + + /** + * @brief Start a process. + * + * Calls do_open( @a cmd , @a mode|pstdout ). + * + * @param cmd a string containing a shell command. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, pmode) + */ + void + open(const std::string& cmd, pmode mode = pstdout) + { + this->do_open(cmd, readable(mode)); + } + + /** + * @brief Start a process. + * + * Calls do_open( @a file , @a argv , @a mode|pstdout ). + * + * @param file a string containing the pathname of a program to execute. + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + void + open( const std::string& file, + const argv_type& argv, + pmode mode = pstdout ) + { + this->do_open(file, argv, readable(mode)); + } + + /** + * @brief Set streambuf to read from process' @c stdout. + * @return @c *this + */ + basic_ipstream& + out() + { + this->buf_.read_err(false); + return *this; + } + + /** + * @brief Set streambuf to read from process' @c stderr. + * @return @c *this + */ + basic_ipstream& + err() + { + this->buf_.read_err(true); + return *this; + } + }; + + + /** + * @class basic_opstream + * @brief Class template for Output PStreams. + * + * Writing to an open opstream writes to the standard input of the command; + * the command's standard output is the same as that of the process that + * created the pstream object, unless altered by the command itself. + */ + + template > + class basic_opstream + : public std::basic_ostream + , public pstream_common + , virtual public pstreams + { + typedef std::basic_ostream ostream_type; + typedef pstream_common pbase_type; + + using pbase_type::buf_; // declare name in this scope + + public: + /// Type used to specify how to connect to the process. + typedef typename pbase_type::pmode pmode; + + /// Type used to hold the arguments for a command. + typedef typename pbase_type::argv_type argv_type; + + /// Default constructor, creates an uninitialised stream. + basic_opstream() + : ostream_type(NULL), pbase_type() + { } + + /** + * @brief Constructor that initialises the stream by starting a process. + * + * Initialises the stream buffer by calling do_open() with the supplied + * arguments. + * + * @param cmd a string containing a shell command. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, pmode) + */ + explicit + basic_opstream(const std::string& cmd, pmode mode = pstdin) + : ostream_type(NULL), pbase_type(cmd, mode|pstdin) + { } + + /** + * @brief Constructor that initialises the stream by starting a process. + * + * Initialises the stream buffer by calling do_open() with the supplied + * arguments. + * + * @param file a string containing the pathname of a program to execute. + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + basic_opstream( const std::string& file, + const argv_type& argv, + pmode mode = pstdin ) + : ostream_type(NULL), pbase_type(file, argv, mode|pstdin) + { } + + /** + * @brief Constructor that initialises the stream by starting a process. + * + * Initialises the stream buffer by calling + * @c do_open(argv[0],argv,mode|pstdin) + * + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + explicit + basic_opstream(const argv_type& argv, pmode mode = pstdin) + : ostream_type(NULL), pbase_type(argv.at(0), argv, mode|pstdin) + { } + +#if __cplusplus >= 201103L + /** + * @brief Constructor that initialises the stream by starting a process. + * + * @param args a list of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + template + explicit + basic_opstream(std::initializer_list args, pmode mode = pstdin) + : basic_opstream(argv_type(args.begin(), args.end()), mode) + { } +#endif + + /** + * @brief Destructor + * + * Closes the stream and waits for the child to exit. + */ + ~basic_opstream() { } + + /** + * @brief Start a process. + * + * Calls do_open( @a cmd , @a mode|pstdin ). + * + * @param cmd a string containing a shell command. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, pmode) + */ + void + open(const std::string& cmd, pmode mode = pstdin) + { + this->do_open(cmd, mode|pstdin); + } + + /** + * @brief Start a process. + * + * Calls do_open( @a file , @a argv , @a mode|pstdin ). + * + * @param file a string containing the pathname of a program to execute. + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + void + open( const std::string& file, + const argv_type& argv, + pmode mode = pstdin) + { + this->do_open(file, argv, mode|pstdin); + } + }; + + + /** + * @class basic_pstream + * @brief Class template for Bidirectional PStreams. + * + * Writing to a pstream opened with @c pmode @c pstdin writes to the + * standard input of the command. + * Reading from a pstream opened with @c pmode @c pstdout and/or @c pstderr + * reads the command's standard output and/or standard error. + * Any of the process' @c stdin, @c stdout or @c stderr that is not + * connected to the pstream (as specified by the @c pmode) + * will be the same as the process that created the pstream object, + * unless altered by the command itself. + */ + template > + class basic_pstream + : public std::basic_iostream + , public pstream_common + , virtual public pstreams + { + typedef std::basic_iostream iostream_type; + typedef pstream_common pbase_type; + + using pbase_type::buf_; // declare name in this scope + + public: + /// Type used to specify how to connect to the process. + typedef typename pbase_type::pmode pmode; + + /// Type used to hold the arguments for a command. + typedef typename pbase_type::argv_type argv_type; + + /// Default constructor, creates an uninitialised stream. + basic_pstream() + : iostream_type(NULL), pbase_type() + { } + + /** + * @brief Constructor that initialises the stream by starting a process. + * + * Initialises the stream buffer by calling do_open() with the supplied + * arguments. + * + * @param cmd a string containing a shell command. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, pmode) + */ + explicit + basic_pstream(const std::string& cmd, pmode mode = pstdout|pstdin) + : iostream_type(NULL), pbase_type(cmd, mode) + { } + + /** + * @brief Constructor that initialises the stream by starting a process. + * + * Initialises the stream buffer by calling do_open() with the supplied + * arguments. + * + * @param file a string containing the pathname of a program to execute. + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + basic_pstream( const std::string& file, + const argv_type& argv, + pmode mode = pstdout|pstdin ) + : iostream_type(NULL), pbase_type(file, argv, mode) + { } + + /** + * @brief Constructor that initialises the stream by starting a process. + * + * Initialises the stream buffer by calling + * @c do_open(argv[0],argv,mode) + * + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + explicit + basic_pstream(const argv_type& argv, pmode mode = pstdout|pstdin) + : iostream_type(NULL), pbase_type(argv.at(0), argv, mode) + { } + +#if __cplusplus >= 201103L + /** + * @brief Constructor that initialises the stream by starting a process. + * + * @param l a list of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + template + explicit + basic_pstream(std::initializer_list l, pmode mode = pstdout|pstdin) + : basic_pstream(argv_type(l.begin(), l.end()), mode) + { } +#endif + + /** + * @brief Destructor + * + * Closes the stream and waits for the child to exit. + */ + ~basic_pstream() { } + + /** + * @brief Start a process. + * + * Calls do_open( @a cnd , @a mode ). + * + * @param cmd a string containing a shell command. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, pmode) + */ + void + open(const std::string& cmd, pmode mode = pstdout|pstdin) + { + this->do_open(cmd, mode); + } + + /** + * @brief Start a process. + * + * Calls do_open( @a file , @a argv , @a mode ). + * + * @param file a string containing the pathname of a program to execute. + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + void + open( const std::string& file, + const argv_type& argv, + pmode mode = pstdout|pstdin ) + { + this->do_open(file, argv, mode); + } + + /** + * @brief Set streambuf to read from process' @c stdout. + * @return @c *this + */ + basic_pstream& + out() + { + this->buf_.read_err(false); + return *this; + } + + /** + * @brief Set streambuf to read from process' @c stderr. + * @return @c *this + */ + basic_pstream& + err() + { + this->buf_.read_err(true); + return *this; + } + }; + + + /** + * @class basic_rpstream + * @brief Class template for Restricted PStreams. + * + * Writing to an rpstream opened with @c pmode @c pstdin writes to the + * standard input of the command. + * It is not possible to read directly from an rpstream object, to use + * an rpstream as in istream you must call either basic_rpstream::out() + * or basic_rpstream::err(). This is to prevent accidental reads from + * the wrong input source. If the rpstream was not opened with @c pmode + * @c pstderr then the class cannot read the process' @c stderr, and + * basic_rpstream::err() will return an istream that reads from the + * process' @c stdout, and vice versa. + * Reading from an rpstream opened with @c pmode @c pstdout and/or + * @c pstderr reads the command's standard output and/or standard error. + * Any of the process' @c stdin, @c stdout or @c stderr that is not + * connected to the pstream (as specified by the @c pmode) + * will be the same as the process that created the pstream object, + * unless altered by the command itself. + */ + + template > + class basic_rpstream + : public std::basic_ostream + , private std::basic_istream + , private pstream_common + , virtual public pstreams + { + typedef std::basic_ostream ostream_type; + typedef std::basic_istream istream_type; + typedef pstream_common pbase_type; + + using pbase_type::buf_; // declare name in this scope + + public: + /// Type used to specify how to connect to the process. + typedef typename pbase_type::pmode pmode; + + /// Type used to hold the arguments for a command. + typedef typename pbase_type::argv_type argv_type; + + /// Default constructor, creates an uninitialised stream. + basic_rpstream() + : ostream_type(NULL), istream_type(NULL), pbase_type() + { } + + /** + * @brief Constructor that initialises the stream by starting a process. + * + * Initialises the stream buffer by calling do_open() with the supplied + * arguments. + * + * @param cmd a string containing a shell command. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, pmode) + */ + explicit + basic_rpstream(const std::string& cmd, pmode mode = pstdout|pstdin) + : ostream_type(NULL) , istream_type(NULL) , pbase_type(cmd, mode) + { } + + /** + * @brief Constructor that initialises the stream by starting a process. + * + * Initialises the stream buffer by calling do_open() with the supplied + * arguments. + * + * @param file a string containing the pathname of a program to execute. + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + basic_rpstream( const std::string& file, + const argv_type& argv, + pmode mode = pstdout|pstdin ) + : ostream_type(NULL), istream_type(NULL), pbase_type(file, argv, mode) + { } + + /** + * @brief Constructor that initialises the stream by starting a process. + * + * Initialises the stream buffer by calling + * @c do_open(argv[0],argv,mode) + * + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + explicit + basic_rpstream(const argv_type& argv, pmode mode = pstdout|pstdin) + : ostream_type(NULL), istream_type(NULL), + pbase_type(argv.at(0), argv, mode) + { } + +#if __cplusplus >= 201103L + /** + * @brief Constructor that initialises the stream by starting a process. + * + * @param l a list of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + template + explicit + basic_rpstream(std::initializer_list l, pmode mode = pstdout|pstdin) + : basic_rpstream(argv_type(l.begin(), l.end()), mode) + { } +#endif + + /// Destructor + ~basic_rpstream() { } + + /** + * @brief Start a process. + * + * Calls do_open( @a cmd , @a mode ). + * + * @param cmd a string containing a shell command. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, pmode) + */ + void + open(const std::string& cmd, pmode mode = pstdout|pstdin) + { + this->do_open(cmd, mode); + } + + /** + * @brief Start a process. + * + * Calls do_open( @a file , @a argv , @a mode ). + * + * @param file a string containing the pathname of a program to execute. + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + void + open( const std::string& file, + const argv_type& argv, + pmode mode = pstdout|pstdin ) + { + this->do_open(file, argv, mode); + } + + /** + * @brief Obtain a reference to the istream that reads + * the process' @c stdout. + * @return @c *this + */ + istream_type& + out() + { + this->buf_.read_err(false); + return *this; + } + + /** + * @brief Obtain a reference to the istream that reads + * the process' @c stderr. + * @return @c *this + */ + istream_type& + err() + { + this->buf_.read_err(true); + return *this; + } + }; + + + /// Type definition for common template specialisation. + typedef basic_pstreambuf pstreambuf; + /// Type definition for common template specialisation. + typedef basic_ipstream ipstream; + /// Type definition for common template specialisation. + typedef basic_opstream opstream; + /// Type definition for common template specialisation. + typedef basic_pstream pstream; + /// Type definition for common template specialisation. + typedef basic_rpstream rpstream; + + + /** + * When inserted into an output pstream the manipulator calls + * basic_pstreambuf::peof() to close the output pipe, + * causing the child process to receive the end-of-file indicator + * on subsequent reads from its @c stdin stream. + * + * @brief Manipulator to close the pipe connected to the process' stdin. + * @param s An output PStream class. + * @return The stream object the manipulator was invoked on. + * @see basic_pstreambuf::peof() + * @relates basic_opstream basic_pstream basic_rpstream + */ + template + inline std::basic_ostream& + peof(std::basic_ostream& s) + { + typedef basic_pstreambuf pstreambuf_type; + if (pstreambuf_type* p = dynamic_cast(s.rdbuf())) + p->peof(); + return s; + } + + + /* + * member definitions for pstreambuf + */ + + + /** + * @class basic_pstreambuf + * Provides underlying streambuf functionality for the PStreams classes. + */ + + /** Creates an uninitialised stream buffer. */ + template + inline + basic_pstreambuf::basic_pstreambuf() + : ppid_(-1) // initialise to -1 to indicate no process run yet. + , wpipe_(-1) + , wbuffer_(NULL) + , rsrc_(rsrc_out) + , status_(-1) + , error_(0) + { + init_rbuffers(); + } + + /** + * Initialises the stream buffer by calling open() with the supplied + * arguments. + * + * @param cmd a string containing a shell command. + * @param mode the I/O mode to use when opening the pipe. + * @see open() + */ + template + inline + basic_pstreambuf::basic_pstreambuf(const std::string& cmd, pmode mode) + : ppid_(-1) // initialise to -1 to indicate no process run yet. + , wpipe_(-1) + , wbuffer_(NULL) + , rsrc_(rsrc_out) + , status_(-1) + , error_(0) + { + init_rbuffers(); + open(cmd, mode); + } + + /** + * Initialises the stream buffer by calling open() with the supplied + * arguments. + * + * @param file a string containing the name of a program to execute. + * @param argv a vector of argument strings passsed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see open() + */ + template + inline + basic_pstreambuf::basic_pstreambuf( const std::string& file, + const argv_type& argv, + pmode mode ) + : ppid_(-1) // initialise to -1 to indicate no process run yet. + , wpipe_(-1) + , wbuffer_(NULL) + , rsrc_(rsrc_out) + , status_(-1) + , error_(0) + { + init_rbuffers(); + open(file, argv, mode); + } + + /** + * Closes the stream by calling close(). + * @see close() + */ + template + inline + basic_pstreambuf::~basic_pstreambuf() + { + close(); + } + + /** + * Starts a new process by passing @a command to the shell (/bin/sh) + * and opens pipes to the process with the specified @a mode. + * + * If @a mode contains @c pstdout the initial read source will be + * the child process' stdout, otherwise if @a mode contains @c pstderr + * the initial read source will be the child's stderr. + * + * Will duplicate the actions of the shell in searching for an + * executable file if the specified file name does not contain a slash (/) + * character. + * + * @warning + * There is no way to tell whether the shell command succeeded, this + * function will always succeed unless resource limits (such as + * memory usage, or number of processes or open files) are exceeded. + * This means is_open() will return true even if @a command cannot + * be executed. + * Use pstreambuf::open(const std::string&, const argv_type&, pmode) + * if you need to know whether the command failed to execute. + * + * @param command a string containing a shell command. + * @param mode a bitwise OR of one or more of @c out, @c in, @c err. + * @return NULL if the shell could not be started or the + * pipes could not be opened, @c this otherwise. + * @see execl(3) + */ + template + basic_pstreambuf* + basic_pstreambuf::open(const std::string& command, pmode mode) + { + const char * shell_path = "/bin/sh"; +#if 0 + const std::string argv[] = { "sh", "-c", command }; + return this->open(shell_path, argv_type(argv, argv+3), mode); +#else + basic_pstreambuf* ret = NULL; + + if (!is_open()) + { + switch(fork(mode)) + { + case 0 : + // this is the new process, exec command + ::execl(shell_path, "sh", "-c", command.c_str(), (char*)NULL); + + // can only reach this point if exec() failed + + // parent can get exit code from waitpid() + ::_exit(errno); + // using std::exit() would make static dtors run twice + + case -1 : + // couldn't fork, error already handled in pstreambuf::fork() + break; + + default : + // this is the parent process + // activate buffers + create_buffers(mode); + ret = this; + } + } + return ret; +#endif + } + + /** + * @brief Helper function to close a file descriptor. + * + * Inspects @a fd and calls close(3) if it has a non-negative value. + * + * @param fd a file descriptor. + * @relates basic_pstreambuf + */ + inline void + close_fd(pstreams::fd_type& fd) + { + if (fd >= 0 && ::close(fd) == 0) + fd = -1; + } + + /** + * @brief Helper function to close an array of file descriptors. + * + * Calls @c close_fd() on each member of the array. + * The length of the array is determined automatically by + * template argument deduction to avoid errors. + * + * @param fds an array of file descriptors. + * @relates basic_pstreambuf + */ + template + inline void + close_fd_array(pstreams::fd_type (&fds)[N]) + { + for (std::size_t i = 0; i < N; ++i) + close_fd(fds[i]); + } + + /** + * Starts a new process by executing @a file with the arguments in + * @a argv and opens pipes to the process with the specified @a mode. + * + * By convention @c argv[0] should be the file name of the file being + * executed. + * + * If @a mode contains @c pstdout the initial read source will be + * the child process' stdout, otherwise if @a mode contains @c pstderr + * the initial read source will be the child's stderr. + * + * Will duplicate the actions of the shell in searching for an + * executable file if the specified file name does not contain a slash (/) + * character. + * + * Iff @a file is successfully executed then is_open() will return true. + * Otherwise, pstreambuf::error() can be used to obtain the value of + * @c errno that was set by execvp(3) in the child process. + * + * The exit status of the new process will be returned by + * pstreambuf::status() after pstreambuf::exited() returns true. + * + * @param file a string containing the pathname of a program to execute. + * @param argv a vector of argument strings passed to the new program. + * @param mode a bitwise OR of one or more of @c out, @c in and @c err. + * @return NULL if a pipe could not be opened or if the program could + * not be executed, @c this otherwise. + * @see execvp(3) + */ + template + basic_pstreambuf* + basic_pstreambuf::open( const std::string& file, + const argv_type& argv, + pmode mode ) + { + basic_pstreambuf* ret = NULL; + + if (!is_open()) + { + // constants for read/write ends of pipe + enum { RD, WR }; + + // open another pipe and set close-on-exec + fd_type ck_exec[] = { -1, -1 }; + if (-1 == ::pipe(ck_exec) + || -1 == ::fcntl(ck_exec[RD], F_SETFD, FD_CLOEXEC) + || -1 == ::fcntl(ck_exec[WR], F_SETFD, FD_CLOEXEC)) + { + error_ = errno; + close_fd_array(ck_exec); + } + else + { + switch(fork(mode)) + { + case 0 : + // this is the new process, exec command + { + char** arg_v = new char*[argv.size()+1]; + for (std::size_t i = 0; i < argv.size(); ++i) + { + const std::string& src = argv[i]; + char*& dest = arg_v[i]; + dest = new char[src.size()+1]; + dest[ src.copy(dest, src.size()) ] = '\0'; + } + arg_v[argv.size()] = NULL; + + ::execvp(file.c_str(), arg_v); + + // can only reach this point if exec() failed + + // parent can get error code from ck_exec pipe + error_ = errno; + + while (::write(ck_exec[WR], &error_, sizeof(error_)) == -1 + && errno == EINTR) + { } + + ::close(ck_exec[WR]); + ::close(ck_exec[RD]); + + ::_exit(error_); + // using std::exit() would make static dtors run twice + } + + case -1 : + // couldn't fork, error already handled in pstreambuf::fork() + close_fd_array(ck_exec); + break; + + default : + // this is the parent process + + // check child called exec() successfully + ::close(ck_exec[WR]); + switch (::read(ck_exec[RD], &error_, sizeof(error_))) + { + case 0: + // activate buffers + create_buffers(mode); + ret = this; + break; + case -1: + error_ = errno; + break; + default: + // error_ contains error code from child + // call wait() to clean up and set ppid_ to 0 + this->wait(); + break; + } + ::close(ck_exec[RD]); + } + } + } + return ret; + } + + /** + * Creates pipes as specified by @a mode and calls @c fork() to create + * a new process. If the fork is successful the parent process stores + * the child's PID and the opened pipes and the child process replaces + * its standard streams with the opened pipes. + * + * If an error occurs the error code will be set to one of the possible + * errors for @c pipe() or @c fork(). + * See your system's documentation for these error codes. + * + * @param mode an OR of pmodes specifying which of the child's + * standard streams to connect to. + * @return On success the PID of the child is returned in the parent's + * context and zero is returned in the child's context. + * On error -1 is returned and the error code is set appropriately. + */ + template + pid_t + basic_pstreambuf::fork(pmode mode) + { + pid_t pid = -1; + + // Three pairs of file descriptors, for pipes connected to the + // process' stdin, stdout and stderr + // (stored in a single array so close_fd_array() can close all at once) + fd_type fd[] = { -1, -1, -1, -1, -1, -1 }; + fd_type* const pin = fd; + fd_type* const pout = fd+2; + fd_type* const perr = fd+4; + + // constants for read/write ends of pipe + enum { RD, WR }; + + // N.B. + // For the pstreambuf pin is an output stream and + // pout and perr are input streams. + + if (!error_ && mode&pstdin && ::pipe(pin)) + error_ = errno; + + if (!error_ && mode&pstdout && ::pipe(pout)) + error_ = errno; + + if (!error_ && mode&pstderr && ::pipe(perr)) + error_ = errno; + + if (!error_) + { + pid = ::fork(); + switch (pid) + { + case 0 : + { + // this is the new process + + // for each open pipe close one end and redirect the + // respective standard stream to the other end + + if (*pin >= 0) + { + ::close(pin[WR]); + ::dup2(pin[RD], STDIN_FILENO); + ::close(pin[RD]); + } + if (*pout >= 0) + { + ::close(pout[RD]); + ::dup2(pout[WR], STDOUT_FILENO); + ::close(pout[WR]); + } + if (*perr >= 0) + { + ::close(perr[RD]); + ::dup2(perr[WR], STDERR_FILENO); + ::close(perr[WR]); + } + +#ifdef _POSIX_JOB_CONTROL + if (mode&newpg) + ::setpgid(0, 0); // Change to a new process group +#endif + + break; + } + case -1 : + { + // couldn't fork for some reason + error_ = errno; + // close any open pipes + close_fd_array(fd); + break; + } + default : + { + // this is the parent process, store process' pid + ppid_ = pid; + + // store one end of open pipes and close other end + if (*pin >= 0) + { + wpipe_ = pin[WR]; + ::close(pin[RD]); + } + if (*pout >= 0) + { + rpipe_[rsrc_out] = pout[RD]; + ::close(pout[WR]); + } + if (*perr >= 0) + { + rpipe_[rsrc_err] = perr[RD]; + ::close(perr[WR]); + } + } + } + } + else + { + // close any pipes we opened before failure + close_fd_array(fd); + } + return pid; + } + + /** + * Closes all pipes and calls wait() to wait for the process to finish. + * If an error occurs the error code will be set to one of the possible + * errors for @c waitpid(). + * See your system's documentation for these errors. + * + * @return @c this on successful close or @c NULL if there is no + * process to close or if an error occurs. + */ + template + basic_pstreambuf* + basic_pstreambuf::close() + { + const bool running = is_open(); + + sync(); // this might call wait() and reap the child process + + // rather than trying to work out whether or not we need to clean up + // just do it anyway, all cleanup functions are safe to call twice. + + destroy_buffers(pstdin|pstdout|pstderr); + + // close pipes before wait() so child gets EOF/SIGPIPE + close_fd(wpipe_); + close_fd_array(rpipe_); + + do + { + error_ = 0; + } while (wait() == -1 && error() == EINTR); + + return running ? this : NULL; + } + + /** + * Called on construction to initialise the arrays used for reading. + */ + template + inline void + basic_pstreambuf::init_rbuffers() + { + rpipe_[rsrc_out] = rpipe_[rsrc_err] = -1; + rbuffer_[rsrc_out] = rbuffer_[rsrc_err] = NULL; + rbufstate_[0] = rbufstate_[1] = rbufstate_[2] = NULL; + } + + template + void + basic_pstreambuf::create_buffers(pmode mode) + { + if (mode & pstdin) + { + delete[] wbuffer_; + wbuffer_ = new char_type[bufsz]; + this->setp(wbuffer_, wbuffer_ + bufsz); + } + if (mode & pstdout) + { + delete[] rbuffer_[rsrc_out]; + rbuffer_[rsrc_out] = new char_type[bufsz]; + rsrc_ = rsrc_out; + this->setg(rbuffer_[rsrc_out] + pbsz, rbuffer_[rsrc_out] + pbsz, + rbuffer_[rsrc_out] + pbsz); + } + if (mode & pstderr) + { + delete[] rbuffer_[rsrc_err]; + rbuffer_[rsrc_err] = new char_type[bufsz]; + if (!(mode & pstdout)) + { + rsrc_ = rsrc_err; + this->setg(rbuffer_[rsrc_err] + pbsz, rbuffer_[rsrc_err] + pbsz, + rbuffer_[rsrc_err] + pbsz); + } + } + } + + template + void + basic_pstreambuf::destroy_buffers(pmode mode) + { + if (mode & pstdin) + { + this->setp(NULL, NULL); + delete[] wbuffer_; + wbuffer_ = NULL; + } + if (mode & pstdout) + { + if (rsrc_ == rsrc_out) + this->setg(NULL, NULL, NULL); + delete[] rbuffer_[rsrc_out]; + rbuffer_[rsrc_out] = NULL; + } + if (mode & pstderr) + { + if (rsrc_ == rsrc_err) + this->setg(NULL, NULL, NULL); + delete[] rbuffer_[rsrc_err]; + rbuffer_[rsrc_err] = NULL; + } + } + + template + typename basic_pstreambuf::buf_read_src + basic_pstreambuf::switch_read_buffer(buf_read_src src) + { + if (rsrc_ != src) + { + char_type* tmpbufstate[] = {this->eback(), this->gptr(), this->egptr()}; + this->setg(rbufstate_[0], rbufstate_[1], rbufstate_[2]); + for (std::size_t i = 0; i < 3; ++i) + rbufstate_[i] = tmpbufstate[i]; + rsrc_ = src; + } + return rsrc_; + } + + /** + * Suspends execution and waits for the associated process to exit, or + * until a signal is delivered whose action is to terminate the current + * process or to call a signal handling function. If the process has + * already exited (i.e. it is a "zombie" process) then wait() returns + * immediately. Waiting for the child process causes all its system + * resources to be freed. + * + * error() will return EINTR if wait() is interrupted by a signal. + * + * @param nohang true to return immediately if the process has not exited. + * @return 1 if the process has exited and wait() has not yet been called. + * 0 if @a nohang is true and the process has not exited yet. + * -1 if no process has been started or if an error occurs, + * in which case the error can be found using error(). + */ + template + int + basic_pstreambuf::wait(bool nohang) + { + int child_exited = -1; + if (is_open()) + { + int exit_status; + switch(::waitpid(ppid_, &exit_status, nohang ? WNOHANG : 0)) + { + case 0 : + // nohang was true and process has not exited + child_exited = 0; + break; + case -1 : + error_ = errno; + break; + default : + // process has exited + ppid_ = 0; + status_ = exit_status; + child_exited = 1; + // Close wpipe, would get SIGPIPE if we used it. + destroy_buffers(pstdin); + close_fd(wpipe_); + // Must free read buffers and pipes on destruction + // or next call to open()/close() + break; + } + } + return child_exited; + } + + /** + * Sends the specified signal to the process. A signal can be used to + * terminate a child process that would not exit otherwise. + * + * If an error occurs the error code will be set to one of the possible + * errors for @c kill(). See your system's documentation for these errors. + * + * @param signal A signal to send to the child process. + * @return @c this or @c NULL if @c kill() fails. + */ + template + inline basic_pstreambuf* + basic_pstreambuf::kill(int signal) + { + basic_pstreambuf* ret = NULL; + if (is_open()) + { + if (::kill(ppid_, signal)) + error_ = errno; + else + { +#if 0 + // TODO call exited() to check for exit and clean up? leave to user? + if (signal==SIGTERM || signal==SIGKILL) + this->exited(); +#endif + ret = this; + } + } + return ret; + } + + /** + * Sends the specified signal to the process group of the child process. + * A signal can be used to terminate a child process that would not exit + * otherwise, or to kill the process and its own children. + * + * If an error occurs the error code will be set to one of the possible + * errors for @c getpgid() or @c kill(). See your system's documentation + * for these errors. If the child is in the current process group then + * NULL will be returned and the error code set to EPERM. + * + * @param signal A signal to send to the child process. + * @return @c this on success or @c NULL on failure. + */ + template + inline basic_pstreambuf* + basic_pstreambuf::killpg(int signal) + { + basic_pstreambuf* ret = NULL; +#ifdef _POSIX_JOB_CONTROL + if (is_open()) + { + pid_t pgid = ::getpgid(ppid_); + if (pgid == -1) + error_ = errno; + else if (pgid == ::getpgrp()) + error_ = EPERM; // Don't commit suicide + else if (::killpg(pgid, signal)) + error_ = errno; + else + ret = this; + } +#else + error_ = ENOTSUP; +#endif + return ret; + } + + /** + * This function can call pstreambuf::wait() and so may change the + * object's state if the child process has already exited. + * + * @return True if the associated process has exited, false otherwise. + * @see basic_pstreambuf::wait() + */ + template + inline bool + basic_pstreambuf::exited() + { + return ppid_ == 0 || wait(true)==1; + } + + + /** + * @return The exit status of the child process, or -1 if wait() + * has not yet been called to wait for the child to exit. + * @see basic_pstreambuf::wait() + */ + template + inline int + basic_pstreambuf::status() const + { + return status_; + } + + /** + * @return The error code of the most recently failed operation, or zero. + */ + template + inline int + basic_pstreambuf::error() const + { + return error_; + } + + /** + * Closes the output pipe, causing the child process to receive the + * end-of-file indicator on subsequent reads from its @c stdin stream. + */ + template + inline void + basic_pstreambuf::peof() + { + sync(); + destroy_buffers(pstdin); + close_fd(wpipe_); + } + + /** + * Unlike pstreambuf::exited(), this function will not call wait() and + * so will not change the object's state. This means that once a child + * process is executed successfully this function will continue to + * return true even after the process exits (until wait() is called.) + * + * @return true if a previous call to open() succeeded and wait() has + * not been called and determined that the process has exited, + * false otherwise. + */ + template + inline bool + basic_pstreambuf::is_open() const + { + return ppid_ > 0; + } + + /** + * Toggle the stream used for reading. If @a readerr is @c true then the + * process' @c stderr output will be used for subsequent extractions, if + * @a readerr is false the the process' stdout will be used. + * @param readerr @c true to read @c stderr, @c false to read @c stdout. + * @return @c true if the requested stream is open and will be used for + * subsequent extractions, @c false otherwise. + */ + template + inline bool + basic_pstreambuf::read_err(bool readerr) + { + buf_read_src src = readerr ? rsrc_err : rsrc_out; + if (rpipe_[src]>=0) + { + switch_read_buffer(src); + return true; + } + return false; + } + + /** + * Called when the internal character buffer is not present or is full, + * to transfer the buffer contents to the pipe. + * + * @param c a character to be written to the pipe. + * @return @c traits_type::eof() if an error occurs, otherwise if @a c + * is not equal to @c traits_type::eof() it will be buffered and + * a value other than @c traits_type::eof() returned to indicate + * success. + */ + template + typename basic_pstreambuf::int_type + basic_pstreambuf::overflow(int_type c) + { + if (!empty_buffer()) + return traits_type::eof(); + else if (!traits_type::eq_int_type(c, traits_type::eof())) + return this->sputc(c); + else + return traits_type::not_eof(c); + } + + + template + int + basic_pstreambuf::sync() + { + return !exited() && empty_buffer() ? 0 : -1; + } + + /** + * @param s character buffer. + * @param n buffer length. + * @return the number of characters written. + */ + template + std::streamsize + basic_pstreambuf::xsputn(const char_type* s, std::streamsize n) + { + std::streamsize done = 0; + while (done < n) + { + if (std::streamsize nbuf = this->epptr() - this->pptr()) + { + nbuf = std::min(nbuf, n - done); + traits_type::copy(this->pptr(), s + done, nbuf); + this->pbump(nbuf); + done += nbuf; + } + else if (!empty_buffer()) + break; + } + return done; + } + + /** + * @return true if the buffer was emptied, false otherwise. + */ + template + bool + basic_pstreambuf::empty_buffer() + { + const std::streamsize count = this->pptr() - this->pbase(); + if (count > 0) + { + const std::streamsize written = this->write(this->wbuffer_, count); + if (written > 0) + { + if (const std::streamsize unwritten = count - written) + traits_type::move(this->pbase(), this->pbase()+written, unwritten); + this->pbump(-written); + return true; + } + } + return false; + } + + /** + * Called when the internal character buffer is is empty, to re-fill it + * from the pipe. + * + * @return The first available character in the buffer, + * or @c traits_type::eof() in case of failure. + */ + template + typename basic_pstreambuf::int_type + basic_pstreambuf::underflow() + { + if (this->gptr() < this->egptr() || fill_buffer()) + return traits_type::to_int_type(*this->gptr()); + else + return traits_type::eof(); + } + + /** + * Attempts to make @a c available as the next character to be read by + * @c sgetc(). + * + * @param c a character to make available for extraction. + * @return @a c if the character can be made available, + * @c traits_type::eof() otherwise. + */ + template + typename basic_pstreambuf::int_type + basic_pstreambuf::pbackfail(int_type c) + { + if (this->gptr() != this->eback()) + { + this->gbump(-1); + if (!traits_type::eq_int_type(c, traits_type::eof())) + *this->gptr() = traits_type::to_char_type(c); + return traits_type::not_eof(c); + } + else + return traits_type::eof(); + } + + template + std::streamsize + basic_pstreambuf::showmanyc() + { + int avail = 0; + if (sizeof(char_type) == 1) + avail = fill_buffer(true) ? this->egptr() - this->gptr() : -1; +#ifdef FIONREAD + else + { + if (::ioctl(rpipe(), FIONREAD, &avail) == -1) + avail = -1; + else if (avail) + avail /= sizeof(char_type); + } +#endif + return std::streamsize(avail); + } + + /** + * @return true if the buffer was filled, false otherwise. + */ + template + bool + basic_pstreambuf::fill_buffer(bool non_blocking) + { + const std::streamsize pb1 = this->gptr() - this->eback(); + const std::streamsize pb2 = pbsz; + const std::streamsize npb = std::min(pb1, pb2); + + char_type* const rbuf = rbuffer(); + + if (npb) + traits_type::move(rbuf + pbsz - npb, this->gptr() - npb, npb); + + std::streamsize rc = -1; + + if (non_blocking) + { + const int flags = ::fcntl(rpipe(), F_GETFL); + if (flags != -1) + { + const bool blocking = !(flags & O_NONBLOCK); + if (blocking) + ::fcntl(rpipe(), F_SETFL, flags | O_NONBLOCK); // set non-blocking + + error_ = 0; + rc = read(rbuf + pbsz, bufsz - pbsz); + + if (rc == -1 && error_ == EAGAIN) // nothing available + rc = 0; + else if (rc == 0) // EOF + rc = -1; + + if (blocking) + ::fcntl(rpipe(), F_SETFL, flags); // restore + } + } + else + rc = read(rbuf + pbsz, bufsz - pbsz); + + if (rc > 0 || (rc == 0 && non_blocking)) + { + this->setg( rbuf + pbsz - npb, + rbuf + pbsz, + rbuf + pbsz + rc ); + return true; + } + else + { + this->setg(NULL, NULL, NULL); + return false; + } + } + + /** + * Writes up to @a n characters to the pipe from the buffer @a s. + * + * @param s character buffer. + * @param n buffer length. + * @return the number of characters written. + */ + template + inline std::streamsize + basic_pstreambuf::write(const char_type* s, std::streamsize n) + { + std::streamsize nwritten = 0; + if (wpipe() >= 0) + { + nwritten = ::write(wpipe(), s, n * sizeof(char_type)); + if (nwritten == -1) + error_ = errno; + else + nwritten /= sizeof(char_type); + } + return nwritten; + } + + /** + * Reads up to @a n characters from the pipe to the buffer @a s. + * + * @param s character buffer. + * @param n buffer length. + * @return the number of characters read. + */ + template + inline std::streamsize + basic_pstreambuf::read(char_type* s, std::streamsize n) + { + std::streamsize nread = 0; + if (rpipe() >= 0) + { + nread = ::read(rpipe(), s, n * sizeof(char_type)); + if (nread == -1) + error_ = errno; + else + nread /= sizeof(char_type); + } + return nread; + } + + /** @return a reference to the output file descriptor */ + template + inline pstreams::fd_type& + basic_pstreambuf::wpipe() + { + return wpipe_; + } + + /** @return a reference to the active input file descriptor */ + template + inline pstreams::fd_type& + basic_pstreambuf::rpipe() + { + return rpipe_[rsrc_]; + } + + /** @return a reference to the specified input file descriptor */ + template + inline pstreams::fd_type& + basic_pstreambuf::rpipe(buf_read_src which) + { + return rpipe_[which]; + } + + /** @return a pointer to the start of the active input buffer area. */ + template + inline typename basic_pstreambuf::char_type* + basic_pstreambuf::rbuffer() + { + return rbuffer_[rsrc_]; + } + + + /* + * member definitions for pstream_common + */ + + /** + * @class pstream_common + * Abstract Base Class providing common functionality for basic_ipstream, + * basic_opstream and basic_pstream. + * pstream_common manages the basic_pstreambuf stream buffer that is used + * by the derived classes to initialise an iostream class. + */ + + /** Creates an uninitialised stream. */ + template + inline + pstream_common::pstream_common() + : std::basic_ios(NULL) + , command_() + , buf_() + { + this->std::basic_ios::rdbuf(&buf_); + } + + /** + * Initialises the stream buffer by calling + * do_open( @a command , @a mode ) + * + * @param cmd a string containing a shell command. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, pmode) + */ + template + inline + pstream_common::pstream_common(const std::string& cmd, pmode mode) + : std::basic_ios(NULL) + , command_(cmd) + , buf_() + { + this->std::basic_ios::rdbuf(&buf_); + do_open(cmd, mode); + } + + /** + * Initialises the stream buffer by calling + * do_open( @a file , @a argv , @a mode ) + * + * @param file a string containing the pathname of a program to execute. + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see do_open(const std::string&, const argv_type&, pmode) + */ + template + inline + pstream_common::pstream_common( const std::string& file, + const argv_type& argv, + pmode mode ) + : std::basic_ios(NULL) + , command_(file) + , buf_() + { + this->std::basic_ios::rdbuf(&buf_); + do_open(file, argv, mode); + } + + /** + * This is a pure virtual function to make @c pstream_common abstract. + * Because it is the destructor it will be called by derived classes + * and so must be defined. It is also protected, to discourage use of + * the PStreams classes through pointers or references to the base class. + * + * @sa If defining a pure virtual seems odd you should read + * http://www.gotw.ca/gotw/031.htm (and the rest of the site as well!) + */ + template + inline + pstream_common::~pstream_common() + { + } + + /** + * Calls rdbuf()->open( @a command , @a mode ) + * and sets @c failbit on error. + * + * @param cmd a string containing a shell command. + * @param mode the I/O mode to use when opening the pipe. + * @see basic_pstreambuf::open(const std::string&, pmode) + */ + template + inline void + pstream_common::do_open(const std::string& cmd, pmode mode) + { + if (!buf_.open((command_=cmd), mode)) + this->setstate(std::ios_base::failbit); + } + + /** + * Calls rdbuf()->open( @a file, @a argv, @a mode ) + * and sets @c failbit on error. + * + * @param file a string containing the pathname of a program to execute. + * @param argv a vector of argument strings passed to the new program. + * @param mode the I/O mode to use when opening the pipe. + * @see basic_pstreambuf::open(const std::string&, const argv_type&, pmode) + */ + template + inline void + pstream_common::do_open( const std::string& file, + const argv_type& argv, + pmode mode ) + { + if (!buf_.open((command_=file), argv, mode)) + this->setstate(std::ios_base::failbit); + } + + /** Calls rdbuf->close() and sets @c failbit on error. */ + template + inline void + pstream_common::close() + { + if (!buf_.close()) + this->setstate(std::ios_base::failbit); + } + + /** + * @return rdbuf()->is_open(). + * @see basic_pstreambuf::is_open() + */ + template + inline bool + pstream_common::is_open() const + { + return buf_.is_open(); + } + + /** @return a string containing the command used to initialise the stream. */ + template + inline const std::string& + pstream_common::command() const + { + return command_; + } + + /** @return a pointer to the private stream buffer member. */ + // TODO document behaviour if buffer replaced. + template + inline typename pstream_common::streambuf_type* + pstream_common::rdbuf() const + { + return const_cast(&buf_); + } + + +#if REDI_EVISCERATE_PSTREAMS + /** + * @def REDI_EVISCERATE_PSTREAMS + * If this macro has a non-zero value then certain internals of the + * @c basic_pstreambuf template class are exposed. In general this is + * a Bad Thing, as the internal implementation is largely undocumented + * and may be subject to change at any time, so this feature is only + * provided because it might make PStreams useful in situations where + * it is necessary to do Bad Things. + */ + + /** + * @warning This function exposes the internals of the stream buffer and + * should be used with caution. It is the caller's responsibility + * to flush streams etc. in order to clear any buffered data. + * The POSIX.1 function fdopen(3) is used to obtain the + * @c FILE pointers from the streambuf's private file descriptor + * members so consult your system's documentation for + * fdopen(3). + * + * @param in A FILE* that will refer to the process' stdin. + * @param out A FILE* that will refer to the process' stdout. + * @param err A FILE* that will refer to the process' stderr. + * @return An OR of zero or more of @c pstdin, @c pstdout, @c pstderr. + * + * For each open stream shared with the child process a @c FILE* is + * obtained and assigned to the corresponding parameter. For closed + * streams @c NULL is assigned to the parameter. + * The return value can be tested to see which parameters should be + * @c !NULL by masking with the corresponding @c pmode value. + * + * @see fdopen(3) + */ + template + std::size_t + basic_pstreambuf::fopen(FILE*& in, FILE*& out, FILE*& err) + { + in = out = err = NULL; + std::size_t open_files = 0; + if (wpipe() > -1) + { + if ((in = ::fdopen(wpipe(), "w"))) + { + open_files |= pstdin; + } + } + if (rpipe(rsrc_out) > -1) + { + if ((out = ::fdopen(rpipe(rsrc_out), "r"))) + { + open_files |= pstdout; + } + } + if (rpipe(rsrc_err) > -1) + { + if ((err = ::fdopen(rpipe(rsrc_err), "r"))) + { + open_files |= pstderr; + } + } + return open_files; + } + + /** + * @warning This function exposes the internals of the stream buffer and + * should be used with caution. + * + * @param in A FILE* that will refer to the process' stdin. + * @param out A FILE* that will refer to the process' stdout. + * @param err A FILE* that will refer to the process' stderr. + * @return A bitwise-or of zero or more of @c pstdin, @c pstdout, @c pstderr. + * @see basic_pstreambuf::fopen() + */ + template + inline std::size_t + pstream_common::fopen(FILE*& fin, FILE*& fout, FILE*& ferr) + { + return buf_.fopen(fin, fout, ferr); + } + +#endif // REDI_EVISCERATE_PSTREAMS + + +} // namespace redi + +/** + * @mainpage PStreams Reference + * @htmlinclude mainpage.html + */ + +#endif // REDI_PSTREAM_H_SEEN + +// vim: ts=2 sw=2 expandtab + diff --git a/projects/hip/bin/hipcc b/projects/hip/bin/hipcc index b991bd40aa..6565eaacb8 100755 --- a/projects/hip/bin/hipcc +++ b/projects/hip/bin/hipcc @@ -93,8 +93,6 @@ if ($HIP_PLATFORM eq "hcc") { $HIP_ATP_MARKER=$ENV{'HIP_ATP_MARKER'} // 1; $marker_path = "$ROCM_PATH/profiler/CXLActivityLogger"; - $ROCM_TARGET=$ENV{'ROCM_TARGET'} // "gfx803"; - # HCC* may be used to compile src/hip_hcc.o (and also feed the HIPCXXFLAGS below) $HCC = "$HCC_HOME/bin/hcc"; $HCCFLAGS = "-hc -D__HIPCC__ -I$HCC_HOME/include "; @@ -128,7 +126,7 @@ if ($HIP_PLATFORM eq "hcc") { # Force -stdlib=libc++ on UB14.04 $HOST_OSVER= `cat /etc/os-release | grep "^VERSION_ID\=" | cut -d= -f2 | tr -d '\n'`; if (($HOST_OSNAME eq "ubuntu" and $HOST_OSVER eq "\"14.04\"") - or ($HOST_OSNAME eq "\"centos\"" and $HOST_OSVER eq "\"7\"") + or ($HOST_OSNAME eq "\"centos\"" and $HOST_OSVER eq "\"7\"") or ($HOST_OSNAME eq "\"rhel\"" and $HOST_OSVER eq "\"7.4\"")) { $HIPCXXFLAGS .= " -stdlib=libc++"; $setStdLib = 1; @@ -202,7 +200,7 @@ if ($verbose & 0x4) { # Handle code object generation my $ISACMD=""; if($HIP_PLATFORM eq "hcc"){ - $ISACMD .= "set ROCM_PATH=$ROCM_PATH && set ROCM_TARGET=$ROCM_TARGET && $HIP_PATH/bin/hccgenco.sh "; + $ISACMD .= "$HIP_PATH/bin/lpl "; if($ARGV[0] eq "--genco"){ foreach $isaarg (@ARGV[1..$#ARGV]){ $ISACMD .= " "; diff --git a/projects/hip/include/hip/hcc_detail/code_object_bundle.hpp b/projects/hip/include/hip/hcc_detail/code_object_bundle.hpp index 72f9d35c73..81ecb16409 100644 --- a/projects/hip/include/hip/hcc_detail/code_object_bundle.hpp +++ b/projects/hip/include/hip/hcc_detail/code_object_bundle.hpp @@ -37,17 +37,16 @@ namespace hip_impl hsa_isa_t triple_to_hsa_isa(const std::string& triple); struct Bundled_code { - union { + union Header { struct { std::uint64_t offset; std::uint64_t bundle_sz; std::uint64_t triple_sz; }; - std::uint8_t cbuf[ - sizeof(offset) + sizeof(bundle_sz) + sizeof(triple_sz)]; - }; + char cbuf[sizeof(offset) + sizeof(bundle_sz) + sizeof(triple_sz)]; + } header; std::string triple; - std::vector blob; + std::vector blob; }; class Bundled_code_header { @@ -57,14 +56,13 @@ namespace hip_impl static constexpr auto magic_string_sz_ = sizeof(magic_string_) - 1; // DATA - union { + union Header_ { struct { - std::uint8_t bundler_magic_string_[magic_string_sz_]; + char bundler_magic_string_[magic_string_sz_]; std::uint64_t bundle_cnt_; }; - std::uint8_t cbuf_[ - sizeof(bundler_magic_string_) + sizeof(bundle_cnt_)]; - }; + char cbuf_[sizeof(bundler_magic_string_) + sizeof(bundle_cnt_)]; + } header_; std::vector bundles_; // FRIENDS - MANIPULATORS @@ -78,22 +76,24 @@ namespace hip_impl { if (f == l) return false; - std::copy_n(f, sizeof(x.cbuf_), x.cbuf_); + std::copy_n(f, sizeof(x.header_.cbuf_), x.header_.cbuf_); if (valid(x)) { - x.bundles_.resize(x.bundle_cnt_); + x.bundles_.resize(x.header_.bundle_cnt_); - auto it = f + sizeof(x.cbuf_); + auto it = f + sizeof(x.header_.cbuf_); for (auto&& y : x.bundles_) { - std::copy_n(it, sizeof(y.cbuf), y.cbuf); - it += sizeof(y.cbuf); + std::copy_n(it, sizeof(y.header.cbuf), y.header.cbuf); + it += sizeof(y.header.cbuf); - y.triple.insert(y.triple.cend(), it, it + y.triple_sz); + y.triple.assign(it, it + y.header.triple_sz); std::copy_n( - f + y.offset, y.bundle_sz, std::back_inserter(y.blob)); + f + y.header.offset, + y.header.bundle_sz, + std::back_inserter(y.blob)); - it += y.triple_sz; + it += y.header.triple_sz; } return true; @@ -103,7 +103,7 @@ namespace hip_impl } friend inline - bool read(const std::vector& blob, Bundled_code_header& x) + bool read(const std::vector& blob, Bundled_code_header& x) { return read(blob.cbegin(), blob.cend(), x); } @@ -111,7 +111,7 @@ namespace hip_impl inline bool read(std::istream& is, Bundled_code_header& x) { - return read(std::vector{ + return read(std::vector{ std::istreambuf_iterator{is}, std::istreambuf_iterator{}}, x); @@ -123,9 +123,9 @@ namespace hip_impl bool valid(const Bundled_code_header& x) { return std::equal( - x.bundler_magic_string_, - x.bundler_magic_string_ + magic_string_sz_, - x.magic_string_); + magic_string_, + magic_string_ + magic_string_sz_, + x.header_.bundler_magic_string_); } friend inline @@ -139,7 +139,9 @@ namespace hip_impl template Bundled_code_header(RandomAccessIterator f, RandomAccessIterator l); explicit - Bundled_code_header(const std::vector& blob); + Bundled_code_header(const std::vector& blob); + explicit + Bundled_code_header(const void* maybe_blob); Bundled_code_header(const Bundled_code_header&) = default; Bundled_code_header(Bundled_code_header&&) = default; ~Bundled_code_header() = default; diff --git a/projects/hip/samples/0_Intro/module_api/launchKernelHcc.cpp b/projects/hip/samples/0_Intro/module_api/launchKernelHcc.cpp index 7c90198b6a..eb10f64876 100644 --- a/projects/hip/samples/0_Intro/module_api/launchKernelHcc.cpp +++ b/projects/hip/samples/0_Intro/module_api/launchKernelHcc.cpp @@ -35,7 +35,7 @@ THE SOFTWARE. #define LEN 64 #define SIZE LEN<<2 -#define fileName "vcpy_kernel.code" +#define fileName "vcpy_kernel.code.adipose" #define kernel_name "hello_world" #define HIP_CHECK(status) \ @@ -72,7 +72,6 @@ int main(){ uint32_t one = 1; struct { - uint32_t _hidden[6]; void * _Ad; void * _Bd; } args; diff --git a/projects/hip/samples/0_Intro/module_api/runKernel.cpp b/projects/hip/samples/0_Intro/module_api/runKernel.cpp index fb34f80b7b..84dbc395a1 100644 --- a/projects/hip/samples/0_Intro/module_api/runKernel.cpp +++ b/projects/hip/samples/0_Intro/module_api/runKernel.cpp @@ -30,7 +30,7 @@ THE SOFTWARE. #define LEN 64 #define SIZE LEN<<2 -#define fileName "vcpy_kernel.code" +#define fileName "vcpy_kernel.code.adipose" #define kernel_name "hello_world" #define HIP_CHECK(status) \ @@ -68,7 +68,6 @@ int main(){ uint32_t one = 1; struct { - uint32_t _hidden[6]; void * _Ad; void * _Bd; } args; diff --git a/projects/hip/samples/0_Intro/module_api/vcpy_kernel.cpp b/projects/hip/samples/0_Intro/module_api/vcpy_kernel.cpp index b728d67c93..a65d3a2e1e 100644 --- a/projects/hip/samples/0_Intro/module_api/vcpy_kernel.cpp +++ b/projects/hip/samples/0_Intro/module_api/vcpy_kernel.cpp @@ -22,7 +22,7 @@ THE SOFTWARE. #include "hip/hip_runtime.h" -extern "C" __global__ void hello_world(hipLaunchParm lp, float *a, float *b) +extern "C" __global__ void hello_world(float *a, float *b) { int tx = hipThreadIdx_x; b[tx] = a[tx]; diff --git a/projects/hip/samples/0_Intro/module_api_global/runKernel.cpp b/projects/hip/samples/0_Intro/module_api_global/runKernel.cpp index 3f84720edf..74f4fb84e2 100644 --- a/projects/hip/samples/0_Intro/module_api_global/runKernel.cpp +++ b/projects/hip/samples/0_Intro/module_api_global/runKernel.cpp @@ -30,7 +30,7 @@ THE SOFTWARE. #define LEN 64 #define SIZE LEN*sizeof(float) -#define fileName "vcpy_kernel.code" +#define fileName "vcpy_kernel.code.adipose" float myDeviceGlobal; float myDeviceGlobalArray[16]; #define HIP_CHECK(cmd) \ @@ -80,7 +80,6 @@ int main(){ uint32_t one = 1; struct { - uint32_t _hidden[6]; // genco path + wrapper-gen pass use hidden arguments. void * _Ad; void * _Bd; } args; @@ -111,7 +110,7 @@ int main(){ HIP_LAUNCH_PARAM_END }; - { + { hipFunction_t Function; HIP_CHECK(hipModuleGetFunction(&Function, Module, "hello_world")); HIP_CHECK(hipModuleLaunchKernel(Function, 1, 1, 1, LEN, 1, 1, 0, 0, NULL, (void**)&config)); @@ -136,7 +135,7 @@ int main(){ }; } - { + { hipFunction_t Function; HIP_CHECK(hipModuleGetFunction(&Function, Module, "test_globals")); HIP_CHECK(hipModuleLaunchKernel(Function, 1, 1, 1, LEN, 1, 1, 0, 0, NULL, (void**)&config)); diff --git a/projects/hip/samples/0_Intro/module_api_global/vcpy_kernel.cpp b/projects/hip/samples/0_Intro/module_api_global/vcpy_kernel.cpp index f24a261585..8bf67b0fa0 100644 --- a/projects/hip/samples/0_Intro/module_api_global/vcpy_kernel.cpp +++ b/projects/hip/samples/0_Intro/module_api_global/vcpy_kernel.cpp @@ -27,13 +27,13 @@ THE SOFTWARE. extern float myDeviceGlobal; extern float myDeviceGlobalArray[16];; -extern "C" __global__ void hello_world(hipLaunchParm lp, const float *a, float *b) +extern "C" __global__ void hello_world(const float *a, float *b) { int tx = hipThreadIdx_x; b[tx] = a[tx]; } -extern "C" __global__ void test_globals(hipLaunchParm lp, const float *a, float *b) +extern "C" __global__ void test_globals(const float *a, float *b) { int tx = hipThreadIdx_x; b[tx] = a[tx] + myDeviceGlobal+ myDeviceGlobalArray[tx%ARRAY_SIZE] ; diff --git a/projects/hip/src/code_object_bundle.cpp b/projects/hip/src/code_object_bundle.cpp index d7d2cd1e10..aab7200533 100644 --- a/projects/hip/src/code_object_bundle.cpp +++ b/projects/hip/src/code_object_bundle.cpp @@ -2,22 +2,26 @@ #include +#include +#include #include #include #include +using namespace std; + hsa_isa_t hip_impl::triple_to_hsa_isa(const std::string& triple) { static constexpr const char prefix[] = "hcc-amdgcn--amdhsa-gfx"; - static constexpr std::size_t prefix_sz = sizeof(prefix) - 1; + static constexpr size_t prefix_sz = sizeof(prefix) - 1; hsa_isa_t r = {}; auto idx = triple.find(prefix); - if (idx != std::string::npos) { + if (idx != string::npos) { idx += prefix_sz; - std::string tmp = "AMD:AMDGPU"; + string tmp = "AMD:AMDGPU"; while (idx != triple.size()) { tmp.push_back(':'); tmp.push_back(triple[idx++]); @@ -33,7 +37,31 @@ hsa_isa_t hip_impl::triple_to_hsa_isa(const std::string& triple) constexpr const char hip_impl::Bundled_code_header::magic_string_[]; // CREATORS -hip_impl::Bundled_code_header::Bundled_code_header( - const std::vector& x) +hip_impl::Bundled_code_header::Bundled_code_header(const vector& x) : Bundled_code_header{x.cbegin(), x.cend()} -{} \ No newline at end of file +{} + +hip_impl::Bundled_code_header::Bundled_code_header(const void* p) +{ // This is a pretty terrible interface, useful only because + // hipLoadModuleData is so poorly specified (for no fault of its own). + if (!p) return; + + auto ph = static_cast(p); + + if (!equal( + magic_string_, + magic_string_ + magic_string_sz_, + ph->bundler_magic_string_)) { + return; + } + + size_t sz = sizeof(Header_) + ph->bundle_cnt_ * sizeof(Bundled_code::Header); + auto pb = static_cast(p) + sizeof(Header_); + auto n = ph->bundle_cnt_; + while (n--) { + sz += reinterpret_cast(pb)->bundle_sz; + pb += sizeof(Bundled_code::Header); + } + + read(static_cast(p), static_cast(p) + sz, *this); +} \ No newline at end of file diff --git a/projects/hip/src/hip_module.cpp b/projects/hip/src/hip_module.cpp index 45a44b3666..0b1208de91 100644 --- a/projects/hip/src/hip_module.cpp +++ b/projects/hip/src/hip_module.cpp @@ -96,23 +96,6 @@ if (hsaStatus != HSA_STATUS_SUCCESS) {\ return ihipLogStatus(hipStatus);\ } -hipError_t hipModuleLoad(hipModule_t *module, const char *fname) -{ - HIP_INIT_API(module, fname); - - if (!fname) return ihipLogStatus(hipErrorInvalidValue); - - ifstream file{fname}; - - if (!file.is_open()) return ihipLogStatus(hipErrorFileNotFound); - - vector tmp{ - istreambuf_iterator{file}, istreambuf_iterator{}}; - - return hipModuleLoadData(module, tmp.data()); -} - - hipError_t hipModuleUnload(hipModule_t hmod) { HIP_INIT_API(hmod); @@ -473,6 +456,29 @@ namespace return string{s, s + sz}; } + + string code_object_blob_for_agent( + const void* maybe_bundled_code, hsa_agent_t agent) + { + if (!maybe_bundled_code) return {}; + + Bundled_code_header tmp{maybe_bundled_code}; + + if (!valid(tmp)) return {}; + + const auto agent_isa = isa(agent); + + const auto it = find_if( + bundles(tmp).cbegin(), + bundles(tmp).cend(), + [=](const Bundled_code& x) { + return agent_isa == triple_to_hsa_isa(x.triple);; + }); + + if (it == bundles(tmp).cend()) return {}; + + return string{it->blob.cbegin(), it->blob.cend()}; + } } // Anonymous namespace, internal linkage. hipError_t ihipModuleGetFunction( @@ -526,6 +532,22 @@ hipError_t hipModuleGetGlobal(hipDeviceptr_t *dptr, size_t *bytes, return ihipLogStatus(r); } +hipError_t hipModuleLoad(hipModule_t *module, const char *fname) +{ + HIP_INIT_API(module, fname); + + if (!fname) return ihipLogStatus(hipErrorInvalidValue); + + ifstream file{fname}; + + if (!file.is_open()) return ihipLogStatus(hipErrorFileNotFound); + + vector tmp{ + istreambuf_iterator{file}, istreambuf_iterator{}}; + + return hipModuleLoadData(module, tmp.data()); +} + hipError_t hipModuleLoadData(hipModule_t *module, const void *image) { HIP_INIT_API(module, image); @@ -543,8 +565,12 @@ hipError_t hipModuleLoadData(hipModule_t *module, const void *image) nullptr, &(*module)->executable); + auto tmp = code_object_blob_for_agent(image, this_agent()); + (*module)->executable = hip_impl::load_executable( - read_elf_file_as_string(image), (*module)->executable, this_agent()); + tmp.empty() ? read_elf_file_as_string(image) : tmp, + (*module)->executable, + this_agent()); return ihipLogStatus( (*module)->executable.handle ? hipSuccess : hipErrorUnknown); diff --git a/projects/hip/src/hsa_helpers.hpp b/projects/hip/src/hsa_helpers.hpp index d8e09b7aa9..0bb8c09834 100644 --- a/projects/hip/src/hsa_helpers.hpp +++ b/projects/hip/src/hsa_helpers.hpp @@ -24,8 +24,27 @@ THE SOFTWARE. #include #include +#include #include +inline +constexpr +bool operator==(hsa_isa_t x, hsa_isa_t y) +{ + return x.handle == y.handle; +} + +namespace std +{ + template<> + struct hash { + size_t operator()(hsa_isa_t x) const + { + return hash{}(x.handle); + } + }; +} + namespace hip_impl { inline @@ -57,6 +76,19 @@ namespace hip_impl return r; } + inline + hsa_isa_t isa(hsa_agent_t x) + { + hsa_isa_t r = {}; + hsa_agent_iterate_isas(x, [](hsa_isa_t i, void* o) { + *static_cast(o) = i; // Pick the first. + + return HSA_STATUS_INFO_BREAK; + }, &r); + + return r; + } + inline std::uint64_t kernel_object(hsa_executable_symbol_t x) { diff --git a/projects/hip/src/program_state.cpp b/projects/hip/src/program_state.cpp index e867887da2..64e8e832ba 100644 --- a/projects/hip/src/program_state.cpp +++ b/projects/hip/src/program_state.cpp @@ -30,34 +30,16 @@ using namespace ELFIO; using namespace hip_impl; using namespace std; -namespace std -{ - template<> - struct hash { - size_t operator()(hsa_isa_t x) const - { - return hash{}(x.handle); - } - }; -} - -inline -constexpr -bool operator==(hsa_isa_t x, hsa_isa_t y) -{ - return x.handle == y.handle; -} - namespace { struct Symbol { - std::string name; + string name; ELFIO::Elf64_Addr value = 0; - ELFIO::Elf_Xword size = 0; - ELFIO::Elf_Half sect_idx = 0; - std::uint8_t bind = 0; - std::uint8_t type = 0; - std::uint8_t other = 0; + Elf_Xword size = 0; + Elf_Half sect_idx = 0; + uint8_t bind = 0; + uint8_t type = 0; + uint8_t other = 0; }; inline @@ -185,7 +167,7 @@ namespace } } - vector code_object_blob_for_process() + vector code_object_blob_for_process() { static constexpr const char self[] = "/proc/self/exe"; static constexpr const char kernel_section[] = ".kernel"; @@ -200,7 +182,7 @@ namespace return x->get_name() == kernel_section; }); - vector r; + vector r; if (kernels) { r.insert( r.end(), @@ -211,13 +193,13 @@ namespace return r; } - const unordered_map>>& code_object_blobs() + const unordered_map>>& code_object_blobs() { - static unordered_map>> r; + static unordered_map>> r; static once_flag f; call_once(f, []() { - static vector> blobs{ + static vector> blobs{ code_object_blob_for_process()}; dl_iterate_phdr([](dl_phdr_info* info, std::size_t, void*) { @@ -481,7 +463,7 @@ namespace hip_impl const auto code_object_dynsym = find_section_if(reader, [](const ELFIO::section* x) { - return x->get_type() == SHT_DYNSYM; + return x->get_type() == SHT_DYNSYM; }); associate_code_object_symbols_with_host_allocation( diff --git a/projects/hip/tests/src/runtimeApi/synchronization/copy_coherency.cpp b/projects/hip/tests/src/runtimeApi/synchronization/copy_coherency.cpp index b2a66f61e2..d8ff6630e8 100644 --- a/projects/hip/tests/src/runtimeApi/synchronization/copy_coherency.cpp +++ b/projects/hip/tests/src/runtimeApi/synchronization/copy_coherency.cpp @@ -69,7 +69,6 @@ void MemcpyFunction::load(const char *fileName, const char *functionName) void MemcpyFunction::launch(int * dst, const int * src, size_t numElements, hipStream_t s) { struct { - uint32_t _hidden[6]; int* _dst; const int* _src; size_t _numElements;