What is the recommanded directory and install structure for a root lib?

Welcome to the ROOT Forum!
The directory structure of the project is not really related to the install directory (set by CMAKE_INSTALL_PREFIX). in general, I use a directory for the project (project_name) with a source and a build directory. And here is an example of CMakeLists.txt:

# Check if cmake has the required version
CMAKE_MINIMUM_REQUIRED(VERSION 3.16.4 FATAL_ERROR)

set(PROJECT_NAME MyProject)
project(${PROJECT_NAME})

find_package(ROOT REQUIRED)

set(CMAKE_CXX_FLAGS "${ROOT_CXX_FLAGS}" )

include(${ROOT_USE_FILE})
include_directories(${ROOT_INCLUDE_DIRS} ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR})
link_directories(${ROOT_LIBRARY_DIR})

set(HEADERS
  header1.h
  header2.h
  header3.h
)

set(SOURCES
  source1.cxx
  source2.cxx
  source3.cxx
)

ROOT_GENERATE_DICTIONARY(${PROJECT_NAME}Dict.cxx ${HEADERS} LINKDEF LinkDef.h)

add_library(lib${PROJECT_NAME} SHARED ${SOURCES} ${PROJECT_NAME}Dict.cxx)
target_link_libraries(lib${PROJECT_NAME} ${ROOT_LIBRARIES})
if(MSVC)
  set_target_properties(lib${PROJECT_NAME} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE)
endif()
install(TARGETS ${PROJECT_NAME} RUNTIME DESTINATION ${PROJECT_NAME} COMPONENT applications)
install(DIRECTORY data/  DESTINATION ${PROJECT_NAME}/data COMPONENT data)
install(DIRECTORY icons/  DESTINATION ${PROJECT_NAME}/icons COMPONENT icons)

See also How to use CMake to compile with ROOT libraries - #2 by bellenot