How to link my project library to my pybind module?

I have a cpp program called my_exec.

I want to create a python-binding for it. (with pybind11)

In my exec I have 1 simple function :

int add(int i, int j)
{
    return i+j;
}

And this is my pybind .cpp file :

#include <pybind11/pybind11.h>
#include "lib.h" // my add function is defined here and implement in lib.cpp

PYBIND11_MODULE(cpp, m)
{
    m.doc() = "first python binding.";
    m.def("add", &add, "A function that adds 2 numbers");
    m.def("mul", &mul, "A function that multiply 2 numbers");
}

So here is my problem :
I can’t build my pybind module with CMake because I can’t (I don’t know how) link my program my_exec (containing the add function implementation) to it.

This is my CMakeLists.txt file :

# only for cmake --version >= 3.5.1
cmake_minimum_required(VERSION 3.5.1)

if (CMAKE_BUILD_TYPE MATCHES Debug)
    message("Cmake in debug mode.")
else ()
    message("Cmake in release mode.")
endif (CMAKE_BUILD_TYPE MATCHES Debug)

# project name
project(my_exec)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)

#lsp file
set(CMAKE_EXPORT_COMPILE_COMMANDS 1)

# I../includes
include_directories(
src 
/media/romain/Donnees/Programmation/C++/frameworks
/media/romain/Donnees/Programmation/C++/frameworks/files
/media/romain/Donnees/Programmation/C++/libs/json
/media/romain/Donnees/Programmation/C++/libs/boost-install/include
)

#Link directories
link_directories(build/libs)

# puts all .cpp files inside src to the SOURCES variable
file(GLOB SOURCES src/*.cpp 
/media/romain/Donnees/Programmation/C++/frameworks/debug.cpp
/media/romain/Donnees/Programmation/C++/frameworks/str.cpp
)

if (CMAKE_BUILD_TYPE MATCHES Debug)
    add_compile_definitions(mydebug)
    add_compile_options(-Og)
else()
    add_compile_options(-O3)
endif(CMAKE_BUILD_TYPE MATCHES Debug)

# compiles the files defined by SOURCES to generate the executable defined by EXEC
add_executable(${PROJECT_NAME} ${SOURCES})

add_subdirectory(pybind) #where is the pybind CMakeLists file

#make the executable linkable by other libs (runtime ones - here for my module pybind)
set_target_properties(${PROJECT_NAME} PROPERTIES ENABLE_EXPORTS on)

# linkers
target_link_libraries(${PROJECT_NAME} 
pthread
stdc++fs
)

And this is my pybind CMakeLists.txt (locate in the subdir pybind) :

cmake_minimum_required(VERSION 3.5)
project(pybind)
include_directories(/media/romain/Donnees/Programmation/C++/libs/pybind11/include)
add_subdirectory(/media/romain/Donnees/Programmation/C++/libs/pybind11
    ./pybind11_build)


pybind11_add_module(cpp
    ../src/pybind/cpp.cpp)

target_link_libraries(${PROJECT_NAME} PRIVATE 
    my_exec)

I can’t compile because CMake give a error when trying to link my_exec :

Cannot specify link libraries for target "pybind" which is not built by
  this project.

So, how to link an external lib/exec to a pybind Cmake project?

EDIT:

---

Thanks to @starball anwser, I managed to compiled.
But unfortunatly, I think python does not support module linked to an executable.
It has to be a static or shared library.

So I had to compile my program my_exec as a library like that :

add_library(my_exec SHARED ${SOURCES})