Finally I was able to solve my problem with some research:

The idea is simple, since addresses has a fixed length (20 bytes), so we can simple pack them next to each other and unpack it unambiguously. 

Example packing:

```
address1:
0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
address2:
0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb

packed:
0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb

```
This tight packing can be achieved in Solidity by using `abi.encodePacked()`. You can find it here in the docs: https://docs.soliditylang.org/en/v0.5.3/abi-spec.html#non-standard-packed-mode

To pack any number of addresses I simple used it in a loop in the following way (may not be the most gas effective method):
```solidity
function encodeAddressArray(address[] calldata addresses) external pure returns(bytes memory data){
    for(uint i=0; i<addresses.length; i++){
        data = abi.encodePacked(data, addresses[i]);
    }
}
```

To unpack simply take the first 20 bytes, which gives the first address and the second 20 bytes gives the second address and so on. For that I used the array slice. (Keep in mind, that slicing only works on `calldata` and `x[start:end]` includes `x[start]` but excludes `x[end]`.)

```solidity
function decodeAddressArray(bytes calldata data)external pure returns(address[] memory addresses){
    uint n = data.length/20;
    addresses = new address[](n);
        
    for(uint i=0; i<n; i++){
        addresses[i] = bytesToAddress(data[i*20:(i+1)*20]);
    }
}
```

where `bytesToAddress()` converts one address from bytes. This can be done by loading the corresponding bytes into the address variable using `mload()` in assembly:

```solidity
function bytesToAddress(bytes calldata data) private pure returns (address addr) {
    bytes memory b = data;
    assembly {
        addr := mload(add(b, 20))
    } 
}
```

Finally the whole process is summarized in the following tiny library:

```solidity

library AddressCoder{
    function bytesToAddress(bytes calldata data) private pure returns (address addr) {
        bytes memory b = data;
        assembly {
          addr := mload(add(b, 20))
        } 
    }

    
    function decodeAddressArray(bytes calldata data)external pure returns(address[] memory addresses){
        uint n = data.length/20;
        addresses = new address[](n);
        
        for(uint i=0; i<n; i++){
            addresses[i] = bytesToAddress(data[i*20:(i+1)*20]);
        }
    }
    
    
    function encodeAddressArray(address[] calldata addresses) external pure returns(bytes memory data){
        for(uint i=0; i<addresses.length; i++){
            data = abi.encodePacked(data, addresses[i]);
        }
    }
}
```

This encoding is also pretty simple in `web3.js`:
```javascript
const encodeAddressArray = addresses => {
    let hex = "0x";
    hex += addresses.map(address => address.substr(2, 40)).join("");

    return web3.utils.hexToBytes(hex);
}
```