How to avoid RegionDestroyedException in WAN receiver site when using member group specific regions

Products

• VMware Tanzu Gemfire

Issue/Introduction

Given a scenario where WAN replication between GemFire clusters is enabled, each cluster contains multiple member groups of CacheServers and some of the regions being replicated exist only in members of a particular group; you may see RegionDestroyedExceptions in receiving-side CacheServer logs.

For example:

Sender cluser 1:
groupA -- member 1 - region X (Instrument)
groupB -- member 2 - region Y (Trade)

Receiver cluser 2:
groupA -- member 1 - region X (Instrument)
groupB -- member 2 - region Y (Trade)

Using the above configuration, you would likely find RegionDestroyedExceptions in the CacheServers's logs on the receiver site like below:

[ServerConnection on port 1571 Thread 130 : 2021-01-01 03:14:31,885] BaseCommand WARN - Server connection from [identity(192.###.##.###(SenderGemfireServer1:32323)<v7>:41002,connection=1; port=38228]: Wrote batch exception:

org.apache.geode.internal.cache.wan.BatchException70: Exception occurred while processing a batch on the receiver running on DistributedSystem with Id: 2, DistributedMember on which the receiver is running: 192.###.##.##(ReceiverGemfireServer2:59287)<v2>:41001
...
Caused by: org.apache.geode.cache.RegionDestroyedException: Region /Instrument was not found during batch create request 2709

at org.apache.geode.internal.cache.tier.sockets.command.GatewayReceiverCommand.handleRegionNull(GatewayReceiverCommand.java:77) ~[geode-core-9.9.1.jar:?]

at org.apache.geode.internal.cache.tier.sockets.command.GatewayReceiverCommand.cmdExecute(GatewayReceiverCommand.java:404) ~[geode-core-9.9.1.jar:?]

... 9 more

Resolution

At this time, WAN replication doesn't support member groups so gateway senders will connect to gateway receivers randomly you will see a "RegionDestroyedException: Region /XXX was not found" exception in the receiver site whenever the sender for a region connects to a receiver that does not host that region. The workaround to avoid this kind of exception is to put proxy partition regions on the receiver CacheServers for each of the other group's data regions.

Here is the detailed example on how to work around this issue:

gfsh script: 
# Set screen width
set variable --name=APP_RESULT_VIEWER --value=any
 
###############
# Site London #
############### 
# Start locator
start locator --name=locator-ln --port=10331 --locators=localhost[10331] --mcast-port=0 --J=-Dgemfire.remote-locators=localhost[10332] --J=-Dgemfire.distributed-system-id=1 --J=-Dgemfire.jmx-manager-start=true --J=-Dgemfire.jmx-manager-http-port=8081 --J=-Dgemfire.jmx-manager-port=1091
 
# Configure PDX
configure pdx --read-serialized=true
 
# Start Server in instrument group
start server --groups=instrument --name=server-ln-1 --locators=localhost[10331] --server-port=0 --statistic-archive-file=cacheserver.gfs --J=-Dgemfire.log-file=cacheserver.log --J=-Dgemfire.conserve-sockets=false
 
# Start Server in trade group
start server --groups=trade --name=server-ln-2 --locators=localhost[10331] --server-port=0 --statistic-archive-file=cacheserver.gfs --J=-Dgemfire.log-file=cacheserver.log --J=-Dgemfire.conserve-sockets=false
 
# Create Gateway Sender in instrument group
create gateway-sender --group=instrument --id=ny-instrument --remote-distributed-system-id=2 --parallel=true
 
# Create Gateway Sender in trade group
create gateway-sender --group=trade --id=ny-trade --remote-distributed-system-id=2 --parallel=true
 
# Sleep to enable Gateway Senders to be known by the locator through JMX
sleep --time=5
 
# Create Regions in instrument group
create region --group=instrument --name=Instrument --type=PARTITION_REDUNDANT --gateway-sender-id=ny-instrument
create region --group=instrument --name=Trade --type=PARTITION_PROXY_REDUNDANT --gateway-sender-id=ny-trade
 
# Create Regions in trade group
create region --group=trade --name=Instrument --type=PARTITION_PROXY_REDUNDANT --gateway-sender-id=ny-instrument
create region --group=trade --name=Trade --type=PARTITION_REDUNDANT --gateway-sender-id=ny-trade
 
# Disconnect from locator
disconnect
 
################
# Site NewYork #
################
 
# Start locator
start locator --name=locator-ny --port=10332 --locators=localhost[10332] --mcast-port=0 --J=-Dgemfire.remote-locators=localhost[10331] --J=-Dgemfire.distributed-system-id=2 --J=-Dgemfire.jmx-manager-start=true --J=-Dgemfire.jmx-manager-http-port=8082 --J=-Dgemfire.jmx-manager-port=1092
 
# Configure PDX
configure pdx --read-serialized=true
 
# Start Server in instrument group
start server --groups=instrument --name=server-ny-1 --locators=localhost[10332] --server-port=0 --statistic-archive-file=cacheserver.gfs --J=-Dgemfire.log-file=cacheserver.log --J=-Dgemfire.conserve-sockets=false
 
# Start Server in trade group
start server --groups=trade --name=server-ny-2 --locators=localhost[10332] --server-port=0 --statistic-archive-file=cacheserver.gfs --J=-Dgemfire.log-file=cacheserver.log --J=-Dgemfire.conserve-sockets=false
 
# Start GatewayReceiver
create gateway-receiver
 
# Create Regions in instrument group
create region --group=instrument --name=Instrument --type=PARTITION_REDUNDANT
create region --group=instrument --name=Trade --type=PARTITION_PROXY_REDUNDANT
 
# Create Regions in trade group
create region --group=trade --name=Instrument --type=PARTITION_PROXY_REDUNDANT
create region --group=trade --name=Trade --type=PARTITION_REDUNDANT
 
# Disconnect from locator
disconnect
 
That generates xml like:
 
Site london
---------------
Configuration for  'Instrument'
 
<cache>
    <gateway-sender id="ny-instrument" parallel="true" remote-distributed-system-id="2"/>
    <region name="Instrument" refid="PARTITION_REDUNDANT">
        <region-attributes data-policy="partition" gateway-sender-ids="ny-instrument">
            <partition-attributes redundant-copies="1"/>
        </region-attributes>
    </region>
    <region name="Trade" refid="PARTITION_PROXY_REDUNDANT">
        <region-attributes data-policy="partition" gateway-sender-ids="ny-trade">
            <partition-attributes local-max-memory="0" redundant-copies="1"/>
        </region-attributes>
    </region>
</cache>
 
 
Configuration for  'Trade'
 
<cache>
    <gateway-sender id="ny-trade" parallel="true" remote-distributed-system-id="2"/>
    <region name="Instrument" refid="PARTITION_PROXY_REDUNDANT">
        <region-attributes data-policy="partition" gateway-sender-ids="ny-instrument">
            <partition-attributes local-max-memory="0" redundant-copies="1"/>
        </region-attributes>
    </region>
    <region name="Trade" refid="PARTITION_REDUNDANT">
        <region-attributes data-policy="partition" gateway-sender-ids="ny-trade">
            <partition-attributes redundant-copies="1"/>
        </region-attributes>
    </region>
</cache>
 
Site NewYork
---------------
Configuration for  'Instrument'
 
<cache>
    <gateway-receiver/>
    <region name="Instrument" refid="PARTITION_REDUNDANT">
        <region-attributes data-policy="partition">
            <partition-attributes redundant-copies="1"/>
        </region-attributes>
    </region>
    <region name="Trade" refid="PARTITION_PROXY_REDUNDANT">
        <region-attributes data-policy="partition">
            <partition-attributes local-max-memory="0" redundant-copies="1"/>
        </region-attributes>
    </region>
</cache>
 
Configuration for  'Trade'
 
<cache>
    <gateway-receiver/>
    <region name="Instrument" refid="PARTITION_PROXY_REDUNDANT">
        <region-attributes data-policy="partition">
            <partition-attributes local-max-memory="0" redundant-copies="1"/>
        </region-attributes>
    </region>
    <region name="Trade" refid="PARTITION_REDUNDANT">
        <region-attributes data-policy="partition">
            <partition-attributes redundant-copies="1"/>
        </region-attributes>
    </region>
</cache>