6

I had problems getting the node to resync all past ledgers and had to start from scratch. I can see that stellar-core is writing ledgers to disk, but I'm not sure how to check the status of it, or when they will be ready for horizon.

All processes seem to be running fine:

root@stellar:/logs# supervisorctl
horizon                          RUNNING   pid 21, uptime 0:47:25
postgresql                       RUNNING   pid 19, uptime 0:47:25
stellar-core                     RUNNING   pid 20, uptime 0:47:25

From http://localhost:2011/metrics

 {
  "_links": {
    "self": {
      "href": "/metrics"
    }
  },
  "goroutines": {
    "value": 13
  },
  "history.elder_ledger": {
    "value": 0
  },
  "history.latest_ledger": {
    "value": 0
  },
  "history.open_connections": {
    "value": 1
  },
  "ingester.clear_ledger": {
    "15m.rate": 0,
    "1m.rate": 0,
    "5m.rate": 0,
    "75%": 0,
    "95%": 0,
    "99%": 0,
    "99.9%": 0,
    "count": 0,
    "max": 0,
    "mean": 0,
    "mean.rate": 0,
    "median": 0,
    "min": 0,
    "stddev": 0
  },
  "ingester.ingest_ledger": {
    "15m.rate": 0,
    "1m.rate": 0,
    "5m.rate": 0,
    "75%": 0,
    "95%": 0,
    "99%": 0,
    "99.9%": 0,
    "count": 0,
    "max": 0,
    "mean": 0,
    "mean.rate": 0,
    "median": 0,
    "min": 0,
    "stddev": 0
  },
  "logging.debug": {
    "15m.rate": 0,
    "1m.rate": 0,
    "5m.rate": 0,
    "count": 0,
    "mean.rate": 0
  },
  "logging.error": {
    "15m.rate": 0,
    "1m.rate": 0,
    "5m.rate": 0,
    "count": 0,
    "mean.rate": 0
  },
  "logging.info": {
    "15m.rate": 0.10086432581308394,
    "1m.rate": 0.06796872292817097,
    "5m.rate": 0.17705540288221983,
    "count": 424,
    "mean.rate": 0.07895019134610619
  },
  "logging.panic": {
    "15m.rate": 0,
    "1m.rate": 0,
    "5m.rate": 0,
    "count": 0,
    "mean.rate": 0
  },
  "logging.warning": {
    "15m.rate": 1,
    "1m.rate": 1,
    "5m.rate": 1,
    "count": 5371,
    "mean.rate": 1.0001855509718358
  },
  "requests.failed": {
    "15m.rate": 0.002247286557022966,
    "1m.rate": 0.0025507909104836974,
    "5m.rate": 0.003958520624411395,
    "count": 47,
    "mean.rate": 0.008752329109473907
  },
  "requests.succeeded": {
    "15m.rate": 0.04792717165000722,
    "1m.rate": 0.031433570553601776,
    "5m.rate": 0.08456917910523751,
    "count": 164,
    "mean.rate": 0.030540041988960435
  },
  "requests.total": {
    "15m.rate": 0.050174458207030176,
    "1m.rate": 0.03398436146408548,
    "5m.rate": 0.08852769972964883,
    "75%": 2137053,
    "95%": 2986408.8,
    "99%": 3736216.4799999995,
    "99.9%": 5655952,
    "count": 211,
    "max": 5655952,
    "mean": 1533346.5071090048,
    "mean.rate": 0.03929237111868916,
    "median": 1495648,
    "min": 53933,
    "stddev": 846915.7387153778
  },
  "stellar_core.latest_ledger": {
    "value": 1
  },
  "stellar_core.open_connections": {
    "value": 1
  },
  "txsub.buffered": {
    "value": 0
  },
  "txsub.failed": {
    "15m.rate": 0,
    "1m.rate": 0,
    "5m.rate": 0,
    "count": 0,
    "mean.rate": 0
  },
  "txsub.open": {
    "value": 0
  },
  "txsub.succeeded": {
    "15m.rate": 0,
    "1m.rate": 0,
    "5m.rate": 0,
    "count": 0,
    "mean.rate": 0
  },
  "txsub.total": {
    "15m.rate": 0,
    "1m.rate": 0,
    "5m.rate": 0,
    "75%": 0,
    "95%": 0,
    "99%": 0,
    "99.9%": 0,
    "count": 0,
    "max": 0,
    "mean": 0,
    "mean.rate": 0,
    "median": 0,
    "min": 0,
    "stddev": 0
  }
}

Is there a recommended way to view this? I'm unable to run any API calls for now:

http://localhost:2011/ledgers/1000

{
  "type": "https://stellar.org/horizon-errors/before_history",
  "title": "Data Requested Is Before Recorded History",
  "status": 410,
  "detail": "This horizon instance is configured to only track a portion of the stellar network's latest history. This request is asking for results prior to the recorded history known to this horizon instance."
}

UPDATE

The node will not move past ledger 15350312:

"history.latest_ledger": {
    "value": 15350312
  },
 "history.open_connections": {
"value": 1

Am I connecting to the wrong nodes?

My core config:

HTTP_PORT=11626
PUBLIC_HTTP_PORT=true
LOG_FILE_PATH=""

DATABASE="postgresql://dbname=core host=localhost user=stellar password=__PGPASS__"
NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015"
CATCHUP_COMPLETE=true


NODE_NAMES=[
"GAOO3LWBC4XF6VWRP5ESJ6IBHAISVJMSBTALHOQM2EZG7Q477UWA6L7U  eno",
"GAXP5DW4CVCW2BJNPFGTWCEGZTJKTNWFQQBE5SCWNJIJ54BOHR3WQC3W  moni",
"GBFZFQRGOPQC5OEAWO76NOY6LBRLUNH4I5QYPUYAK53QSQWVTQ2D4FT5  dzham",
"GDXWQCSKVYAJSUGR2HBYVFVR7NA7YWYSYK3XYKKFO553OQGOHAUP2PX2  jianing",
"GCJCSMSPIWKKPR7WEPIQG63PDF7JGGEENRC33OKVBSPUDIRL6ZZ5M7OO  tempo.eu.com",
"GCCW4H2DKAC7YYW62H3ZBDRRE5KXRLYLI4T5QOSO6EAMUOE37ICSKKRJ  sparrow_tw",
"GD5DJQDDBKGAYNEAXU562HYGOOSYAEOO6AS53PZXBOZGCP5M2OPGMZV3  fuxi.lab",
"GBGGNBZVYNMVLCWNQRO7ASU6XX2MRPITAGLASRWOWLB4ZIIPHMGNMC4I  huang.lab",
"GDPJ4DPPFEIP2YTSQNOKT7NMLPKU2FFVOEIJMG36RCMBWBUR4GTXLL57  nezha.lab",
"GCDLFPQ76D6YUSCUECLKI3AFEVXFWVRY2RZH2YQNYII35FDECWUGV24T  SnT.Lux",
"GBAR4OY6T6M4P344IF5II5DNWHVUJU7OLQPSMG2FWVJAFF642BX5E3GB  telindus",
# non validating
"GCGB2S2KGYARPVIA37HYZXVRM2YZUEXA6S33ZU5BUDC6THSB62LZSTYH  sdf_watcher1",
"GCM6QMP3DLRPTAZW2UZPCPX2LF3SXWXKPMP3GKFZBDSF3QZGV2G5QSTK  sdf_watcher2",
"GABMKJM6I25XI4K7U6XWMULOUQIQ27BCTMLS6BYYSOWKTBUXVRJSXHYQ  sdf_watcher3",
# seem down
"GB6REF5GOGGSEHZ3L2YK6K4T4KX3YDMWHDCPMV7MZJDLHBDNZXEPRBGM  donovan",
"GBGR22MRCIVW2UZHFXMY5UIBJGPYABPQXQ5GGMNCSUM2KHE3N6CNH6G5  nelisky1",
"GA2DE5AQF32LU5OZ5OKAFGPA2DLW4H6JHPGYJUVTNS3W7N2YZCTQFFV6  nelisky2",
"GDJ73EX25GGUVMUBCK6DPSTJLYP3IC7I3H2URLXJQ5YP56BW756OUHIG  w00kie",
"GAM7A32QZF5PJASRSGVFPAB36WWTHCBHO5CHG3WUFTUQPT7NZX3ONJU4  ptarasov"
]

KNOWN_PEERS=[
"core-live-a.stellar.org:11625",
"core-live-b.stellar.org:11625",
"core-live-c.stellar.org:11625",
"confucius.strllar.org",
"stellar1.bitventure.co",
"stellar.256kw.com"]

[QUORUM_SET]
VALIDATORS=[
"$sdf_watcher1","$eno","$tempo.eu.com","$sdf_watcher2","$sdf_watcher3"
]

[HISTORY.cache]
get="cp /opt/stellar/history-cache/{0} {1}"

# Stellar.org history store
[HISTORY.sdf1]
get="curl -sf http://history.stellar.org/prd/core-live/core_live_001/{0} -o {1}"

[HISTORY.sdf2]
get="curl -sf http://history.stellar.org/prd/core-live/core_live_002/{0} -o {1}"

[HISTORY.sdf3]
get="curl -sf http://history.stellar.org/prd/core-live/core_live_003/{0} -o {1}"
Improve this question

3 Answers 3

Reset to default
1

It looks like I was able to find the stellar-core endpoint for this:

  root@stellar:/# curl 127.0.0.1:11626/info
{
   "info" : {
      "build" : "v9.2.0",
      "ledger" : {
         "age" : 1532468251,
         "baseFee" : 100,
         "baseReserve" : 100000000,
         "closeTime" : 0,
         "hash" : "39c2a3cd4141b2853e70d84601faa44744660334b48f3228e0309342e3f4eb48",
         "num" : 2,
         "version" : 0
      },
      "network" : "Public Global Stellar Network ; September 2015",
      "peers" : {
         "authenticated_count" : 8,
         "pending_count" : 0
      },
      "protocol_version" : 9,
      "quorum" : {
         "19098965" : {
            "agree" : 5,
            "disagree" : 0,
            "fail_at" : 2,
            "hash" : "ba2fc8",
            "missing" : 0,
            "phase" : "EXTERNALIZE"
         }
      },
      "startedOn" : "2018-07-24T19:17:51Z",
      "state" : "Catching up",
      "status" : [ "Catching up: downloading ledger files 128202/298397 (42%)" ]
   }
}
Improve this answer
1

As I read from your supervisor uptime, it seems your instance had run for less than an hour, which is rather new. It takes many hours to finish "catch up" with the running network.

You may check your Horizon main page http://localhost:2011/, and look for information at the bottom like: 

"history_latest_ledger": 19098965,
"history_elder_ledger": 124,

This range (in my case, #124 - #19098965) indicates the data that my Horizon currently has.

As you indicated "title": "Data Requested Is Before Recorded History" for ledger #1000, it is likely #1000 is out of the above range in your case.

If the problem does not solve itself (after hours from your posting time), please post your Core config and also the values for "history_latest_ledger" and "history_elder_ledger".

Improve this answer
2
  • My node isn't able to get past "num" : 15350313,. It keeps downloading ledger files and then transactions, but it doesn't move past that ledger for some reason. "status" : [ "Catching up: downloading ledger files 24886/63390 (39%)" ] Is there a way to get around this? new-db?
    – relik
    Aug 11, 2018 at 19:06
  • From http://localhost:2011/metrics "history.elder_ledger": { "value": 6 }, "history.latest_ledger": { "value": 15350312 },
    – relik
    Aug 11, 2018 at 19:26
1

Based on supervisorctl you are running the docker/quickstart image.

I had a similar issue recently. The cathup was running in circles and could not finish in days. I found the real issue by setting the LOG_FILE_PATH variable in core config and read the log for more information.

I had the DB in a corrupt state and the core could not apply the transactions at a point. I had to clear the DB and restart the cathup process from scratch.

You could speed up (unfortunately not too much) the catchup process by using stellar-archivist and download the archive locally to your server first. Then you just need to copy/mount this archive into a folder that the stellar-core is using for archive. In the docker quickstart this is /opt/stellar/history-cache by default. In this way stellar-core does not need to download the whole archive every time you start from scratch.

Also, please note that the whole cathup process from scratch takes about 3 days (measured on a dedicated server with SSD and 4 * Xeon cores with 16GB Mem).

Improve this answer

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge that you have read and understand our privacy policy and code of conduct.

Not the answer you're looking for? Browse other questions tagged or ask your own question.